diff --git a/src/Forms/GridField/GridFieldExportButton.php b/src/Forms/GridField/GridFieldExportButton.php index 9fc3ad3f36a..9e56b76aad2 100644 --- a/src/Forms/GridField/GridFieldExportButton.php +++ b/src/Forms/GridField/GridFieldExportButton.php @@ -223,6 +223,10 @@ public function generateExportFileData($gridField) // Remove limit as the list may be paginated, we want the full list for the export $items = $items->limit(null); + // Use Generator in applicable cases to reduce memory consumption + $items = $items instanceof DataList + ? $items->getGenerator() + : $items; /** @var DataObject $item */ foreach ($items as $item) { diff --git a/src/ORM/ArrayList.php b/src/ORM/ArrayList.php index d8dca3195fe..74a7cd600ea 100644 --- a/src/ORM/ArrayList.php +++ b/src/ORM/ArrayList.php @@ -2,8 +2,8 @@ namespace SilverStripe\ORM; +use ArrayIterator; use InvalidArgumentException; -use Iterator; use LogicException; use SilverStripe\Dev\Debug; use SilverStripe\Dev\Deprecation; @@ -103,16 +103,19 @@ public function exists() /** * Returns an Iterator for this ArrayList. * This function allows you to use ArrayList in foreach loops + * + * @return ArrayIterator */ - public function getIterator(): Iterator - { - foreach ($this->items as $i => $item) { - if (is_array($item)) { - yield new ArrayData($item); - } else { - yield $item; - } - } + #[\ReturnTypeWillChange] + public function getIterator() + { + $items = array_map( + function ($item) { + return is_array($item) ? new ArrayData($item) : $item; + }, + $this->items ?? [] + ); + return new ArrayIterator($items); } /** diff --git a/src/ORM/Connect/DBSchemaManager.php b/src/ORM/Connect/DBSchemaManager.php index 7a10cef8e93..7f93148cbd6 100644 --- a/src/ORM/Connect/DBSchemaManager.php +++ b/src/ORM/Connect/DBSchemaManager.php @@ -378,7 +378,7 @@ public function requireTable( if ($dbID && isset($options[$dbID])) { if (preg_match('/ENGINE=([^\s]*)/', $options[$dbID] ?? '', $alteredEngineMatches)) { $alteredEngine = $alteredEngineMatches[1]; - $tableStatus = $this->query(sprintf('SHOW TABLE STATUS LIKE \'%s\'', $table))->record(); + $tableStatus = $this->query(sprintf('SHOW TABLE STATUS LIKE \'%s\'', $table))->first(); $tableOptionsChanged = ($tableStatus['Engine'] != $alteredEngine); } } diff --git a/src/ORM/Connect/MySQLQuery.php b/src/ORM/Connect/MySQLQuery.php index d019167014b..c811386430a 100644 --- a/src/ORM/Connect/MySQLQuery.php +++ b/src/ORM/Connect/MySQLQuery.php @@ -2,8 +2,6 @@ namespace SilverStripe\ORM\Connect; -use Iterator; - /** * A result-set from a MySQL database (using MySQLiConnector) * Note that this class is only used for the results of non-prepared statements @@ -47,13 +45,16 @@ public function __destruct() } } - public function getIterator(): Iterator + public function seek($row) { if (is_object($this->handle)) { - while ($data = $this->handle->fetch_assoc()) { - yield $data; - } + // Fix for https://github.com/silverstripe/silverstripe-framework/issues/9097 without breaking the seek() API + $this->handle->data_seek($row); + $result = $this->nextRecord(); + $this->handle->data_seek($row); + return $result; } + return null; } public function numRecords() @@ -61,7 +62,27 @@ public function numRecords() if (is_object($this->handle)) { return $this->handle->num_rows; } - return null; } + + public function nextRecord() + { + $floatTypes = [MYSQLI_TYPE_FLOAT, MYSQLI_TYPE_DOUBLE, MYSQLI_TYPE_DECIMAL, MYSQLI_TYPE_NEWDECIMAL]; + + if (is_object($this->handle) && ($row = $this->handle->fetch_array(MYSQLI_NUM))) { + $data = []; + foreach ($row as $i => $value) { + if (!isset($this->columns[$i])) { + throw new DatabaseException("Can't get metadata for column $i"); + } + if (in_array($this->columns[$i]->type, $floatTypes ?? [])) { + $value = (float)$value; + } + $data[$this->columns[$i]->name] = $value; + } + return $data; + } else { + return false; + } + } } diff --git a/src/ORM/Connect/MySQLStatement.php b/src/ORM/Connect/MySQLStatement.php index eada8ad407b..7bc54765fcd 100644 --- a/src/ORM/Connect/MySQLStatement.php +++ b/src/ORM/Connect/MySQLStatement.php @@ -2,7 +2,6 @@ namespace SilverStripe\ORM\Connect; -use Iterator; use mysqli_result; use mysqli_stmt; @@ -57,26 +56,6 @@ class MySQLStatement extends Query */ protected $boundValues = []; - /** - * Hook the result-set given into a Query class, suitable for use by SilverStripe. - * @param mysqli_stmt $statement The related statement, if present - * @param mysqli_result $metadata The metadata for this statement - */ - public function __construct($statement, $metadata) - { - $this->statement = $statement; - $this->metadata = $metadata; - - // Immediately bind and buffer - $this->bind(); - } - - public function __destruct() - { - $this->statement->close(); - $this->currentRecord = false; - } - /** * Binds this statement to the variables */ @@ -103,20 +82,58 @@ protected function bind() call_user_func_array([$this->statement, 'bind_result'], $variables ?? []); } - public function getIterator(): Iterator + /** + * Hook the result-set given into a Query class, suitable for use by SilverStripe. + * @param mysqli_stmt $statement The related statement, if present + * @param mysqli_result $metadata The metadata for this statement + */ + public function __construct($statement, $metadata) + { + $this->statement = $statement; + $this->metadata = $metadata; + + // Immediately bind and buffer + $this->bind(); + } + + public function __destruct() { - while ($this->statement->fetch()) { - // Dereferenced row - $row = []; - foreach ($this->boundValues as $key => $value) { - $row[$key] = $value; - } - yield $row; - } + $this->statement->close(); + $this->currentRecord = false; + } + + public function seek($row) + { + $this->rowNum = $row - 1; + + // Fix for https://github.com/silverstripe/silverstripe-framework/issues/9097 without breaking the seek() API + $this->statement->data_seek($row); + $result = $this->next(); + $this->statement->data_seek($row); + return $result; } public function numRecords() { return $this->statement->num_rows(); } + + public function nextRecord() + { + // Skip data if out of data + if (!$this->statement->fetch()) { + return false; + } + + // Dereferenced row + $row = []; + foreach ($this->boundValues as $key => $value) { + $floatTypes = [MYSQLI_TYPE_FLOAT, MYSQLI_TYPE_DOUBLE, MYSQLI_TYPE_DECIMAL, MYSQLI_TYPE_NEWDECIMAL]; + if (in_array($this->types[$key], $floatTypes ?? [])) { + $value = (float)$value; + } + $row[$key] = $value; + } + return $row; + } } diff --git a/src/ORM/Connect/PDOQuery.php b/src/ORM/Connect/PDOQuery.php index 24f9712eb8a..7180de28d0e 100644 --- a/src/ORM/Connect/PDOQuery.php +++ b/src/ORM/Connect/PDOQuery.php @@ -2,9 +2,6 @@ namespace SilverStripe\ORM\Connect; -use ArrayIterator; -use Iterator; - /** * A result-set from a PDO database. */ @@ -17,7 +14,7 @@ class PDOQuery extends Query /** * Hook the result-set given into a Query class, suitable for use by SilverStripe. - * @param PDOStatementHandle $statement The internal PDOStatement containing the results + * @param PDOStatement $statement The internal PDOStatement containing the results */ public function __construct(PDOStatementHandle $statement) { @@ -29,13 +26,25 @@ public function __construct(PDOStatementHandle $statement) $statement->closeCursor(); } - public function getIterator(): Iterator + public function seek($row) { - return new ArrayIterator($this->results); + $this->rowNum = $row - 1; + return $this->nextRecord(); } public function numRecords() { - return count($this->results); + return count($this->results ?? []); + } + + public function nextRecord() + { + $index = $this->rowNum + 1; + + if (isset($this->results[$index])) { + return $this->results[$index]; + } else { + return false; + } } } diff --git a/src/ORM/Connect/Query.php b/src/ORM/Connect/Query.php index 8aac231b2b5..af9e0c3a540 100644 --- a/src/ORM/Connect/Query.php +++ b/src/ORM/Connect/Query.php @@ -27,9 +27,30 @@ * on providing the specific data-access methods that are required: {@link nextRecord()}, {@link numRecords()} * and {@link seek()} */ -abstract class Query implements \IteratorAggregate +abstract class Query implements Iterator { + /** + * The current record in the iterator. + * + * @var array + */ + protected $currentRecord = null; + + /** + * The number of the current row in the iterator. + * + * @var int + */ + protected $rowNum = -1; + + /** + * Flag to keep track of whether iteration has begun, to prevent unnecessary seeks + * + * @var bool + */ + protected $queryHasBegun = false; + /** * Return an array containing all the values from a specific column. If no column is set, then the first will be * returned @@ -41,7 +62,7 @@ public function column($column = null) { $result = []; - foreach ($this as $record) { + while ($record = $this->next()) { if ($column) { $result[] = $record[$column]; } else { @@ -61,7 +82,6 @@ public function column($column = null) public function keyedColumn() { $column = []; - foreach ($this as $record) { $val = $record[key($record)]; $column[$val] = $val; @@ -86,22 +106,13 @@ public function map() } /** - * Returns the first record in the result + * Returns the next record in the iterator. * * @return array */ public function record() { - return $this->getIterator()->current(); - } - - /** - * @deprecated Use record() instead - * @return array - */ - public function first() - { - return $this->record(); + return $this->next(); } /** @@ -111,7 +122,7 @@ public function first() */ public function value() { - $record = $this->record(); + $record = $this->next(); if ($record) { return $record[key($record)]; } @@ -153,10 +164,94 @@ public function table() return $result; } + /** + * Iterator function implementation. Rewind the iterator to the first item and return it. + * Makes use of {@link seek()} and {@link numRecords()}, takes care of the plumbing. + * + * @return void + */ + #[\ReturnTypeWillChange] + public function rewind() + { + if ($this->queryHasBegun && $this->numRecords() > 0) { + $this->queryHasBegun = false; + $this->currentRecord = null; + $this->seek(0); + } + } + + /** + * Iterator function implementation. Return the current item of the iterator. + * + * @return array + */ + #[\ReturnTypeWillChange] + public function current() + { + if (!$this->currentRecord) { + return $this->next(); + } else { + return $this->currentRecord; + } + } + + /** + * Iterator function implementation. Return the first item of this iterator. + * + * @return array + */ + public function first() + { + $this->rewind(); + return $this->current(); + } + + /** + * Iterator function implementation. Return the row number of the current item. + * + * @return int + */ + #[\ReturnTypeWillChange] + public function key() + { + return $this->rowNum; + } + + /** + * Iterator function implementation. Return the next record in the iterator. + * Makes use of {@link nextRecord()}, takes care of the plumbing. + * + * @return array + */ + #[\ReturnTypeWillChange] + public function next() + { + $this->queryHasBegun = true; + $this->currentRecord = $this->nextRecord(); + $this->rowNum++; + return $this->currentRecord; + } + + /** + * Iterator function implementation. Check if the iterator is pointing to a valid item. + * + * @return bool + */ + #[\ReturnTypeWillChange] + public function valid() + { + if (!$this->queryHasBegun) { + $this->next(); + } + return $this->currentRecord !== false; + } + /** * Return the next record in the query result. + * + * @return array */ - abstract public function getIterator(): Iterator; + abstract public function nextRecord(); /** * Return the total number of items in the query result. @@ -164,4 +259,12 @@ abstract public function getIterator(): Iterator; * @return int */ abstract public function numRecords(); + + /** + * Go to a specific row number in the query result and return the record. + * + * @param int $rowNum Row number to go to. + * @return array + */ + abstract public function seek($rowNum); } diff --git a/src/ORM/DataList.php b/src/ORM/DataList.php index 8dffc1dfe2b..72dfaf02ef2 100644 --- a/src/ORM/DataList.php +++ b/src/ORM/DataList.php @@ -6,12 +6,10 @@ use SilverStripe\Dev\Debug; use SilverStripe\ORM\Filters\SearchFilter; use SilverStripe\ORM\Queries\SQLConditionGroup; -use SilverStripe\View\TemplateIterator; use SilverStripe\View\ViewableData; use ArrayIterator; use Exception; use InvalidArgumentException; -use Iterator; use LogicException; /** @@ -51,13 +49,6 @@ class DataList extends ViewableData implements SS_List, Filterable, Sortable, Li */ protected $dataQuery; - /** - * A cached Query to save repeated database calls. {@see DataList::getTemplateIteratorCount()} - * - * @var SilverStripe\ORM\Connect\Query - */ - protected $finalisedQuery; - /** * Create a new DataList. * No querying is done on construction, but the initial query schema is set up. @@ -88,7 +79,6 @@ public function dataClass() public function __clone() { $this->dataQuery = clone $this->dataQuery; - $this->finalisedQuery = null; } /** @@ -791,6 +781,20 @@ public function each($callback) return $this; } + /** + * Returns a generator for this DataList + * + * @return \Generator&DataObject[] + */ + public function getGenerator() + { + $query = $this->dataQuery->query()->execute(); + + while ($row = $query->record()) { + yield $this->createDataObject($row); + } + } + public function debug() { $val = "

" . static::class . "