[BUGFIX] Reestablish manipulation of EXT:indexed_search query

Due to huge performance implications on
searching bigger sites, it's now possible
again to manipulate the QueryBuilder instance,
before the final indexed search query is executed.
This achieved by introducing a new PSR-14 event.

Additionally a @todo is resolved by adjusting
the repository to execute the final query
centrally in `doSearch()`.

Resolves: #105007
Related: #97530
Related: #102937
Releases: main, 13.4
Change-Id: Ibf428be5f3554010fb9a18e8d030f7428ce5d954
Reviewed-on: https://review.typo3.org/c/Packages/TYPO3.CMS/+/86593
Tested-by: Garvin Hicking <[email protected]>
Reviewed-by: Benni Mack <[email protected]>
Reviewed-by: Garvin Hicking <[email protected]>
Tested-by: Oliver Bartsch <[email protected]>
Reviewed-by: Oliver Bartsch <[email protected]>
Tested-by: Benni Mack <[email protected]>
Tested-by: core-ci <[email protected]>

typo3/sysext/core/Documentation/Changelog/13.4.x/Important-105007-ManipulationOfFinalSearchQueryInEXTindexed_search.rst

@@ -0,0 +1,63 @@
+..  include:: /Includes.rst.txt
+
+..  _important-105007-1728977233:
+
+=============================================================================
+Important: #105007 - Manipulation of final search query in EXT:indexed_search
+=============================================================================
+
+See :issue:`105007`
+
+Description
+===========
+
+By removing the :typoscript:`searchSkipExtendToSubpagesChecking` option in
+:issue:`97530`, there might have been performance implications for installations
+with a lot of sites. This could be circumvented by adjusting the search query
+manually, using available hooks. Since those hooks have also been removed with
+:issue:`102937`, developers were no longer be able to handle the query
+behaviour.
+
+Therefore, the PSR-14 :php:`BeforeFinalSearchQueryIsExecutedEvent` has been
+introduced which allows developers to manipulate the :php:`QueryBuilder`
+instance again, just before the query gets executed.
+
+Additional context information, provided by the new event:
+
+* :php:`searchWords` - The corresponding search words list
+* :php:`freeIndexUid` - Pointer to which indexing configuration should be searched in. 
+  -1 means no filtering. 0 means only regular indexed content.
+
+.. important::
+
+    The provided query (the :php:`QueryBuilder` instance) is controlled by
+    TYPO3 and is not considered public API. Therefore, developers using this
+    event need to keep track of underlying changes by TYPO3. Such changes might
+    be further performance improvements to the query or changes to the
+    database schema in general.
+
+Example
+=======
+
+..  code-block:: php
+
+    <?php
+    declare(strict_types=1);
+
+    namespace MyVendor\MyExtension\EventListener;
+
+    use TYPO3\CMS\Core\Attribute\AsEventListener;
+    use TYPO3\CMS\IndexedSearch\Event\BeforeFinalSearchQueryIsExecutedEvent;
+
+    final readonly class EventListener
+    {
+        #[AsEventListener(identifier: 'manipulate-search-query')]
+        public function beforeFinalSearchQueryIsExecuted(BeforeFinalSearchQueryIsExecutedEvent $event): void
+        {
+            $event->queryBuilder->andWhere(
+                $event->queryBuilder->expr()->eq('some_column', 'some_value')
+            );
+        }
+    }
+
+..  index:: PHP-API, ext:indexed_search

typo3/sysext/indexed_search/Classes/Domain/Repository/IndexSearchRepository.php

@@ -20,15 +20,18 @@
 use Doctrine\DBAL\Platforms\MariaDBPlatform as DoctrineMariaDBPlatform;
 use Doctrine\DBAL\Platforms\MySQLPlatform as DoctrineMySQLPlatform;
 use Doctrine\DBAL\Result;
+use Psr\EventDispatcher\EventDispatcherInterface;
 use TYPO3\CMS\Core\Configuration\ExtensionConfiguration;
 use TYPO3\CMS\Core\Context\Context;
 use TYPO3\CMS\Core\Database\Connection;
 use TYPO3\CMS\Core\Database\ConnectionPool;
+use TYPO3\CMS\Core\Database\Query\QueryBuilder;
 use TYPO3\CMS\Core\Database\Query\QueryHelper;
 use TYPO3\CMS\Core\Domain\Repository\PageRepository;
 use TYPO3\CMS\Core\TimeTracker\TimeTracker;
 use TYPO3\CMS\Core\Utility\GeneralUtility;
 use TYPO3\CMS\Core\Utility\MathUtility;
+use TYPO3\CMS\IndexedSearch\Event\BeforeFinalSearchQueryIsExecutedEvent;
 use TYPO3\CMS\IndexedSearch\FileContentParser;
 use TYPO3\CMS\IndexedSearch\Type\MediaType;
 use TYPO3\CMS\IndexedSearch\Type\SearchType;
@@ -136,6 +139,7 @@ public function __construct(
         private readonly ExtensionConfiguration $extensionConfiguration,
         private readonly TimeTracker $timeTracker,
         private readonly ConnectionPool $connectionPool,
+        private readonly EventDispatcherInterface $eventDispatcher,
     ) {}
 
     /**
@@ -186,14 +190,22 @@ public function initialize(array $settings, array $searchData, array $externalPa
      */
     public function doSearch(array $searchWords, int $freeIndexUid): array|false
     {
+        $result = null;
         $useMysqlFulltext = (bool)$this->extensionConfiguration->get('indexed_search', 'useMysqlFulltext');
-        // Getting SQL result pointer:
         $this->timeTracker->push('Searching result');
-        // @todo Change method signatures to return the QueryBuilder instead the Result.
         if ($useMysqlFulltext) {
-            $result = $this->getResultRows_SQLpointerMysqlFulltext($searchWords, $freeIndexUid);
+            $queryBuilder = $this->getPreparedQueryBuilder_SQLpointerMysqlFulltext($searchWords, $freeIndexUid);
         } else {
-            $result = $this->getResultRows_SQLpointer($searchWords, $freeIndexUid);
+            $queryBuilder = $this->getPreparedQueryBuilder_SQLpointer($searchWords, $freeIndexUid);
+        }
+        if ($queryBuilder !== false) {
+            $this->eventDispatcher->dispatch(
+                new BeforeFinalSearchQueryIsExecutedEvent($queryBuilder, $searchWords, $freeIndexUid)
+            );
+            // Getting SQL result pointer:
+            $this->timeTracker->push('execFinalQuery');
+            $result = $queryBuilder->executeQuery();
+            $this->timeTracker->pull();
         }
         $this->timeTracker->pull();
         // Organize and process result:
@@ -340,36 +352,32 @@ public function getIndexConfigurationById(int $id): ?array
     }
 
     /**
-     * Gets a SQL result pointer to traverse for the search records.
+     * Gets the QueryBuilder instance prepared for the phash list.
      *
      * @param array $searchWords Search words
      * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
      */
-    protected function getResultRows_SQLpointer(array $searchWords, int $freeIndexUid): Result|false
+    protected function getPreparedQueryBuilder_SQLpointer(array $searchWords, int $freeIndexUid): QueryBuilder|false
     {
         // This SEARCHES for the searchwords in $searchWords AND returns a
         // COMPLETE list of phash-integers of the matches.
         $list = $this->getPhashList($searchWords);
-        // Perform SQL Search / collection of result rows array:
         if ($list) {
-            // Do the search:
-            $this->timeTracker->push('execFinalQuery');
-            $res = $this->execFinalQuery($list, $freeIndexUid);
-            $this->timeTracker->pull();
-            return $res;
+            // Create the search:
+            return $this->prepareFinalQuery($list, $freeIndexUid);
         }
         return false;
     }
 
     /**
-     * Gets a SQL result pointer to traverse for the search records.
+     * Gets the QueryBuilder instance prepared for the search words.
      *
      * mysql fulltext specific version triggered by ext_conf_template setting 'useMysqlFulltext'
      *
      * @param array $searchWordsArray Search words
      * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
      */
-    protected function getResultRows_SQLpointerMysqlFulltext(array $searchWordsArray, int $freeIndexUid): Result|false
+    protected function getPreparedQueryBuilder_SQLpointerMysqlFulltext(array $searchWordsArray, int $freeIndexUid): QueryBuilder|false
     {
         $connection = $this->connectionPool->getConnectionForTable('index_fulltext');
         $platform = $connection->getDatabasePlatform();
@@ -382,15 +390,11 @@ protected function getResultRows_SQLpointerMysqlFulltext(array $searchWordsArray
         }
         // Build the search string, detect which fulltext index to use, and decide whether boolean search is needed or not
         $searchData = $this->getSearchString($searchWordsArray);
-        // Perform SQL Search / collection of result rows array:
-        $resource = false;
         if ($searchData) {
-            // Do the search:
-            $this->timeTracker->push('execFinalQuery');
-            $resource = $this->execFinalQuery_fulltext($searchData, $freeIndexUid);
-            $this->timeTracker->pull();
+            // Create the search:
+            return $this->prepareFinalQuery_fulltext($searchData, $freeIndexUid);
         }
-        return $resource;
+        return false;
     }
 
     /**
@@ -467,14 +471,14 @@ protected function getSearchString(array $searchWordArray): array
     }
 
     /**
-     * Execute final query, based on phash integer list. The main point is sorting the result in the right order.
+     * Execute final query, based on search data. The main point is sorting the result in the right order.
      *
      * mysql fulltext specific helper method
      *
      * @param array $searchData Array with search string, boolean indicator, and fulltext index reference
      * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
      */
-    protected function execFinalQuery_fulltext(array $searchData, int $freeIndexUid): Result
+    protected function prepareFinalQuery_fulltext(array $searchData, int $freeIndexUid): QueryBuilder
     {
         $queryBuilder = $this->connectionPool->getQueryBuilderForTable('index_fulltext');
         $queryBuilder->getRestrictions()->removeAll();
@@ -557,7 +561,7 @@ protected function execFinalQuery_fulltext(array $searchData, int $freeIndexUid)
             'IP.freeIndexSetId'
         );
 
-        return $queryBuilder->executeQuery();
+        return $queryBuilder;
     }
 
     /***********************************
@@ -879,23 +883,22 @@ protected function freeIndexUidWhere(int $freeIndexUid): string
     }
 
     /**
-     * Execute final query, based on phash integer list. The main point is sorting the result in the right order.
+     * Prepare final query, based on phash integer list. The main point is sorting the result in the right order.
      *
      * @param string $list List of phash integers which match the search.
      * @param int $freeIndexUid Pointer to which indexing configuration you want to search in. -1 means no filtering. 0 means only regular indexed content.
      */
-    protected function execFinalQuery(string $list, int $freeIndexUid): Result
+    protected function prepareFinalQuery(string $list, int $freeIndexUid): QueryBuilder
     {
+        $phashList = GeneralUtility::trimExplode(',', $list, true);
         $queryBuilder = $this->connectionPool->getQueryBuilderForTable('index_words');
         $queryBuilder->select('ISEC.*', 'IP.*')
             ->from('index_phash', 'IP')
             ->from('index_section', 'ISEC')
             ->where(
                 $queryBuilder->expr()->in(
                     'IP.phash',
-                    $queryBuilder->quoteArrayBasedValueListToStringList(
-                        GeneralUtility::trimExplode(',', $list, true)
-                    )
+                    $queryBuilder->quoteArrayBasedValueListToStringList($phashList)
                 ),
                 QueryHelper::stripLogicalOperatorPrefix($this->mediaTypeWhere()),
                 QueryHelper::stripLogicalOperatorPrefix($this->languageWhere()),
@@ -1021,7 +1024,7 @@ protected function execFinalQuery(string $list, int $freeIndexUid): Result
             }
         }
 
-        return $queryBuilder->executeQuery();
+        return $queryBuilder;
     }
 
     /**

typo3/sysext/indexed_search/Classes/Event/BeforeFinalSearchQueryIsExecutedEvent.php

@@ -0,0 +1,32 @@
+<?php
+
+declare(strict_types=1);
+
+/*
+ * This file is part of the TYPO3 CMS project.
+ *
+ * It is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License, either version 2
+ * of the License, or any later version.
+ *
+ * For the full copyright and license information, please read the
+ * LICENSE.txt file that was distributed with this source code.
+ *
+ * The TYPO3 project - inspiring people to share!
+ */
+
+namespace TYPO3\CMS\IndexedSearch\Event;
+
+use TYPO3\CMS\Core\Database\Query\QueryBuilder;
+
+/**
+ * Listeners are able to manipulate the QueryBuilder before the search query gets executed
+ */
+final class BeforeFinalSearchQueryIsExecutedEvent
+{
+    public function __construct(
+        public QueryBuilder $queryBuilder,
+        public readonly array $searchWords,
+        public readonly int $freeIndexUid,
+    ) {}
+}

typo3/sysext/indexed_search/Tests/Functional/IndexSearchRepositoryTest.php

@@ -19,9 +19,12 @@
 
 use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\Attributes\Test;
+use Symfony\Component\DependencyInjection\Container;
 use TYPO3\CMS\Core\Context\Context;
 use TYPO3\CMS\Core\Context\UserAspect;
+use TYPO3\CMS\Core\EventDispatcher\ListenerProvider;
 use TYPO3\CMS\IndexedSearch\Domain\Repository\IndexSearchRepository;
+use TYPO3\CMS\IndexedSearch\Event\BeforeFinalSearchQueryIsExecutedEvent;
 use TYPO3\CMS\IndexedSearch\Indexer;
 use TYPO3\CMS\IndexedSearch\Type\MediaType;
 use TYPO3\CMS\IndexedSearch\Type\SearchType;
@@ -169,6 +172,35 @@ public function searchByMediaTypeSetsAppropriateQuerybuilderWhereCondition(?stri
         self::assertSame($expectedSql, preg_replace('@["\'`]@imsU', '', $mediaTypeWhere->invoke($searchRepository)));
     }
 
+    #[Test]
+    public function beforeFinalSearchQueryIsExecutedEventIsDispatched(): void
+    {
+        /** @var Container $container */
+        $container = $this->get('service_container');
+        $container->set(
+            'before-final-search-query-is-executed-listener',
+            static function (BeforeFinalSearchQueryIsExecutedEvent $event) use (&$beforeFinalSearchQueryIsExecutedListener) {
+                $beforeFinalSearchQueryIsExecutedListener = $event;
+                $beforeFinalSearchQueryIsExecutedListener->queryBuilder->andWhere(
+                    $beforeFinalSearchQueryIsExecutedListener->queryBuilder->expr()->in(
+                        'ISEC.page_id',
+                        [1, 2, 3]
+                    )
+                );
+            }
+        );
+
+        $eventListener = $container->get(ListenerProvider::class);
+        $eventListener->addListener(BeforeFinalSearchQueryIsExecutedEvent::class, 'before-final-search-query-is-executed-listener');
+
+        $searchRepository = $this->getSearchRepository();
+        $searchRepository->doSearch([['sword' => 'lorem']], -1);
+        self::assertInstanceOf(BeforeFinalSearchQueryIsExecutedEvent::class, $beforeFinalSearchQueryIsExecutedListener);
+        self::assertSame([['sword' => 'lorem']], $beforeFinalSearchQueryIsExecutedListener->searchWords);
+        self::assertSame(-1, $beforeFinalSearchQueryIsExecutedListener->freeIndexUid);
+        self::assertStringContainsString('ISEC.page_id IN (1, 2, 3)', preg_replace('/["\'`]/', '', $beforeFinalSearchQueryIsExecutedListener->queryBuilder->getSQL()));
+    }
+
     private function getSearchRepository(SearchType $searchType = SearchType::PART_OF_WORD): IndexSearchRepository
     {
         $searchRepository = $this->get(IndexSearchRepository::class);