Skip to content

Commit

Permalink
Merge pull request magento#2514 from magento-performance/MAGETWO-90563
Browse files Browse the repository at this point in the history 
[performance] MAGETWO-90563: [Forwardport] Implement segmentation for Category Product Indexer
  • Loading branch information
@kandy
kandy authored May 8, 2018
2 parents b2eb795 + 52971ba commit a53347e
Show file tree
Hide file tree
Showing 27 changed files with 1,054 additions and 192 deletions.
25 changes: 23 additions & 2 deletions app/code/Magento/AdvancedSearch/Model/ResourceModel/Index.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@
use Magento\Framework\Model\ResourceModel\Db\Context;
use Magento\Framework\EntityManager\MetadataPool;
use Magento\Catalog\Api\Data\CategoryInterface;
use Magento\Framework\App\ObjectManager;
use Magento\Framework\Indexer\ScopeResolver\IndexScopeResolver as TableResolver;
use Magento\Framework\Search\Request\Dimension;
use Magento\Catalog\Model\Indexer\Category\Product\AbstractAction;

/**
* @api
Expand All @@ -29,22 +33,30 @@ class Index extends AbstractDb
*/
protected $metadataPool;

/**
* @var TableResolver
*/
private $tableResolver;

/**
* Index constructor.
* @param Context $context
* @param StoreManagerInterface $storeManager
* @param MetadataPool $metadataPool
* @param null $connectionName
* @param TableResolver|null $tableResolver
*/
public function __construct(
Context $context,
StoreManagerInterface $storeManager,
MetadataPool $metadataPool,
$connectionName = null
$connectionName = null,
TableResolver $tableResolver = null
) {
parent::__construct($context, $connectionName);
$this->storeManager = $storeManager;
$this->metadataPool = $metadataPool;
$this->tableResolver = $tableResolver ?: ObjectManager::getInstance()->get(TableResolver::class);
}

/**
Expand Down Expand Up @@ -116,8 +128,17 @@ public function getCategoryProductIndexData($storeId = null, $productIds = null)
{
$connection = $this->getConnection();

$catalogCategoryProductDimension = new Dimension(\Magento\Store\Model\Store::ENTITY, $storeId);

$catalogCategoryProductTableName = $this->tableResolver->resolve(
AbstractAction::MAIN_INDEX_TABLE,
[
$catalogCategoryProductDimension
]
);

$select = $connection->select()->from(
[$this->getTable('catalog_category_product_index')],
[$catalogCategoryProductTableName],
['category_id', 'product_id', 'position', 'store_id']
)->where(
'store_id = ?',
Expand Down
44 changes: 37 additions & 7 deletions app/code/Magento/Catalog/Model/Indexer/Category/Product/AbstractAction.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,8 @@
use Magento\Catalog\Api\Data\ProductInterface;
use Magento\Catalog\Model\Product;
use Magento\Framework\App\ObjectManager;
use Magento\Framework\DB\Query\Generator as QueryGenerator;
use Magento\Framework\App\ResourceConnection;
use Magento\Framework\DB\Query\Generator as QueryGenerator;
use Magento\Framework\DB\Select;
use Magento\Framework\EntityManager\MetadataPool;
use Magento\Store\Model\Store;
Expand Down Expand Up @@ -42,6 +42,7 @@ abstract class AbstractAction

/**
* Suffix for table to show it is temporary
* @deprecated
*/
const TEMPORARY_TABLE_SUFFIX = '_tmp';

Expand Down Expand Up @@ -106,6 +107,11 @@ abstract class AbstractAction
*/
protected $metadataPool;

/**
* @var TableMaintainer
*/
protected $tableMaintainer;

/**
* @var string
* @since 101.0.0
Expand All @@ -121,22 +127,25 @@ abstract class AbstractAction
* @param ResourceConnection $resource
* @param \Magento\Store\Model\StoreManagerInterface $storeManager
* @param \Magento\Catalog\Model\Config $config
* @param QueryGenerator|null $queryGenerator
* @param QueryGenerator $queryGenerator
* @param MetadataPool|null $metadataPool
* @param TableMaintainer|null $tableMaintainer
*/
public function __construct(
\Magento\Framework\App\ResourceConnection $resource,
\Magento\Store\Model\StoreManagerInterface $storeManager,
\Magento\Catalog\Model\Config $config,
QueryGenerator $queryGenerator = null,
MetadataPool $metadataPool = null
MetadataPool $metadataPool = null,
TableMaintainer $tableMaintainer = null
) {
$this->resource = $resource;
$this->connection = $resource->getConnection();
$this->storeManager = $storeManager;
$this->config = $config;
$this->queryGenerator = $queryGenerator ?: ObjectManager::getInstance()->get(QueryGenerator::class);
$this->metadataPool = $metadataPool ?: ObjectManager::getInstance()->get(MetadataPool::class);
$this->tableMaintainer = $tableMaintainer ?: ObjectManager::getInstance()->get(TableMaintainer::class);
}

/**
Expand Down Expand Up @@ -180,6 +189,7 @@ protected function getTable($table)
* The name is switched between 'catalog_category_product_index' and 'catalog_category_product_index_replica'
*
* @return string
* @deprecated
*/
protected function getMainTable()
{
Expand All @@ -190,6 +200,7 @@ protected function getMainTable()
* Return temporary index table name
*
* @return string
* @deprecated
*/
protected function getMainTmpTable()
{
Expand All @@ -198,6 +209,19 @@ protected function getMainTmpTable()
: $this->getMainTable();
}

/**
* Return index table name
*
* @param int $storeId
* @return string
*/
protected function getIndexTable($storeId)
{
return $this->useTempTable
? $this->tableMaintainer->getMainReplicaTable($storeId)
: $this->tableMaintainer->getMainTable($storeId);
}

/**
* Return category path by id
*
Expand Down Expand Up @@ -319,7 +343,7 @@ protected function getNonAnchorCategoriesSelect(Store $store)
}

/**
* Add filtering by child products to select.
* Add filtering by child products to select
*
* It's used for correct handling of composite products.
* This method makes assumption that select already joins `catalog_product_entity` as `cpe`.
Expand Down Expand Up @@ -422,7 +446,7 @@ protected function reindexNonAnchorCategories(Store $store)
$this->connection->query(
$this->connection->insertFromSelect(
$select,
$this->getMainTmpTable(),
$this->getIndexTable($store->getId()),
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
)
Expand Down Expand Up @@ -612,6 +636,12 @@ protected function makeTempCategoryTreeIndex()
['type' => \Magento\Framework\DB\Adapter\AdapterInterface::INDEX_TYPE_PRIMARY]
);

$temporaryTable->addIndex(
'child_id',
['child_id'],
['type' => \Magento\Framework\DB\Adapter\AdapterInterface::INDEX_TYPE_INDEX]
);

// Drop the temporary table in case it already exists on this (persistent?) connection.
$this->connection->dropTemporaryTable($temporaryName);
$this->connection->createTemporaryTable($temporaryTable);
Expand Down Expand Up @@ -678,7 +708,7 @@ protected function reindexAnchorCategories(Store $store)
$this->connection->query(
$this->connection->insertFromSelect(
$select,
$this->getMainTmpTable(),
$this->getIndexTable($store->getId()),
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
)
Expand Down Expand Up @@ -804,7 +834,7 @@ protected function reindexRootCategory(Store $store)
$this->connection->query(
$this->connection->insertFromSelect(
$select,
$this->getMainTmpTable(),
$this->getIndexTable($store->getId()),
['category_id', 'product_id', 'position', 'is_parent', 'store_id', 'visibility'],
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
)
Expand Down
84 changes: 50 additions & 34 deletions app/code/Magento/Catalog/Model/Indexer/Category/Product/Action/Full.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,18 +88,35 @@ public function __construct(
}

/**
*
* Clear the table we'll be writing de-normalized data into
* to prevent archived data getting in the way of actual data.
*
* @return void
*/
private function clearCurrentTable()
private function createTables()
{
$this->connection->delete(
$this->activeTableSwitcher
->getAdditionalTableName($this->getMainTable())
);
foreach ($this->storeManager->getStores() as $store) {
$this->tableMaintainer->createTablesForStore($store->getId());
}
}

/**
* @return void
*/
private function clearReplicaTables()
{
foreach ($this->storeManager->getStores() as $store) {
$this->connection->truncateTable($this->tableMaintainer->getMainReplicaTable($store->getId()));
}
}

/**
* @return void
*/
private function switchTables()
{
$tablesToSwitch = [];
foreach ($this->storeManager->getStores() as $store) {
$tablesToSwitch[] = $this->tableMaintainer->getMainTable($store->getId());
}
$this->activeTableSwitcher->switchTable($this->connection, $tablesToSwitch);
}

/**
Expand All @@ -109,22 +126,26 @@ private function clearCurrentTable()
*/
public function execute()
{
$this->clearCurrentTable();
$this->createTables();
$this->clearReplicaTables();
$this->reindex();
$this->activeTableSwitcher->switchTable($this->connection, [$this->getMainTable()]);
$this->switchTables();
return $this;
}

/**
* Publish data from tmp to index
* Publish data from tmp to replica table
*
* @param \Magento\Store\Model\Store $store
* @return void
*/
protected function publishData()
private function publishData($store)
{
$select = $this->connection->select()->from($this->getMainTmpTable());
$columns = array_keys($this->connection->describeTable($this->getMainTable()));
$tableName = $this->activeTableSwitcher->getAdditionalTableName($this->getMainTable());
$select = $this->connection->select()->from($this->tableMaintainer->getMainTmpTable($store->getId()));
$columns = array_keys(
$this->connection->describeTable($this->tableMaintainer->getMainReplicaTable($store->getId()))
);
$tableName = $this->tableMaintainer->getMainReplicaTable($store->getId());

$this->connection->query(
$this->connection->insertFromSelect(
Expand All @@ -136,23 +157,13 @@ protected function publishData()
);
}

/**
* Clear all index data
*
* @return void
*/
protected function clearTmpData()
{
$this->connection->delete($this->getMainTmpTable());
}

/**
* {@inheritdoc}
*/
protected function reindexRootCategory(\Magento\Store\Model\Store $store)
{
if ($this->isIndexRootCategoryNeeded()) {
$this->reindexCategoriesBySelect($this->getAllProducts($store), 'cp.entity_id IN (?)');
$this->reindexCategoriesBySelect($this->getAllProducts($store), 'cp.entity_id IN (?)', $store);
}
}

Expand All @@ -164,7 +175,7 @@ protected function reindexRootCategory(\Magento\Store\Model\Store $store)
*/
protected function reindexAnchorCategories(\Magento\Store\Model\Store $store)
{
$this->reindexCategoriesBySelect($this->getAnchorCategoriesSelect($store), 'ccp.product_id IN (?)');
$this->reindexCategoriesBySelect($this->getAnchorCategoriesSelect($store), 'ccp.product_id IN (?)', $store);
}

/**
Expand All @@ -175,20 +186,25 @@ protected function reindexAnchorCategories(\Magento\Store\Model\Store $store)
*/
protected function reindexNonAnchorCategories(\Magento\Store\Model\Store $store)
{
$this->reindexCategoriesBySelect($this->getNonAnchorCategoriesSelect($store), 'ccp.product_id IN (?)');
$this->reindexCategoriesBySelect($this->getNonAnchorCategoriesSelect($store), 'ccp.product_id IN (?)', $store);
}

/**
* Reindex categories using given SQL select and condition.
*
* @param \Magento\Framework\DB\Select $basicSelect
* @param string $whereCondition
* @param \Magento\Store\Model\Store $store
* @return void
*/
private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSelect, $whereCondition)
private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSelect, $whereCondition, $store)
{
$this->tableMaintainer->createMainTmpTable($store->getId());

$entityMetadata = $this->metadataPool->getMetadata(\Magento\Catalog\Api\Data\ProductInterface::class);
$columns = array_keys($this->connection->describeTable($this->getMainTmpTable()));
$columns = array_keys(
$this->connection->describeTable($this->tableMaintainer->getMainTmpTable($store->getId()))
);
$this->batchSizeManagement->ensureBatchSize($this->connection, $this->batchRowsCount);
$batches = $this->batchProvider->getBatches(
$this->connection,
Expand All @@ -197,7 +213,7 @@ private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSe
$this->batchRowsCount
);
foreach ($batches as $batch) {
$this->clearTmpData();
$this->connection->delete($this->tableMaintainer->getMainTmpTable($store->getId()));
$resultSelect = clone $basicSelect;
$select = $this->connection->select();
$select->distinct(true);
Expand All @@ -207,12 +223,12 @@ private function reindexCategoriesBySelect(\Magento\Framework\DB\Select $basicSe
$this->connection->query(
$this->connection->insertFromSelect(
$resultSelect,
$this->getMainTmpTable(),
$this->tableMaintainer->getMainTmpTable($store->getId()),
$columns,
\Magento\Framework\DB\Adapter\AdapterInterface::INSERT_ON_DUPLICATE
)
);
$this->publishData();
$this->publishData($store);
}
}
}
Loading

0 comments on commit a53347e

Please sign in to comment.