Skip to content

Commit

Permalink
Merge pull request #18 from fredden/large-data-set
Browse files Browse the repository at this point in the history 
Replace use of fetchAll() for database queries
  • Loading branch information
@sprankhub
sprankhub authored Jul 28, 2022
2 parents d16dbeb + 428e642 commit 6147ee4
Show file tree
Hide file tree
Showing 3 changed files with 66 additions and 42 deletions.
8 changes: 4 additions & 4 deletions Console/Command/CleanUpAttributesAndValuesWithoutParentCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,12 @@ public function execute(InputInterface $input, OutputInterface $output): int
foreach ($types as $type) {
$eavTable = $this->resourceConnection->getTableName('eav_attribute');
$entityValueTable = $this->resourceConnection->getTableName($code . '_entity_' . $type);
$query = "SELECT * FROM $entityValueTable WHERE `attribute_id` NOT IN(SELECT attribute_id"
$query = "SELECT COUNT(*) FROM $entityValueTable WHERE `attribute_id` NOT IN(SELECT attribute_id"
. " FROM `$eavTable` WHERE entity_type_id = " . $entityType->getEntityTypeId() . " AND backend_type = '$type')";
$results = $db->fetchAll($query);
$output->writeln("Clean up " . count($results) . " rows in $entityValueTable");
$count = (int) $db->fetchOne($query);
$output->writeln("Clean up $count rows in $entityValueTable");

if (!$isDryRun && count($results) > 0) {
if (!$isDryRun && $count > 0) {
$db->query("DELETE FROM $entityValueTable WHERE `attribute_id` NOT IN(SELECT attribute_id"
. " FROM `$eavTable` WHERE entity_type_id = " . $entityType->getEntityTypeId() . " AND backend_type = '$type')");
}
Expand Down
28 changes: 19 additions & 9 deletions Console/Command/RestoreUseDefaultConfigValueCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

use Magento\Framework\App\Config\Initial\Reader;
use Magento\Framework\App\ResourceConnection;
use Magento\Framework\Model\ResourceModel\IteratorFactory;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
Expand All @@ -14,6 +15,9 @@ class RestoreUseDefaultConfigValueCommand extends Command
/** @var Reader */
private $configReader;

/** @var IteratorFactory */
private $iteratorFactory;

/**
* @var ResourceConnection
*/
Expand All @@ -24,12 +28,14 @@ class RestoreUseDefaultConfigValueCommand extends Command

public function __construct(
Reader $configReader,
IteratorFactory $iteratorFactory,
ResourceConnection $resourceConnection,
string $name = null
) {
parent::__construct($name);

$this->configReader = $configReader;
$this->iteratorFactory = $iteratorFactory;
$this->resourceConnection = $resourceConnection;
}

Expand Down Expand Up @@ -65,21 +71,25 @@ public function execute(InputInterface $input, OutputInterface $output): int

$removedConfigValues = 0;

$db = $this->resourceConnection->getConnection();
$dbRead = $this->resourceConnection->getConnection('core_read');
$dbWrite = $this->resourceConnection->getConnection('core_write');
$tableName = $this->resourceConnection->getTableName('core_config_data');
$configData = $db->fetchAll('SELECT DISTINCT path, value FROM ' . $tableName
. ' WHERE scope_id = 0');

foreach ($configData as $config) {
$count = $db->fetchOne('SELECT COUNT(*) FROM ' . $tableName
$query = $dbRead->query("SELECT DISTINCT path, value FROM $tableName WHERE scope_id = 0");

$iterator = $this->iteratorFactory->create();
$iterator->walk($query, [function (array $result) use ($dbRead, $dbWrite, $isDryRun, $output, &$removedConfigValues, $tableName): void {
$config = $result['row'];

$count = (int) $dbRead->fetchOne('SELECT COUNT(*) FROM ' . $tableName
. ' WHERE path = ? AND BINARY value = ?', [$config['path'], $config['value']]);

if ($count > 1) {
$output->writeln('Config path ' . $config['path'] . ' with value ' . $config['value'] . ' has ' . $count
. ' values; deleting non-default values');

if (!$isDryRun) {
$db->query(
$dbWrite->query(
'DELETE FROM ' . $tableName . ' WHERE path = ? AND BINARY value = ? AND scope_id != ?',
[$config['path'], $config['value'], 0]
);
Expand All @@ -93,12 +103,12 @@ public function execute(InputInterface $input, OutputInterface $output): int

if (!$isDryRun) {
if ($config['value'] === null) {
$db->query(
$dbWrite->query(
"DELETE FROM $tableName WHERE path = ? AND value IS NULL AND scope_id = 0",
[$config['path']]
);
} else {
$db->query(
$dbWrite->query(
"DELETE FROM $tableName WHERE path = ? AND BINARY value = ? AND scope_id = 0",
[$config['path'], $config['value']]
);
Expand All @@ -107,7 +117,7 @@ public function execute(InputInterface $input, OutputInterface $output): int

$removedConfigValues++;
}
}
}]);

$output->writeln('Removed ' . $removedConfigValues . ' values from core_config_data table.');

Expand Down
72 changes: 43 additions & 29 deletions Console/Command/RestoreUseDefaultValueCommand.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

use Magento\Framework\App\ProductMetadataInterface;
use Magento\Framework\App\ResourceConnection;
use Magento\Framework\Model\ResourceModel\IteratorFactory;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
Expand All @@ -12,6 +13,9 @@

class RestoreUseDefaultValueCommand extends Command
{
/** @var IteratorFactory */
protected $iteratorFactory;

/**
* @var ProductMetaDataInterface
*/
Expand All @@ -23,11 +27,14 @@ class RestoreUseDefaultValueCommand extends Command
private $resourceConnection;

public function __construct(
IteratorFactory $iteratorFactory,
ProductMetaDataInterface $productMetaData,
ResourceConnection $resourceConnection,
string $name = null
) {
parent::__construct($name);

$this->iteratorFactory = $iteratorFactory;
$this->productMetaData = $productMetaData;
$this->resourceConnection = $resourceConnection;
}
Expand Down Expand Up @@ -58,7 +65,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
if (!in_array($entity, ['product', 'category'])) {
$output->writeln('Please specify the entity with --entity. Possible options are product or category');

return 1;
return 1; // error.
}

if (!$isDryRun && !$isForce) {
Expand All @@ -76,57 +83,64 @@ public function execute(InputInterface $input, OutputInterface $output): int
}
}

$db = $this->resourceConnection->getConnection();
$dbRead = $this->resourceConnection->getConnection('core_read');
$dbWrite = $this->resourceConnection->getConnection('core_write');
$counts = [];
$tables = ['varchar', 'int', 'decimal', 'text', 'datetime'];
$column = $this->productMetaData->getEdition() === 'Enterprise' ? 'row_id' : 'entity_id';

foreach ($tables as $table) {
// Select all non-global values
$fullTableName = $this->resourceConnection->getTableName('catalog_' . $entity . '_entity_' . $table);
$rows = $db->fetchAll('SELECT * FROM ' . $fullTableName . ' WHERE store_id != 0');

foreach ($rows as $row) {
// NULL values are handled separately
$query = $dbRead->query("SELECT * FROM $fullTableName WHERE store_id != 0 AND value IS NOT NULL");

$iterator = $this->iteratorFactory->create();
$iterator->walk($query, [function (array $result) use ($column, &$counts, $dbRead, $dbWrite, $fullTableName, $isDryRun, $output): void {
$row = $result['row'];

// Select the global value if it's the same as the non-global value
$results = $db->fetchAll(
$query = $dbRead->query(
'SELECT * FROM ' . $fullTableName
. ' WHERE attribute_id = ? AND store_id = ? AND ' . $column . ' = ? AND BINARY value = ?',
[$row['attribute_id'], 0, $row[$column], $row['value']]
);

if (count($results) > 0) {
foreach ($results as $result) {
if (!$isDryRun) {
// Remove the non-global value
$db->query(
'DELETE FROM ' . $fullTableName . ' WHERE value_id = ?',
$row['value_id']
);
}

$output->writeln(
'Deleting value ' . $row['value_id'] . ' "' . $row['value'] . '" in favor of '
. $result['value_id']
. ' for attribute ' . $row['attribute_id'] . ' in table ' . $fullTableName
$iterator = $this->iteratorFactory->create();
$iterator->walk($query, [function (array $result) use (&$counts, $dbWrite, $fullTableName, $isDryRun, $output, $row): void {
$result = $result['row'];

if (!$isDryRun) {
// Remove the non-global value
$dbWrite->query(
'DELETE FROM ' . $fullTableName . ' WHERE value_id = ?',
$row['value_id']
);
}

if (!isset($counts[$row['attribute_id']])) {
$counts[$row['attribute_id']] = 0;
}
$output->writeln(
'Deleting value ' . $row['value_id'] . ' "' . $row['value'] . '" in favor of '
. $result['value_id']
. ' for attribute ' . $row['attribute_id'] . ' in table ' . $fullTableName
);

$counts[$row['attribute_id']]++;
if (!isset($counts[$row['attribute_id']])) {
$counts[$row['attribute_id']] = 0;
}
}
}

$nullValues = $db->fetchOne(
$counts[$row['attribute_id']]++;
}]);
}]);

$nullCount = (int) $dbRead->fetchOne(
'SELECT COUNT(*) FROM ' . $fullTableName . ' WHERE store_id != 0 AND value IS NULL'
);

if (!$isDryRun && $nullValues > 0) {
$output->writeln("Deleting " . $nullValues . " NULL value(s) from " . $fullTableName);
if (!$isDryRun && $nullCount > 0) {
$output->writeln("Deleting $nullCount NULL value(s) from $fullTableName");
// Remove all non-global null values
$db->query(
$dbWrite->query(
'DELETE FROM ' . $fullTableName . ' WHERE store_id != 0 AND value IS NULL'
);
}
Expand Down

0 comments on commit 6147ee4

Please sign in to comment.