Skip to content

Commit

Permalink
Merge pull request #1219 from magento-tango/MAGETWO-69894
Browse files Browse the repository at this point in the history 
[Tango] Configuration values return null when redis reaches max_memory
  • Loading branch information
Sergii Kovalenko authored Jun 23, 2017
2 parents 10c6ffa + e773529 commit d816ce7
Show file tree
Hide file tree
Showing 7 changed files with 412 additions and 146 deletions.
235 changes: 133 additions & 102 deletions app/code/Magento/Config/App/Config/Type/System.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,11 @@
namespace Magento\Config\App\Config\Type;

use Magento\Framework\App\Config\ConfigTypeInterface;
use Magento\Framework\DataObject;
use Magento\Framework\App\ObjectManager;
use Magento\Config\App\Config\Type\System\Reader;

/**
* Class process source, cache them and retrieve value by path
* System configuration type
*/
class System implements ConfigTypeInterface
{
Expand All @@ -23,9 +24,9 @@ class System implements ConfigTypeInterface
private $source;

/**
* @var DataObject
* @var array
*/
private $data;
private $data = [];

/**
* @var \Magento\Framework\App\Config\Spi\PostProcessorInterface
Expand Down Expand Up @@ -65,14 +66,18 @@ class System implements ConfigTypeInterface
private $configType;

/**
* Key name for flag which displays whether configuration is cached or not.
* @var Reader
*/
private $reader;

/**
* List of scopes that were retrieved from configuration storage
*
* Once configuration is cached additional flag pushed to cache storage
* to be able check cache existence without data load.
* Is used to make sure that we don't try to load non-existing configuration scopes.
*
* @var string
* @var array
*/
private $cacheExistenceKey;
private $availableDataScopes = null;

/**
* @param \Magento\Framework\App\Config\ConfigSourceInterface $source
Expand All @@ -83,6 +88,7 @@ class System implements ConfigTypeInterface
* @param \Magento\Framework\App\Config\Spi\PreProcessorInterface $preProcessor
* @param int $cachingNestedLevel
* @param string $configType
* @param Reader $reader
*/
public function __construct(
\Magento\Framework\App\Config\ConfigSourceInterface $source,
Expand All @@ -92,7 +98,8 @@ public function __construct(
\Magento\Framework\Serialize\SerializerInterface $serializer,
\Magento\Framework\App\Config\Spi\PreProcessorInterface $preProcessor,
$cachingNestedLevel = 1,
$configType = self::CONFIG_TYPE
$configType = self::CONFIG_TYPE,
Reader $reader = null
) {
$this->source = $source;
$this->postProcessor = $postProcessor;
Expand All @@ -102,150 +109,174 @@ public function __construct(
$this->fallback = $fallback;
$this->serializer = $serializer;
$this->configType = $configType;
$this->cacheExistenceKey = $this->configType . '_CACHE_EXISTS';
$this->reader = $reader ?: ObjectManager::getInstance()->get(Reader::class);
}

/**
* System configuration is separated by scopes (default, websites, stores). Configuration of a scope is inherited
* from its parent scope (store inherits website).
*
* Because there can be many scopes on single instance of application, the configuration data can be pretty large,
* so it does not make sense to load all of it on every application request. That is why we cache configuration
* data by scope and only load configuration scope when a value from that scope is requested.
*
* Possible path values:
* '' - will return whole system configuration (default scope + all other scopes)
* 'default' - will return all default scope configuration values
* '{scopeType}' - will return data from all scopes of a specified {scopeType} (websites, stores)
* '{scopeType}/{scopeCode}' - will return data for all values of the scope specified by {scopeCode} and scope type
* '{scopeType}/{scopeCode}/some/config/variable' - will return value of the config variable in the specified scope
*
* @inheritdoc
*/
public function get($path = '')
{
if ($path === null) {
$path = '';
if ($path === '') {
$this->data = array_replace_recursive($this->loadAllData(), $this->data);
return $this->data;
}
if ($this->isConfigRead($path)) {
return $this->data->getData($path);
$pathParts = explode('/', $path);
if (count($pathParts) === 1 && $pathParts[0] !== 'default') {
if (!isset($this->data[$pathParts[0]])) {
$data = $this->reader->read();
$this->data = array_replace_recursive($data, $this->data);
}
return $this->data[$pathParts[0]];
}

if (!empty($path) && $this->isCacheExists()) {
return $this->readFromCache($path);
$scopeType = array_shift($pathParts);
if ($scopeType === 'default') {
if (!isset($this->data[$scopeType])) {
$this->data = array_replace_recursive($this->loadDefaultScopeData($scopeType), $this->data);
}
return $this->getDataByPathParts($this->data[$scopeType], $pathParts);
}

$config = $this->loadConfig();
$this->cacheConfig($config);
$this->data = new DataObject($config);
return $this->data->getData($path);
}

/**
* Check whether configuration is cached
*
* In case configuration cache exists method 'load' returns
* value equal to $this->cacheExistenceKey
*
* @return bool
*/
private function isCacheExists()
{
return $this->cache->load($this->cacheExistenceKey) !== false;
$scopeId = array_shift($pathParts);
if (!isset($this->data[$scopeType][$scopeId])) {
$this->data = array_replace_recursive($this->loadScopeData($scopeType, $scopeId), $this->data);
}
return isset($this->data[$scopeType][$scopeId])
? $this->getDataByPathParts($this->data[$scopeType][$scopeId], $pathParts)
: null;
}

/**
* Explode path by '/'(forward slash) separator
*
* In case $path string contains forward slash symbol(/) the $path is exploded and parts array is returned
* In other case empty array is returned
* Load configuration data for all scopes
*
* @param string $path
* @return array
*/
private function getPathParts($path)
private function loadAllData()
{
$pathParts = [];
if (strpos($path, '/') !== false) {
$pathParts = explode('/', $path);
$cachedData = $this->cache->load($this->configType);
if ($cachedData === false) {
$data = $this->reader->read();
} else {
$data = $this->serializer->unserialize($cachedData);
}
return $pathParts;
return $data;
}

/**
* Check whether requested configuration data is read to memory
*
* Because of configuration is cached partially each part can be loaded separately
* Method performs check if corresponding system configuration part is already loaded to memory
* and value can be retrieved directly without cache look up
* Load configuration data for default scope
*
*
* @param string $path
* @return bool
* @param string $scopeType
* @return array
*/
private function isConfigRead($path)
private function loadDefaultScopeData($scopeType)
{
$pathParts = $this->getPathParts($path);
return !empty($pathParts) && isset($this->data[$pathParts[0]][$pathParts[1]]);
$cachedData = $this->cache->load($this->configType . '_' . $scopeType);
if ($cachedData === false) {
$data = $this->reader->read();
$this->cacheData($data);
} else {
$data = [$scopeType => $this->serializer->unserialize($cachedData)];
}
return $data;
}

/**
* Load configuration from all the sources
*
* System configuration is loaded in 3 steps performing consecutive calls to
* Pre Processor, Fallback Processor, Post Processor accordingly
* Load configuration data for a specified scope
*
* @param string $scopeType
* @param string $scopeId
* @return array
*/
private function loadConfig()
private function loadScopeData($scopeType, $scopeId)
{
$data = $this->preProcessor->process($this->source->get());
$this->data = new DataObject($data);
$data = $this->fallback->process($data);
$this->data = new DataObject($data);

return $this->postProcessor->process($data);
$cachedData = $this->cache->load($this->configType . '_' . $scopeType . '_' . $scopeId);
if ($cachedData === false) {
if ($this->availableDataScopes === null) {
$cachedScopeData = $this->cache->load($this->configType . '_scopes');
if ($cachedScopeData !== false) {
$this->availableDataScopes = $this->serializer->unserialize($cachedScopeData);
}
}
if (is_array($this->availableDataScopes) && !isset($this->availableDataScopes[$scopeType][$scopeId])) {
return [$scopeType => [$scopeId => []]];
}
$data = $this->reader->read();
$this->cacheData($data);
} else {
$data = [$scopeType => [$scopeId => $this->serializer->unserialize($cachedData)]];
}
return $data;
}

/**
*
* Load configuration and caching it by parts.
*
* To be cached configuration is loaded first.
* Then it is cached by parts to minimize memory usage on load.
* Additional flag cached as well to give possibility check cache existence without data load.
* Cache configuration data.
* Caches data per scope to avoid reading data for all scopes on every request
*
* @param array $data
* @return void
*/
private function cacheConfig($data)
private function cacheData(array $data)
{
foreach ($data as $scope => $scopeData) {
foreach ($scopeData as $key => $config) {
$this->cache->save(
$this->serializer->serialize($data),
$this->configType,
[self::CACHE_TAG]
);
$this->cache->save(
$this->serializer->serialize($data['default']),
$this->configType . '_default',
[self::CACHE_TAG]
);
$scopes = [];
foreach (['websites', 'stores'] as $curScopeType) {
foreach ($data[$curScopeType] as $curScopeId => $curScopeData) {
$scopes[$curScopeType][$curScopeId] = 1;
$this->cache->save(
$this->serializer->serialize($config),
$this->configType . '_' . $scope . $key,
$this->serializer->serialize($curScopeData),
$this->configType . '_' . $curScopeType . '_' . $curScopeId,
[self::CACHE_TAG]
);
}
}
$this->cache->save($this->cacheExistenceKey, $this->cacheExistenceKey, [self::CACHE_TAG]);
$this->cache->save(
$this->serializer->serialize($scopes),
$this->configType . "_scopes",
[self::CACHE_TAG]
);
}

/**
* Read cached configuration
* Walk nested hash map by keys from $pathParts
*
* Read section of system configuration corresponding to requested $path from cache
* Configuration stored to internal property right after load to prevent additional
* requests to cache storage
*
* @param string $path
* @param array $data to walk in
* @param array $pathParts keys path
* @return mixed
*/
private function readFromCache($path)
private function getDataByPathParts($data, $pathParts)
{
if ($this->data === null) {
$this->data = new DataObject();
}

$result = null;
$pathParts = $this->getPathParts($path);
if (!empty($pathParts)) {
$result = $this->cache->load($this->configType . '_' . $pathParts[0] . $pathParts[1]);
if ($result !== false) {
$readData = $this->data->getData();
$readData[$pathParts[0]][$pathParts[1]] = $this->serializer->unserialize($result);
$this->data->setData($readData);
foreach ($pathParts as $key) {
if ((array)$data === $data && isset($data[$key])) {
$data = $data[$key];
} elseif ($data instanceof \Magento\Framework\DataObject) {
$data = $data->getDataByKey($key);
} else {
return null;
}
}

return $this->data->getData($path);
return $data;
}

/**
Expand All @@ -259,7 +290,7 @@ private function readFromCache($path)
*/
public function clean()
{
$this->data = null;
$this->data = [];
$this->cache->clean(\Zend_Cache::CLEANING_MODE_MATCHING_TAG, [self::CACHE_TAG]);
}
}
Loading

0 comments on commit d816ce7

Please sign in to comment.