Skip to content

Commit

Permalink
Batch daemon clear config (#596)
Browse files Browse the repository at this point in the history
* Clear batch config if the serialized data is corrupted.

If you attempt to store a closure into shared memory, it will error out
and leave corrupt data in shared memory. When the daemon tries to read
the corrupt data, it crashes, but currently does not clear the config.
Thus, the batch daemon will continue to crash until the shared memory is
manually cleared.

Instead, if we detect corrupt data, we will clear the corrupt data from
shared memory.

* Clear shared memory when the write fails on serialization as well

* If BatchConfig serialization fails, clear it out.

Don't force the readers to do it, although the also will clear it out
if deserialization fails.

* Return the result of the storage write
  • Loading branch information
chingor13 authored and dwsupplee committed Jul 19, 2017
1 parent 404cddc commit c2c6b16
Show file tree
Hide file tree
Showing 4 changed files with 54 additions and 13 deletions.
22 changes: 13 additions & 9 deletions Batch/BatchRunner.php
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,12 @@ public function registerJob($identifier, $func, array $options = [])
$this->config = $this->configStorage->load();
$this->config->registerJob($identifier, $func, $options);

$result = $this->configStorage->save($this->config);
if ($result === false) {
return false;
try {
$result = $this->configStorage->save($this->config);
} finally {
$this->configStorage->unlock();
}
$this->configStorage->unlock();
return true;
return $result;
}

/**
Expand Down Expand Up @@ -182,11 +182,15 @@ public function loadConfig()
if ($result === false) {
throw new \RuntimeException('Failed to lock the configStorage');
}
$result = $this->configStorage->load();
$this->configStorage->unlock();
if ($result === false) {
throw new \RuntimeException('Failed to load the BatchConfig');
try {
$result = $this->configStorage->load();
} catch (\RuntimeException $e) {
$this->configStorage->clear();
throw $e;
} finally {
$this->configStorage->unlock();
}

$this->config = $result;
return true;
}
Expand Down
5 changes: 5 additions & 0 deletions Batch/ConfigStorageInterface.php
Original file line number Diff line number Diff line change
Expand Up @@ -55,4 +55,9 @@ public function save(BatchConfig $config);
* @throws \RuntimeException when failed to load the BatchConfig.
*/
public function load();

/**
* Clear the BatchConfig from storage.
*/
public function clear();
}
8 changes: 8 additions & 0 deletions Batch/InMemoryConfigStorage.php
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,14 @@ public function load()
return $this->config;
}

/**
* Clear the BatchConfig from storage.
*/
public function clear()
{
$this->config = new BatchConfig();
}

/**
* Hold the items in memory and run the job in the same process when it
* meets the condition.
Expand Down
32 changes: 28 additions & 4 deletions Batch/SysvConfigStorage.php
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ public function unlock()
*
* @param BatchConfig $config A BatchConfig to save.
* @return bool
* @throws \RuntimeException when failed to attach to the shared memory.
* @throws \RuntimeException when failed to attach to the shared memory or serialization fails
*/
public function save(BatchConfig $config)
{
Expand All @@ -79,16 +79,24 @@ public function save(BatchConfig $config)
'Failed to attach to the shared memory'
);
}
$result = shm_put_var($shmid, self::VAR_KEY, $config);
shm_detach($shmid);

// If the variable write fails, clear the memory and re-raise the exception
try {
$result = shm_put_var($shmid, self::VAR_KEY, $config);
} catch (\Exception $e) {
$this->clear();
throw new \RuntimeException($e->getMessage());
} finally {
shm_detach($shmid);
}
return $result;
}

/**
* Load a BatchConfig from the storage.
*
* @return BatchConfig
* @throws \RuntimeException when failed to attach to the shared memory.
* @throws \RuntimeException when failed to attach to the shared memory or deserialization fails
*/
public function load()
{
Expand All @@ -103,6 +111,22 @@ public function load()
} else {
$result = shm_get_var($shmid, self::VAR_KEY);
}
shm_detach($shmid);

if ($result === false) {
throw new \RuntimeException(
'Failed to deserialize data from shared memory'
);
}
return $result;
}

/**
* Clear the BatchConfig from storage.
*/
public function clear()
{
$shmid = shm_attach($this->sysvKey);
shm_remove_var($shmid, self::VAR_KEY);
}
}

0 comments on commit c2c6b16

Please sign in to comment.