Skip to content

Commit

Permalink
Clarify active runtime ID usage in the executor
Browse files Browse the repository at this point in the history 
The runtimeId will be the ID passed into the executor. From there,
the runtimeName will be the hostname + runtimeId. We want to prefix
with hostname to support multiple executors on the same machine.

The runtimeName will be used for the container name and the active
runtimes table.
  • Loading branch information
@stnguyen90
stnguyen90 committed Mar 12, 2024
1 parent 27ed18a commit 203d0e5
Show file tree
Hide file tree
Showing 2 changed files with 48 additions and 51 deletions.
95 changes: 46 additions & 49 deletions app/http.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -303,21 +303,21 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
->action(function (string $runtimeId, string $timeoutStr, Response $response, Log $log) {
$timeout = \intval($timeoutStr);

$runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name)
$runtimeName = System::getHostname() . '-' . $runtimeId;

$response->sendHeader('Content-Type', 'text/event-stream');
$response->sendHeader('Cache-Control', 'no-cache');

// Wait for runtime
for ($i = 0; $i < 10; $i++) {
$output = '';
$code = Console::execute('docker container inspect ' . \escapeshellarg($runtimeId), '', $output);
$code = Console::execute('docker container inspect ' . \escapeshellarg($runtimeName), '', $output);
if ($code === 0) {
break;
}

if ($i === 9) {
$runtimeIdTokens = explode("-", $runtimeId);
$runtimeIdTokens = explode("-", $runtimeName);
$executorId = $runtimeIdTokens[0];
$functionId = $runtimeIdTokens[1];
$deploymentId = $runtimeIdTokens[2];
Expand Down Expand Up @@ -357,7 +357,7 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
});

$output = '';
Console::execute('docker exec ' . \escapeshellarg($runtimeId) . ' tail -F /var/tmp/logs.txt', '', $output, $timeout, function (string $outputChunk, mixed $process) use (&$logsChunk, &$logsProcess) {
Console::execute('docker exec ' . \escapeshellarg($runtimeName) . ' tail -F /var/tmp/logs.txt', '', $output, $timeout, function (string $outputChunk, mixed $process) use (&$logsChunk, &$logsProcess) {
$logsProcess = $process;

if (!empty($outputChunk)) {
Expand Down Expand Up @@ -390,15 +390,14 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
->inject('response')
->inject('log')
->action(function (string $runtimeId, string $image, string $entrypoint, string $source, string $destination, array $variables, string $runtimeEntrypoint, string $command, int $timeout, bool $remove, int $cpus, int $memory, string $version, Orchestration $orchestration, Table $activeRuntimes, Response $response, Log $log) {
$activeRuntimeId = $runtimeId; // Used with Swoole table (key)
$runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name)
$runtimeName = System::getHostname() . '-' . $runtimeId;

$runtimeHostname = \uniqid();

$log->addTag('runtimeId', $activeRuntimeId);
$log->addTag('runtimeId', $runtimeId);

if ($activeRuntimes->exists($activeRuntimeId)) {
if ($activeRuntimes->get($activeRuntimeId)['status'] == 'pending') {
if ($activeRuntimes->exists($runtimeName)) {
if ($activeRuntimes->get($runtimeName)['status'] == 'pending') {
throw new \Exception('A runtime with the same ID is already being created. Attempt a execution soon.', 500);
}

Expand All @@ -411,8 +410,8 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):

$secret = \bin2hex(\random_bytes(16));

$activeRuntimes->set($activeRuntimeId, [
'name' => $activeRuntimeId,
$activeRuntimes->set($runtimeName, [
'name' => $runtimeName,
'hostname' => $runtimeHostname,
'created' => $startTime,
'updated' => $startTime,
Expand All @@ -423,7 +422,7 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
/**
* Temporary file paths in the executor
*/
$tmpFolder = "tmp/$runtimeId/";
$tmpFolder = "tmp/$runtimeName/";
$tmpSource = "/{$tmpFolder}src/code.tar.gz";
$tmpBuild = "/{$tmpFolder}builds/code.tar.gz";

Expand Down Expand Up @@ -486,13 +485,13 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
/** Keep the container alive if we have commands to be executed */
$containerId = $orchestration->run(
image: $image,
name: $runtimeId,
name: $runtimeName,
hostname: $runtimeHostname,
vars: $variables,
command: $runtimeEntrypointCommands,
labels: [
'openruntimes-executor' => System::getHostname(),
'openruntimes-runtime-id' => $activeRuntimeId
'openruntimes-runtime-id' => $runtimeId
],
volumes: [
\dirname($tmpSource) . ':/tmp:rw',
Expand All @@ -516,7 +515,7 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
];

$status = $orchestration->execute(
name: $runtimeId,
name: $runtimeName,
command: $commands,
output: $output,
timeout: $timeout
Expand Down Expand Up @@ -563,8 +562,8 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
'duration' => $duration,
]);

$activeRuntimes->set($activeRuntimeId, [
'name' => $activeRuntimeId,
$activeRuntimes->set($runtimeName, [
'name' => $runtimeName,
'hostname' => $runtimeHostname,
'created' => $startTime,
'updated' => \microtime(true),
Expand All @@ -578,7 +577,7 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
try {
$logs = '';
$status = $orchestration->execute(
name: $runtimeId,
name: $runtimeName,
command: [ 'sh', '-c', 'cat /var/tmp/logs.txt' ],
output: $logs,
timeout: 15
Expand All @@ -599,11 +598,11 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):

// Silently try to kill container
try {
$orchestration->remove($activeRuntimeId, true);
$orchestration->remove($runtimeName, true);
} catch (Throwable $th) {
}

$activeRuntimes->del($activeRuntimeId);
$activeRuntimes->del($runtimeName);

throw new Exception($error, 500);
}
Expand All @@ -616,11 +615,11 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):

// Silently try to kill container
try {
$orchestration->remove($activeRuntimeId, true);
$orchestration->remove($runtimeName, true);
} catch (Throwable $th) {
}

$activeRuntimes->del($activeRuntimeId);
$activeRuntimes->del($runtimeName);
}

$response
Expand Down Expand Up @@ -651,15 +650,15 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
->inject('response')
->inject('log')
->action(function (string $runtimeId, Table $activeRuntimes, Response $response, Log $log) {
$activeRuntimeId = $runtimeId; // Used with Swoole table (key)
$runtimeName = System::getHostname() . '-' . $runtimeId;

$log->addTag('runtimeId', $activeRuntimeId);
$log->addTag('runtimeId', $runtimeId);

if (!$activeRuntimes->exists($activeRuntimeId)) {
if (!$activeRuntimes->exists($runtimeName)) {
throw new Exception('Runtime not found', 404);
}

$runtime = $activeRuntimes->get($activeRuntimeId);
$runtime = $activeRuntimes->get($runtimeName);

$response
->setStatusCode(Response::STATUS_CODE_OK)
Expand All @@ -674,17 +673,16 @@ function removeAllRuntimes(Table $activeRuntimes, Orchestration $orchestration):
->inject('response')
->inject('log')
->action(function (string $runtimeId, Orchestration $orchestration, Table $activeRuntimes, Response $response, Log $log) {
$activeRuntimeId = $runtimeId; // Used with Swoole table (key)
$runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name)
$runtimeName = System::getHostname() . '-' . $runtimeId;

$log->addTag('runtimeId', $activeRuntimeId);
$log->addTag('runtimeId', $runtimeId);

if (!$activeRuntimes->exists($activeRuntimeId)) {
if (!$activeRuntimes->exists($runtimeName)) {
throw new Exception('Runtime not found', 404);
}

$orchestration->remove($runtimeId, true);
$activeRuntimes->del($activeRuntimeId);
$orchestration->remove($runtimeName, true);
$activeRuntimes->del($runtimeName);

$response
->setStatusCode(Response::STATUS_CODE_OK)
Expand Down Expand Up @@ -718,10 +716,9 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
$payload = '';
}

$activeRuntimeId = $runtimeId; // Used with Swoole table (key)
$runtimeId = System::getHostname() . '-' . $runtimeId; // Used in Docker (name)
$runtimeName = System::getHostname() . '-' . $runtimeId;

$log->addTag('runtimeId', $activeRuntimeId);
$log->addTag('runtimeId', $runtimeId);

$variables = \array_merge($variables, [
'INERNAL_EXECUTOR_HOSTNAME' => System::getHostname()
Expand All @@ -730,21 +727,21 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
$coldStartDuration = 0;

// Prepare runtime
if (!$activeRuntimes->exists($activeRuntimeId)) {
if (!$activeRuntimes->exists($runtimeName)) {
if (empty($image) || empty($source) || empty($entrypoint)) {
throw new Exception('Runtime not found. Please start it first or provide runtime-related parameters.', 401);
}

// Prepare request to executor
$sendCreateRuntimeRequest = function () use ($activeRuntimeId, $image, $source, $entrypoint, $variables, $cpus, $memory, $version, $runtimeEntrypoint) {
$sendCreateRuntimeRequest = function () use ($runtimeId, $image, $source, $entrypoint, $variables, $cpus, $memory, $version, $runtimeEntrypoint) {
$statusCode = 0;
$errNo = -1;
$executorResponse = '';

$ch = \curl_init();

$body = \json_encode([
'runtimeId' => $activeRuntimeId,
'runtimeId' => $runtimeId,
'image' => $image,
'source' => $source,
'entrypoint' => $entrypoint,
Expand Down Expand Up @@ -815,13 +812,13 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
}

// Update swoole table
$runtime = $activeRuntimes->get($activeRuntimeId) ?? [];
$runtime = $activeRuntimes->get($runtimeName) ?? [];
$runtime['updated'] = \time();
$activeRuntimes->set($activeRuntimeId, $runtime);
$activeRuntimes->set($runtimeName, $runtime);

// Ensure runtime started
for ($i = 0; $i < 10; $i++) {
if ($activeRuntimes->get($activeRuntimeId)['status'] !== 'pending') {
if ($activeRuntimes->get($runtimeName)['status'] !== 'pending') {
break;
}

Expand All @@ -833,7 +830,7 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
}

// Ensure we have secret
$runtime = $activeRuntimes->get($activeRuntimeId);
$runtime = $activeRuntimes->get($runtimeName);
$hostname = $runtime['hostname'];
$secret = $runtime['key'];
if (empty($secret)) {
Expand Down Expand Up @@ -1061,9 +1058,9 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
}

// Update swoole table
$runtime = $activeRuntimes->get($activeRuntimeId);
$runtime = $activeRuntimes->get($runtimeName);
$runtime['updated'] = \microtime(true);
$activeRuntimes->set($activeRuntimeId, $runtime);
$activeRuntimes->set($runtimeName, $runtime);

// Finish request
$response
Expand Down Expand Up @@ -1208,17 +1205,17 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
Timer::tick($interval * 1000, function () use ($orchestration, $activeRuntimes) {
Console::info("Running maintenance task ...");
// Stop idling runtimes
foreach ($activeRuntimes as $activeRuntimeId => $runtime) {
foreach ($activeRuntimes as $runtimeName => $runtime) {
$inactiveThreshold = \time() - \intval(Http::getEnv('OPR_EXECUTOR_INACTIVE_TRESHOLD', '60'));
if ($runtime['updated'] < $inactiveThreshold) {
go(function () use ($activeRuntimeId, $runtime, $orchestration, $activeRuntimes) {
go(function () use ($runtimeName, $runtime, $orchestration, $activeRuntimes) {
try {
$orchestration->remove($runtime['name'], true);
Console::success("Successfully removed {$runtime['name']}");
} catch (\Throwable $th) {
Console::error('Inactive Runtime deletion failed: ' . $th->getMessage());
} finally {
$activeRuntimes->del($activeRuntimeId);
$activeRuntimes->del($runtimeName);
}
});
}
Expand All @@ -1232,8 +1229,8 @@ function (string $runtimeId, ?string $payload, string $path, string $method, arr
if (\str_starts_with($entry, $prefix)) {
$isActive = false;

foreach ($activeRuntimes as $activeRuntimeId => $runtime) {
if (\str_ends_with($entry, $activeRuntimeId)) {
foreach ($activeRuntimes as $runtimeName => $runtime) {
if (\str_ends_with($entry, $runtimeName)) {
$isActive = true;
break;
}
Expand Down
4 changes: 2 additions & 2 deletions tests/ExecutorTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -152,12 +152,12 @@ public function testBuild(): array
$response = $this->client->call(Client::METHOD_GET, '/runtimes', [], []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals(1, count($response['body']));
$this->assertEquals('test-build', $response['body'][0]['name']);
$this->assertStringContainsString('test-build', $response['body'][0]['name']);

/** Get runtime */
$response = $this->client->call(Client::METHOD_GET, '/runtimes/test-build', [], []);
$this->assertEquals(200, $response['headers']['status-code']);
$this->assertEquals('test-build', $response['body']['name']);
$this->assertStringContainsString('test-build', $response['body']['name']);

/** Delete runtime */
$response = $this->client->call(Client::METHOD_DELETE, '/runtimes/test-build', [], []);
Expand Down

0 comments on commit 203d0e5

Please sign in to comment.