From fdeb3a0d1ede169ac99421acc66b518743fb86f5 Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Fri, 24 Nov 2017 14:23:49 +0100 Subject: [PATCH 1/2] [cli] support passing lua file as an environment The `-e` or `--environment` flag can be either and environment name or a path to a file. Plain names are resolved to the config directory in APIcast source. --- CHANGELOG.md | 1 + gateway/src/apicast/cli/command/start.lua | 85 ++++++++++---- gateway/src/apicast/cli/configuration.lua | 55 --------- gateway/src/apicast/cli/environment.lua | 135 ++++++++++++++++++++++ t/TestAPIcastBlackbox.pm | 45 ++++++-- 5 files changed, 230 insertions(+), 91 deletions(-) delete mode 100644 gateway/src/apicast/cli/configuration.lua create mode 100644 gateway/src/apicast/cli/environment.lua diff --git a/CHANGELOG.md b/CHANGELOG.md index 09c14661b..1967f1579 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Resolver can resolve nginx upstreams [PR #478](https://github.com/3scale/apicast/pull/478) - Calls 3scale backend with the 'no_body' option enabled. This reduces network traffic in cases where APIcast does not need to parse the response body [PR #483](https://github.com/3scale/apicast/pull/483) - Methods to modify policy chains [PR #505](https://github.com/3scale/apicast/pull/505) +- Ability to load several environment configurations [PR #504](https://github.com/3scale/apicast/pull/504) ## Changed diff --git a/gateway/src/apicast/cli/command/start.lua b/gateway/src/apicast/cli/command/start.lua index 5da8bbf5f..b1d040110 100644 --- a/gateway/src/apicast/cli/command/start.lua +++ b/gateway/src/apicast/cli/command/start.lua @@ -9,7 +9,7 @@ local exec = require('resty.execvp') local resty_env = require('resty.env') local Template = require('apicast.cli.template') -local configuration = require('apicast.cli.configuration') +local Environment = require('apicast.cli.environment') local pl = { path = require('pl.path'), @@ -26,7 +26,7 @@ local _M = { local mt = { __index = _M } -local function pick_openesty(candidates) +local function find_openresty_command(candidates) for i=1, #candidates do local ok = os.execute(('%s -V 2>/dev/null'):format(candidates[i])) @@ -44,10 +44,13 @@ local function update_env(env) end end -local function nginx_config(context, dir, path, env) - update_env(env) - local template = Template:new(context, dir, true) +local function apicast_root() + return resty_env.get('APICAST_DIR') or pl.path.abspath('.') +end + +local function nginx_config(context,path) + local template = Template:new(context, apicast_root(), true) local tmp = pl.path.tmpname() pl.file.write(tmp, template:render(path)) return tmp @@ -75,21 +78,42 @@ local function get_log_level(self, options) return log_level end -function mt:__call(options) - local openresty = resty_env.value('APICAST_OPENRESTY_BINARY') or - resty_env.value('TEST_NGINX_BINARY') or - pick_openesty(self.openresty) - local dir = resty_env.get('APICAST_DIR') or pl.path.abspath('.') - local config = configuration.new(dir) - local path = options.template - local environment = options.dev and 'development' or options.environment - local context = config:load(environment) - local env = { + +local function build_environment_config(options) + local config = Environment.new() + + for i=1, #options.environment do + local ok, err = config:add(options.environment[i]) + + if not ok then + print('not loading ', options.environment[i], ': ', err) + end + end + + if options.dev then config:add('development') end + + config:save() + + return config +end + +local function openresty_binary(candidates) + return resty_env.value('APICAST_OPENRESTY_BINARY') or + resty_env.value('TEST_NGINX_BINARY') or + find_openresty_command(candidates) +end + +local function build_env(options, config) + return { APICAST_CONFIGURATION = options.configuration, APICAST_CONFIGURATION_LOADER = options.boot and 'boot' or 'lazy', APICAST_CONFIGURATION_CACHE = options.cache, - THREESCALE_DEPLOYMENT_ENV = environment, + THREESCALE_DEPLOYMENT_ENV = config.name, } +end + +local function build_context(options, config) + local context = config:context() context.worker_processes = options.workers or context.worker_processes @@ -97,18 +121,30 @@ function mt:__call(options) context.daemon = 'on' end - if options.master and options.master[1] == 'off' then context.master_process = 'off' end - context.prefix = dir - context.ca_bundle = pl.path.abspath(context.ca_bundle or pl.path.join(dir, 'conf', 'ca-bundle.crt')) + context.prefix = apicast_root() + context.ca_bundle = pl.path.abspath(context.ca_bundle or pl.path.join(context.prefix, 'conf', 'ca-bundle.crt')) + + return context +end + +function mt:__call(options) + local openresty = openresty_binary(self.openresty) + local config = build_environment_config(options) + local context = build_context(options, config) + local env = build_env(options, config) + + local template_path = options.template + + update_env(env) -- also use env from the config file - update_env(config.env or {}) + update_env(context.env or {}) - local nginx = nginx_config(context, dir, path, env) + local nginx = nginx_config(context, template_path) local log_level = get_log_level(self, options) local log_file = options.log_file or self.log_file @@ -130,10 +166,9 @@ local function configure(cmd) cmd:usage("Usage: apicast-cli start [OPTIONS]") cmd:option("--template", "Nginx config template.", 'conf/nginx.conf.liquid') - cmd:mutex( - cmd:option('-e --environment', "Deployment to start.", resty_env.get('THREESCALE_DEPLOYMENT_ENV')), - cmd:flag('--dev', 'Start in development environment') - ) + + cmd:option('-e --environment', "Deployment to start. Can also be a path to a Lua file.", resty_env.get('THREESCALE_DEPLOYMENT_ENV') or 'production'):count('*') + cmd:flag('--dev', 'Start in development environment') cmd:flag("-m --master", "Test the nginx config"):args('?') cmd:flag("-t --test", "Test the nginx config") diff --git a/gateway/src/apicast/cli/configuration.lua b/gateway/src/apicast/cli/configuration.lua deleted file mode 100644 index 39eefea27..000000000 --- a/gateway/src/apicast/cli/configuration.lua +++ /dev/null @@ -1,55 +0,0 @@ -local pl_path = require('pl.path') -local resty_env = require('resty.env') -local setmetatable = setmetatable -local loadfile = loadfile -local pcall = pcall -local require = require -local assert = assert -local error = error -local print = print -local pairs = pairs - -local _M = { - default_environment = 'production', - default_config = { - ca_bundle = resty_env.get('SSL_CERT_FILE') - } -} - -local mt = { __index = _M } - -function _M.new(root) - return setmetatable({ root = root }, mt) -end - -function _M:load(env) - local environment = env or self.default_environment - local root = self.root - local name = resty_env.value('APICAST_ENVIRONMENT_CONFIG') or ("%s.lua"):format(environment) - local path = pl_path.abspath(name, pl_path.join(root, 'config')) - - print('loading config for: ', environment, ' environment from ', path) - - local config = loadfile(path, 't', { - print = print, inspect = require('inspect'), - pcall = pcall, require = require, assert = assert, error = error, - }) - - local default_config = {} - - if not config then - return default_config, 'invalid config' - end - - local table = config() - - for k,v in pairs(self.default_config) do - if table[k] == nil then - table[k] = v - end - end - - return table -end - -return _M diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua new file mode 100644 index 000000000..74a7ac5f7 --- /dev/null +++ b/gateway/src/apicast/cli/environment.lua @@ -0,0 +1,135 @@ +--- Environment configuration +-- @module environment +-- This module is providing a configuration to APIcast before and during its initialization. +-- You can load several configuration files. +-- Fields from the ones added later override fields from the previous configurations. +local pl_path = require('pl.path') +local resty_env = require('resty.env') +local linked_list = require('apicast.linked_list') +local setmetatable = setmetatable +local loadfile = loadfile +local pcall = pcall +local require = require +local assert = assert +local error = error +local print = print +local pairs = pairs +local insert = table.insert +local concat = table.concat +local re = require('ngx.re') + +local _M = {} +--- +-- @field default_environment Default environment name. +-- @table self +_M.default_environment = 'production' + +--- Default configuration. +-- @tfield ?string ca_bundle path to CA store file +-- @table environment.default_config default configuration +_M.default_config = { + ca_bundle = resty_env.get('SSL_CERT_FILE'), +} + +local mt = { __index = _M } + +--- Load an environment from files in ENV. +-- @treturn Environment +function _M.load() + local value = resty_env.value('APICAST_LOADED_ENVIRONMENTS') + local env = _M.new() + + if not value then + return env + end + + local environments = re.split(value, '\\|', 'jo') + + for i=1,#environments do + assert(env:add(environments[i])) + end + + return env +end + +--- Initialize new environment. +-- @treturn Environment +function _M.new() + return setmetatable({ _context = linked_list.readonly(_M.default_config), loaded = {} }, mt) +end + +local function expand_environment_name(name) + local root = resty_env.get('APICAST_DIR') or pl_path.abspath('.') + local pwd = resty_env.value('PWD') + + local path = pl_path.abspath(name, pwd) + local exists = pl_path.isfile(path) + + if exists then + return nil, path + end + + path = pl_path.join(root, 'config', ("%s.lua"):format(name)) + exists = pl_path.isfile(path) + + if exists then + return name, path + end +end + +--------------------- +--- @type Environment +-- An instance of @{environment} configuration. + +--- Add an environment name or configuration file. +-- @tparam string env environment name or path to a file +function _M:add(env) + local name, path = expand_environment_name(env) + + if self.loaded[path] then + return true, 'already loaded' + end + + if name and path then + self.name = name + print('loading ', name ,' environment configuration: ', path) + elseif path then + print('loading environment configuration: ', path) + else + return nil, 'no configuration found' + end + + local config = loadfile(path, 't', { + print = print, inspect = require('inspect'), context = self._context, + pcall = pcall, require = require, assert = assert, error = error, + }) + + if not config then + return nil, 'invalid config' + end + + self.loaded[path] = true + + self._context = linked_list.readonly(config(), self._context) + + return true +end + +--- Read/write context +-- @treturn table context with all loaded environments combined +function _M:context() + return linked_list.readwrite({ }, self._context) +end + +--- Store loaded environment file names into ENV. +function _M:save() + local environments = {} + + for file,_ in pairs(self.loaded) do + insert(environments, file) + end + + resty_env.set('APICAST_LOADED_ENVIRONMENTS', concat(environments, '|')) +end + +return _M diff --git a/t/TestAPIcastBlackbox.pm b/t/TestAPIcastBlackbox.pm index 469e5c9d2..3306aba57 100644 --- a/t/TestAPIcastBlackbox.pm +++ b/t/TestAPIcastBlackbox.pm @@ -17,7 +17,7 @@ add_block_preprocessor(sub { my $block = shift; my $seq = $block->seq_num; my $name = $block->name; - my $configuration = Test::Nginx::Util::expand_env_in_config($block->configuration); + my $configuration = $block->configuration; my $backend = $block->backend; my $upstream = $block->upstream; my $sites_d = $block->sites_d || ''; @@ -57,9 +57,12 @@ _EOC_ _EOC_ } - decode_json($configuration); + if (defined $configuration) { + $configuration = Test::Nginx::Util::expand_env_in_config($configuration); + decode_json($configuration); + $block->set_value("configuration", $configuration); + } - $block->set_value("configuration", $configuration); $block->set_value("config", "$name ($seq)"); $block->set_value('sites_d', $sites_d) }); @@ -79,12 +82,25 @@ my $write_nginx_config = sub { my $sites_d = $block->sites_d; - my ($conf, $configuration) = tempfile(); - print $conf $block->configuration; - close $conf; - my ($env, $env_file) = tempfile(); + my $configuration = $block->configuration; + my $conf; + my $configuration_file = $block->configuration_file; + + if (defined $configuration_file) { + chomp($configuration_file); + $configuration_file = "$configuration_file"; + } else { + if (defined $configuration) { + ($conf, $configuration_file) = tempfile(); + print $conf $configuration; + close $conf; + + $configuration_file = "$configuration_file"; + } + } + my ($env, $env_file) = tempfile(); print $env <<_EOC_; return { worker_processes = '$Workers', @@ -96,15 +112,16 @@ return { lua_code_cache = 'on', access_log = '$AccLogFile', port = { apicast = '$ServerPort' }, - env = { APICAST_CONFIGURATION = 'file://$configuration', APICAST_CONFIGURATION_LOADER = 'boot' }, + env = { + THREESCALE_CONFIG_FILE = [[$configuration_file]], + APICAST_CONFIGURATION_LOADER = 'boot', + }, sites_d = [============================[$sites_d]============================], } _EOC_ close $env; - $ENV{APICAST_ENVIRONMENT_CONFIG} = $env_file; - - my $apicast = `bin/apicast --boot --test --environment test --configuration $configuration 2>&1`; + my $apicast = `APICAST_CONFIGURATION_LOADER="" bin/apicast start --test --environment $env_file --configuration $configuration_file 2>&1`; if ($apicast =~ /configuration file (?.+?) test is successful/) { move($+{file}, $ConfFile); @@ -112,6 +129,12 @@ _EOC_ warn "Missing config file: $Test::Nginx::Util::ConfFile"; warn $apicast; } + + if ($PidFile && -f $PidFile) { + unlink $PidFile or warn "Couldn't remove $PidFile.\n"; + } + + $ENV{APICAST_LOADED_ENVIRONMENTS} = $env_file; }; BEGIN { From 20504a8769b7d44bb12c9c0d27d163027acef20a Mon Sep 17 00:00:00 2001 From: Michal Cichra Date: Mon, 27 Nov 2017 14:25:53 +0100 Subject: [PATCH 2/2] [cli] use resty_env.value instead of .get * .value returns nil when the value is empty string * so it can be chain with `or` for a default value --- CHANGELOG.md | 1 + gateway/src/apicast/cli/command/start.lua | 18 +++++++++--------- gateway/src/apicast/cli/environment.lua | 4 ++-- gateway/src/apicast/configuration_loader.lua | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1967f1579..53935a792 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ## Changed - Namespace all APIcast code in `apicast` folder. Possible BREAKING CHANGE for some customizations. [PR #486](https://github.com/3scale/apicast/pull/486) +- CLI ignores environment variables that are empty strings [PR #504](https://github.com/3scale/apicast/pull/504) ## Fixed diff --git a/gateway/src/apicast/cli/command/start.lua b/gateway/src/apicast/cli/command/start.lua index b1d040110..8ad9047d3 100644 --- a/gateway/src/apicast/cli/command/start.lua +++ b/gateway/src/apicast/cli/command/start.lua @@ -46,7 +46,7 @@ end local function apicast_root() - return resty_env.get('APICAST_DIR') or pl.path.abspath('.') + return resty_env.value('APICAST_DIR') or pl.path.abspath('.') end local function nginx_config(context,path) @@ -167,7 +167,7 @@ local function configure(cmd) cmd:option("--template", "Nginx config template.", 'conf/nginx.conf.liquid') - cmd:option('-e --environment', "Deployment to start. Can also be a path to a Lua file.", resty_env.get('THREESCALE_DEPLOYMENT_ENV') or 'production'):count('*') + cmd:option('-e --environment', "Deployment to start. Can also be a path to a Lua file.", resty_env.value('THREESCALE_DEPLOYMENT_ENV') or 'production'):count('*') cmd:flag('--dev', 'Start in development environment') cmd:flag("-m --master", "Test the nginx config"):args('?') @@ -175,23 +175,23 @@ local function configure(cmd) cmd:flag("--debug", "Debug mode. Prints more information.") cmd:option("-c --configuration", "Path to custom config file (JSON)", - resty_env.get('APICAST_CONFIGURATION')) + resty_env.value('APICAST_CONFIGURATION')) cmd:flag("-d --daemon", "Daemonize.") cmd:option("-w --workers", "Number of worker processes to start.", - resty_env.get('APICAST_WORKERS') or 1) + resty_env.value('APICAST_WORKERS') or 1) cmd:option("-p --pid", "Path to the PID file.") cmd:mutex( cmd:flag('-b --boot', "Load configuration on boot.", - resty_env.get('APICAST_CONFIGURATION_LOADER') == 'boot'), + resty_env.value('APICAST_CONFIGURATION_LOADER') == 'boot'), cmd:flag('-l --lazy', "Load configuration on demand.", - resty_env.get('APICAST_CONFIGURATION_LOADER') == 'lazy') + resty_env.value('APICAST_CONFIGURATION_LOADER') == 'lazy') ) cmd:option("-i --refresh-interval", "Cache configuration for N seconds. Using 0 will reload on every request (not for production).", - resty_env.get('APICAST_CONFIGURATION_CACHE')) + resty_env.value('APICAST_CONFIGURATION_CACHE')) cmd:mutex( cmd:flag('-v --verbose', @@ -200,8 +200,8 @@ local function configure(cmd) cmd:flag('-q --quiet', "Decrease logging verbosity.") :count(("0-%s"):format(_M.log_level - 1)) ) - cmd:option('--log-level', 'Set log level', resty_env.get('APICAST_LOG_LEVEL') or 'warn') - cmd:option('--log-file', 'Set log file', resty_env.get('APICAST_LOG_FILE') or 'stderr') + cmd:option('--log-level', 'Set log level', resty_env.value('APICAST_LOG_LEVEL') or 'warn') + cmd:option('--log-file', 'Set log file', resty_env.value('APICAST_LOG_FILE') or 'stderr') cmd:epilog([[ Example: apicast start --dev diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index 74a7ac5f7..5626fb53f 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -28,7 +28,7 @@ _M.default_environment = 'production' -- @tfield ?string ca_bundle path to CA store file -- @table environment.default_config default configuration _M.default_config = { - ca_bundle = resty_env.get('SSL_CERT_FILE'), + ca_bundle = resty_env.value('SSL_CERT_FILE'), } local mt = { __index = _M } @@ -59,7 +59,7 @@ function _M.new() end local function expand_environment_name(name) - local root = resty_env.get('APICAST_DIR') or pl_path.abspath('.') + local root = resty_env.value('APICAST_DIR') or pl_path.abspath('.') local pwd = resty_env.value('PWD') local path = pl_path.abspath(name, pwd) diff --git a/gateway/src/apicast/configuration_loader.lua b/gateway/src/apicast/configuration_loader.lua index 5e70028be..11ab3c2f2 100644 --- a/gateway/src/apicast/configuration_loader.lua +++ b/gateway/src/apicast/configuration_loader.lua @@ -217,7 +217,7 @@ local modes = { } function _M.new(mode) - mode = mode or env.get('APICAST_CONFIGURATION_LOADER') or modes.default + mode = mode or env.value('APICAST_CONFIGURATION_LOADER') or modes.default local loader = modes[mode] ngx.log(ngx.INFO, 'using ', mode, ' configuration loader') return assert(loader, 'invalid config loader mode')