[policy] implement sandboxed policy loader

* allows sandboxed load of code
  - includes vendored dependencies
  - policy can access just own source tree + shared APIcast code
  - loading same policy twice results in two different instances

CHANGELOG.md

@@ -47,6 +47,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Split 3scale authorization to rewrite and access phase [PR #556](https://github.com/3scale/apicast/pull/556)
 - Extract `mapping_rule` module from the `configuration` module [PR #571](https://github.com/3scale/apicast/pull/571)
 - Renamed `apicast/policy/policy.lua` to `apicast/policy.lua` [PR #569](https://github.com/3scale/apicast/pull/569)
+- Sandbox loading policies [PR #566](https://github.com/3scale/apicast/pull/566)
 
 ## [3.2.0-alpha2] - 2017-11-30
 

gateway/src/apicast/configuration.lua

@@ -60,7 +60,7 @@ local function build_policy_chain(policies)
   local chain = {}
 
   for i=1, #policies do
-    chain[i] = policy_chain.load(policies[i].name, policies[i].configuration)
+    chain[i] = policy_chain.load_policy(policies[i].name, policies[i].version, policies[i].configuration)
   end
 
   return policy_chain.new(chain)

gateway/src/apicast/policy_chain.lua

@@ -11,12 +11,15 @@ local rawset = rawset
 local type = type
 local require = require
 local insert = table.insert
+local sub = string.sub
+local format = string.format
 local noop = function() end
 
 require('apicast.loader')
 
 local linked_list = require('apicast.linked_list')
 local policy_phases = require('apicast.policy').phases
+local policy_loader = require('apicast.policy_loader')
 
 local _M = {
 
@@ -46,7 +49,7 @@ function _M.build(modules)
 
     for i=1, #list do
         -- TODO: make this error better, possibly not crash and just log and skip the module
-        chain[i] = _M.load(list[i]) or error("module " .. list[i] .. ' could not be loaded')
+        chain[i] = _M.load_policy(list[i]) or error(format('module %q could not be loaded', list[i]))
     end
 
     return _M.new(chain)
@@ -72,10 +75,14 @@ end
 -- @tparam string|table module the module or its name
 -- @tparam ?table ... params needed to initialize the module
 -- @treturn object The module instantiated
-function _M.load(module, ...)
+function _M.load_policy(module, version, ...)
     if type(module) == 'string' then
-        ngx.log(ngx.DEBUG, 'loading policy module: ', module)
-        local mod = require(module)
+        if sub(module, 1, 14) == 'apicast.policy' then
+            module = sub(module, 16)
+            version = 'builtin'
+        end
+
+        local mod = policy_loader(module, version or 'builtin')
 
         if mod then
             return mod.new(...)

gateway/src/apicast/policy_loader.lua

@@ -0,0 +1,280 @@
+--- Policy loader
+-- This module loads a policy defined by its name and version.
+-- It uses sandboxed require to isolate dependencies and not mutate global state.
+-- That allows for loading several versions of the same policy with different dependencies.
+-- And even loading several independent copies of the same policy with no shared state.
+-- Each object returned by the loader is new table and shares only shared APIcast code.
+
+local format = string.format
+local error = error
+local type = type
+local loadfile = loadfile
+local getenv = os.getenv
+local insert = table.insert
+local setmetatable = setmetatable
+local concat = table.concat
+
+local _G = _G
+local _M = {}
+
+local searchpath = package.searchpath
+local root_loaded = package.loaded
+
+local root_require = require
+
+local preload = package.preload
+
+--- create a require function not using the global namespace
+-- loading code from policy namespace should have no effect on the global namespace
+-- but poliocy can load shared libraries that would be cached globally
+local function gen_require(package)
+
+  local function not_found(modname, err)
+    return error(format("module '%s' not found:%s", modname, err), 0)
+  end
+
+  --- helper function to safely use the native require function
+  local function fallback(modname)
+    local mod
+
+    mod = package.loaded[modname]
+
+    if not mod then
+      ngx.log(ngx.DEBUG, 'native require for: ', modname)
+      mod = root_require(modname)
+    end
+
+    return mod
+  end
+
+  --- helper function to find and return correct loader for a module
+  local function find_loader(modname)
+    local loader, file, err, ret
+
+    -- http://www.lua.org/manual/5.2/manual.html#pdf-package.searchers
+
+    -- When looking for a module, require calls each of these searchers in ascending order,
+    for i=1, #package.searchers do
+      -- with the module name (the argument given to require) as its sole parameter.
+      ret, err = package.searchers[i](modname)
+
+      -- The function can return another function (the module loader)
+      -- plus an extra value that will be passed to that loader,
+      if type(ret) == 'function' then
+        loader = ret
+        file = err
+        break
+      -- or a string explaining why it did not find that module
+      elseif type(ret) == 'string' then
+        err = ret
+      end
+      -- (or nil if it has nothing to say).
+    end
+
+    return loader, file, err
+  end
+
+  --- reimplemented require function
+  -- - return a module if it was already loaded (globally or locally)
+  -- - try to find loader function
+  -- - fallback to global require
+  -- @tparam string modname module name
+  -- @tparam boolean exclusive load only policy code, turns off the fallback loader
+  return function(modname, exclusive)
+    -- http://www.lua.org/manual/5.2/manual.html#pdf-require
+    ngx.log(ngx.DEBUG, 'sandbox require: ', modname)
+
+    -- The function starts by looking into the package.loaded table
+    -- to determine whether modname is already loaded.
+    -- NOTE: this is different from the spec: use the top level package.loaded,
+    --       otherwise it would try to sandbox load already loaded shared code
+    local mod = root_loaded[modname]
+
+    --  If it is, then require returns the value stored at package.loaded[modname].
+    if mod then return mod end
+
+    -- Otherwise, it tries to find a loader for the module.
+    local loader, file, err = find_loader(modname)
+
+    -- Once a loader is found,
+    if loader then
+      ngx.log(ngx.DEBUG, 'sandboxed require for: ', modname, ' file: ', file)
+      -- require calls the loader with two arguments:
+      --   modname and an extra value dependent on how it got the loader.
+      -- (If the loader came from a file, this extra value is the file name.)
+      mod = loader(modname, file)
+    elseif not exclusive then
+      ngx.log(ngx.DEBUG, 'fallback loader for: ', modname, ' error: ', err)
+      mod = fallback(modname)
+    else
+      -- If there is any error loading or running the module,
+      -- or if it cannot find any loader for the module, then require raises an error.
+      return not_found(modname, err)
+    end
+
+    -- If the loader returns any non-nil value,
+    if mod ~= nil then
+      -- require assigns the returned value to package.loaded[modname].
+      package.loaded[modname] = mod
+
+      -- If the loader does not return a non-nil value
+      -- and has not assigned any value to package.loaded[modname],
+    elseif not package.loaded[modname] then
+      -- then require assigns true to this entry.
+      package.loaded[modname] = true
+    end
+
+    -- In any case, require returns the final value of package.loaded[modname].
+    return package.loaded[modname]
+  end
+end
+
+local function export(list, env)
+  assert(env, 'missing env')
+  list:gsub('%S+', function(id)
+    local module, method = id:match('([^%.]+)%.([^%.]+)')
+    if module then
+      env[module]         = env[module] or {}
+      env[module][method] = _G[module][method]
+    else
+      env[id] = _G[id]
+    end
+  end)
+
+  return env
+end
+
+--- this is environment exposed to the policies
+-- that means this is very light sandbox so policies don't mutate global env
+-- and most importantly we replace the require function with our own
+-- The env intentionally does not expose getfenv so sandboxed code can't get top level globals.
+-- And also does not expose functions for loading code from filesystem (loadfile, dofile).
+-- Neither exposes debug functions unless openresty was compiled --with-debug.
+-- But it exposes ngx as the same object, so it can be changed from within the policy.
+_M.env = export([[
+ _VERSION assert print xpcall pcall error
+ unpack next ipairs pairs select
+ collectgarbage gcinfo newproxy loadstring load
+ setmetatable getmetatable
+ tonumber tostring type
+ rawget rawequal rawlen rawset
+
+ bit.arshift bit.band bit.bnot bit.bor bit.bswap bit.bxor
+ bit.lshift bit.rol bit.ror bit.rshift bit.tobit bit.tohex
+
+ coroutine.create coroutine.resume coroutine.running coroutine.status
+ coroutine.wrap   coroutine.yield coroutine.isyieldable
+
+ debug.traceback
+
+ io.open io.close io.flush io.tmpfile io.type
+ io.input io.output io.stderr io.stdin io.stdout
+ io.popen io.read io.lines io.write
+
+ math.abs math.acos math.asin math.atan math.atan2
+ math.ceil math.cos math.cosh math.deg math.exp math.floor
+ math.fmod math.frexp math.ldexp math.log math.pi
+ math.log10 math.max math.min math.modf math.pow
+ math.rad math.random math.randomseed math.huge
+ math.sin math.sinh math.sqrt math.tan math.tanh
+
+ os.clock os.date os.time os.difftime
+ os.execute os.getenv
+ os.rename os.tmpname os.remove
+
+ string.byte string.char string.dump string.find
+ string.format string.lower string.upper string.len
+ string.gmatch string.match string.gsub string.sub
+ string.rep string.reverse
+
+ table.concat table.foreach table.foreachi table.getn
+ table.insert table.maxn table.move table.pack
+ table.remove table.sort table.unpack
+
+ ngx
+]], {})
+
+_M.env._G = _M.env
+
+-- add debug functions only when nginx was compiled --with-debug
+if ngx.config.debug then
+  _M.env = export([[ debug.debug debug.getfenv debug.gethook debug.getinfo
+      debug.getlocal debug.getmetatable debug.getregistry
+      debug.getupvalue debug.getuservalue debug.setfenv
+      debug.sethook debug.setlocal debug.setmetatable
+      debug.setupvalue debug.setuservalue debug.upvalueid debug.upvaluejoin
+  ]], _M.env)
+end
+
+local mt = {
+  __call = function(loader, ...) return loader.env.require(...) end
+}
+
+function _M.new(name, version, dir)
+  local apicast_dir = dir or getenv('APICAST_DIR') or '.'
+
+  local path = {
+    -- first path contains
+    format('%s/policies/%s/%s/?.lua', apicast_dir, name, version),
+  }
+
+  if version == 'builtin' then
+    insert(path, format('%s/src/apicast/policy/%s/?.lua', apicast_dir, name))
+  end
+
+  -- need to create global variable package that mimics the native one
+  local package = {
+    loaded = {},
+    preload = preload,
+    searchers = {}, -- http://www.lua.org/manual/5.2/manual.html#pdf-package.searchers
+    searchpath = searchpath,
+    path = concat(path, ';'),
+    cpath = '', -- no C libraries allowed in policies
+  }
+
+  -- creating new env for each policy means they can't accidentaly share global variables
+  local env = setmetatable({
+    require = gen_require(package),
+    package = package,
+  }, { __index = _M.env })
+
+  -- The first searcher simply looks for a loader in the package.preload table.
+  insert(package.searchers, function(modname) return package.preload[modname] end)
+  -- The second searcher looks for a loader as a Lua library, using the path stored at package.path.
+  -- The search is done as described in function package.searchpath.
+  insert(package.searchers, function(modname)
+    local file, err = searchpath(modname, package.path)
+    local loader
+
+    if file then
+      loader, err = loadfile(file, 'bt', env)
+
+      ngx.log(ngx.DEBUG, 'loading file: ', file)
+
+      if loader then return loader, file end
+    end
+
+    return err
+  end)
+
+  local self = {
+    env = env,
+    name = name,
+    version = version,
+  }
+
+  return setmetatable(self, mt)
+end
+
+function _M:call(name, version, dir)
+  local v = version or 'builtin'
+  local loader = self.new(name, v, dir)
+
+  ngx.log(ngx.DEBUG, 'loading policy: ', name, ' version: ', v)
+
+  -- passing the "exclusive" flag for the require so it does not fallback to native require
+  -- it should load only policies and not other code and fail if there is no such policy
+  return loader('policy', true)
+end
+
+return setmetatable(_M, { __call = _M.call })

spec/fixtures/policies/test/1.0.0-0/dependency.lua

@@ -0,0 +1,3 @@
+return {
+  '1.0 dependency'
+}

spec/fixtures/policies/test/1.0.0-0/policy.lua

@@ -0,0 +1,5 @@
+local Policy = require('apicast.policy').new('Test', '1.0.0-0')
+
+Policy.dependency = require('dependency')
+
+return Policy

spec/fixtures/policies/test/2.0.0-0/dependency.lua

@@ -0,0 +1,3 @@
+return {
+  '2.0 dependency'
+}

spec/fixtures/policies/test/2.0.0-0/policy.lua

@@ -0,0 +1,5 @@
+local Policy = require('apicast.policy').new('Test', '1.0.0-0')
+
+Policy.dependency = require('dependency')
+
+return Policy