From 536c26f4577effe898fa9d0ab2559ff6a0c635cf Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 23 Jun 2020 12:06:28 -0500 Subject: [PATCH 01/33] tl: add build options Adds luafilesystem dependency and the following config options - build_dir - source_dir - files - include - exclude - Original include -> include_dir --- docs/compiler_options.md | 80 ++++- ...{include_spec.lua => include_dir_spec.lua} | 0 tl | 314 ++++++++++++++---- tl-dev-1.rockspec | 3 + 4 files changed, 332 insertions(+), 65 deletions(-) rename spec/cli/{include_spec.lua => include_dir_spec.lua} (100%) diff --git a/docs/compiler_options.md b/docs/compiler_options.md index f634a7a40..85ccc7887 100644 --- a/docs/compiler_options.md +++ b/docs/compiler_options.md @@ -9,7 +9,7 @@ When running `tl`, the compiler will try to read the compilation options from a Here is an example of a `tlconfig.lua` file: ```lua return { - include = { + include_dir = { "folder1/", "folder2/" }, @@ -24,5 +24,81 @@ return { | Command line option | Config key | Type | Description | | --- | --- | --- | --- | | `-l --preload` | `preload_modules` | `{string}` | Execute the equivalent of `require('modulename')` before executing the tl script(s). | -| `-I --include` | `include` | `{string}` | Prepend this directory to the module search path. +| `-I --include-dir` | `include_dir` | `{string}` | Prepend this directory to the module search path. | `--skip-compat53` | | | Skip compat53 insertions. +|| `include` | `{string}` | The set of files to compile/check. See below for details on patterns. +|| `exclude` | `{string}` | The set of files to exclude. See below for details on patterns. +| `-s --source-dir` | `source_dir` | `string` | Set the directory to be searched for files. `gen` will compile every .tl file in every subdirectory by default. +| `-b --build-dir` | `build_dir` | `string` | Set the directory for generated files, mimicing the file structure of the source files. +|| `files` | `{string}` | The names of files to be compiled. Does not accept patterns like `include`. + +### Include/Exclude patterns + +Similar to tsconfig.json, the `include` and `exclude` fields can have glob-like patterns in them. +- `*`: Matches any number of characters (excluding directory separators) +- `**/`: Matches any number subdirectories + +In addition, setting the `source_dir` has the effect of prepending `source_dir` to all patterns + +For example: +If our project was laid out as such: +``` +tlconfig.lua +src/ +| foo/ +| | bar.tl +| | baz.tl +| bar/ +| | a/ +| | | foo.tl +| | b/ +| | | foo.tl +``` + +and our tlconfig.lua contained the following: +```lua +return { + source_dir = "src", + build_dir = "build", + include = { + "foo/*.tl", + "bar/**/*.tl" + }, + exclude = { + "foo/bar.tl" + } +} +``` + +Running `tl check` will type check the `include`d files. + +Running `tl gen` with no arguments would produce the following files. +``` +tlconfig.lua +src/ +| foo/ +| | bar.tl +| | baz.tl +| bar/ +| | a/ +| | | foo.tl +| | b/ +| | | foo.tl +build/ +| foo/ +| | baz.lua +| bar/ +| | a/ +| | | foo.lua +| | b/ +| | | foo.lua +``` + +Additionally, complex patterns can be used for whatever convoluted file structure we need. +```lua +return { + include = { + "foo/**/bar/**/baz/**/*.tl" + } +} +``` diff --git a/spec/cli/include_spec.lua b/spec/cli/include_dir_spec.lua similarity index 100% rename from spec/cli/include_spec.lua rename to spec/cli/include_dir_spec.lua diff --git a/tl b/tl index b6339d4e9..c6ffd9e80 100755 --- a/tl +++ b/tl @@ -1,6 +1,7 @@ #!/usr/bin/env lua local version_string = "0.7.1+dev" +local path_separator = package.config:sub(1, 1) local function script_path() local str = debug.getinfo(2, "S").source:sub(2) @@ -37,9 +38,15 @@ end -- FIXME local function validate_config(config) local valid_keys = { - preload_modules = true, + build_dir = true, + files = true, + ignore = true, include = true, - quiet = true + exclude = true, + include_dir = true, + preload_modules = true, + quiet = true, + source_dir = true, } for k, _ in pairs(config) do @@ -56,7 +63,7 @@ end local function get_config() local config = { preload_modules = {}, - include = {}, + include_dir = {}, quiet = false } @@ -88,6 +95,7 @@ package.path = script_path() .. "/?.lua;" .. package.path local tl = require("tl") local argparse = require("argparse") +local lfs = require("lfs") local function get_args_parser() local parser = argparse("tl", "A minimalistic typed dialect of Lua.") @@ -96,10 +104,16 @@ local function get_args_parser() :argname("") :count("*") - parser:option("-I --include", "Prepend this directory to the module search path.") + parser:option("-I --include-dir", "Prepend this directory to the module search path.") :argname("") :count("*") + parser:option("-s --source-dir", "Compile all *.tl files in .") + :argname("") + + parser:option("-b --build-dir", "Put all generated files in .") + :argname("") + parser:flag("--skip-compat53", "Skip compat53 insertions.") parser:flag("--version", "Print version and exit") @@ -110,10 +124,10 @@ local function get_args_parser() parser:command_target("command") local check_command = parser:command("check", "Type-check one or more tl script.") - check_command:argument("script", "The tl script."):args("+") + check_command:argument("script", "The tl script."):args("*") local gen_command = parser:command("gen", "Generate a Lua file for one or more tl script.") - gen_command:argument("script", "The tl script."):args("+") + gen_command:argument("script", "The tl script."):args("*") local run_command = parser:command("run", "Run a tl script.") run_command:argument("script", "The tl script."):args("+") @@ -146,15 +160,17 @@ for _, preload_module_cli in ipairs(args["preload"]) do end end -for _, include_dir_cli in ipairs(args["include"]) do - if not find_in_sequence(tlconfig.include, include_dir_cli) then - table.insert(tlconfig.include, include_dir_cli) +for _, include_dir_cli in ipairs(args["include_dir"]) do + if not find_in_sequence(tlconfig.include_dir, include_dir_cli) then + table.insert(tlconfig.include_dir, include_dir_cli) end end if args["quiet"] then tlconfig["quiet"] = true end +tlconfig["source_dir"] = args["source_dir"] or tlconfig["source_dir"] +tlconfig["build_dir"] = args["build_dir"] or tlconfig["build_dir"] local function report_errors(category, errors) if not errors then @@ -192,8 +208,6 @@ local function get_shared_library_ext() end local function prepend_to_path(directory) - local path_separator = package.config:sub(1, 1) - local path_str = directory if string.sub(path_str, -1) == path_separator then @@ -209,13 +223,11 @@ local function prepend_to_path(directory) package.cpath = lib_path_str .. package.cpath end -for _, include in ipairs(tlconfig["include"]) do +for _, include in ipairs(tlconfig["include_dir"]) do prepend_to_path(include) end -for i, filename in ipairs(args["script"]) do - local modules = i == 1 and tlconfig.preload_modules - +local function setup_env(filename) if not env then local basename, extension = filename:match("(.*)%.([a-z]+)$") extension = extension and extension:lower() @@ -226,8 +238,7 @@ for i, filename in ipairs(args["script"]) do elseif extension == "lua" then lax_mode = true else - -- if we can't decide based on the file extension, default to strict - -- mode + -- if we can't decide based on the file extension, default to strict mode lax_mode = false end @@ -235,7 +246,9 @@ for i, filename in ipairs(args["script"]) do env = tl.init_env(lax_mode, skip_compat53) end +end +local function type_check_and_load(filename, modules) local result, err = tl.process(filename, env, nil, modules) if err then die(err) @@ -245,91 +258,266 @@ for i, filename in ipairs(args["script"]) do local has_syntax_errors = report_errors("syntax error", result.syntax_errors) if has_syntax_errors then exit = 1 - break + return + end + if filename:match("%.tl$") then + local ok = report_type_errors(result) + if not ok then + os.exit(1) + end end - local lua_name = filename:gsub(".tl$", ".lua") + local chunk = (loadstring or load)(tl.pretty_print_ast(result.ast), "@" .. filename) + return chunk +end - if cmd == "run" then - if filename:match("%.tl$") then - local ok = report_type_errors(result) - if not ok then - os.exit(1) - end +-- if were running a script, we don't need to build up a source map +local modules = tlconfig.preload_modules +if cmd == "run" then + setup_env(args["script"][1]) + local chunk = type_check_and_load(args["script"][1], modules) + + -- collect all non-arguments including negative arg values + local neg_arg = {} + local nargs = #args["script"] + local j = #arg + local p = nargs + local n = 1 + while arg[j] do + if arg[j] == args["script"][p] then + p = p - 1 + else + neg_arg[n] = arg[j] + n = n + 1 end + j = j - 1 + end - local chunk = (loadstring or load)(tl.pretty_print_ast(result.ast), "@" .. filename) - - -- collect all non-arguments including negative arg values - local neg_arg = {} - local nargs = #args["script"] - local j = #arg - local p = nargs - local n = 1 - while arg[j] do - if arg[j] == args["script"][p] then - p = p - 1 + -- shift back all non-arguments to negative positions + for p, a in ipairs(neg_arg) do + arg[-p] = a + end + -- put script in arg[0] and arguments in positive positions + for p, a in ipairs(args["script"]) do + arg[p - 1] = a + end + -- cleanup the rest + n = nargs + while arg[n] do + arg[n] = nil + n = n + 1 + end + + tl.loader() + return chunk() +end + +-- for check and gen, build a source map +local src_map = {} +local inc_patterns = {} +local exc_patterns = {} + +-- prepare build and source dirs +local function path_concat(...) + return table.concat({...}, path_separator) +end +local function traverse(dirname) + local files = {} + for file in lfs.dir(dirname) do + if file ~= "." and file ~= ".." then + if lfs.attributes(path_concat(dirname, file)).mode == "directory" then + local dir = traverse(path_concat(dirname, file)) + for input, output in pairs(dir) do + files[input] = output + end else - neg_arg[n] = arg[j] - n = n + 1 + local output = file + if file:match(".tl$") and not file:match(".d.tl$") then + output = file:gsub(".tl$", ".lua") + end + files[path_concat(dirname, file)] = path_concat(dirname, output) end - j = j - 1 end + end + return files +end - -- shift back all non-arguments to negative positions - for p, a in ipairs(neg_arg) do - arg[-p] = a +-- include/exclude pattern matching +local function str_split(str, delimiter) + local idx = 1 + return function() + if not idx then return end + local prev_idx = idx + local s_idx + s_idx, idx = str:find(delimiter, idx, true) + return str:sub(prev_idx, (s_idx or 0) - 1) + end +end +local function patt_match(patt, str) + local matches = true + local idx = 1 + local s_idx + for _, v in ipairs(patt) do + s_idx, idx = str:find(v, idx) + if not s_idx then + matches = false + break end - -- put script in arg[0] and arguments in positive positions - for p, a in ipairs(args["script"]) do - arg[p - 1] = a + end + return matches +end +local function matcher(str) + local chunks = {} + for piece in str_split(str, "**/") do + table.insert(chunks, (piece:gsub("%*", "[^" .. path_separator .. "]*"))) + end + chunks[1] = "^" .. chunks[1] + chunks[#chunks] = chunks[#chunks] .. "$" + return function(str) + return patt_match(chunks, str) + end +end + +if #args["script"] == 0 then + if tlconfig["include"] then + for i, patt in ipairs(tlconfig["include"]) do + if tlconfig["source_dir"] then + patt = path_concat(tlconfig["source_dir"], patt) + end + table.insert(inc_patterns, matcher(patt)) end - -- cleanup the rest - n = nargs - while arg[n] do - arg[n] = nil - n = n + 1 + end + if tlconfig["exclude"] then + for i, patt in ipairs(tlconfig["exclude"]) do + if tlconfig["source_dir"] then + patt = path_concat(tlconfig["source_dir"], patt) + end + table.insert(exc_patterns, matcher(patt)) + end + end + + if tlconfig["source_dir"] then + src_map = traverse(tlconfig["source_dir"]) + else + src_map = traverse(lfs.currentdir()) + end +else + for i, filename in ipairs(args["script"]) do + src_map[filename] = filename:gsub(".tl$", ".lua") + end +end + +local curr_dir = lfs.currentdir() +local function remove_lead_path(path, leading) + return (path:gsub("^" .. leading .. path_separator, "")) +end +local function to_relative(path) + return remove_lead_path(path, curr_dir) +end + +if tlconfig["build_dir"] and cmd == "gen" then + table.insert(exc_patterns, matcher(path_concat(tlconfig["build_dir"], "**/"))) + for input_file, output_file in pairs(src_map) do + output_file = to_relative(output_file) + if tlconfig["source_dir"] then + output_file = remove_lead_path(output_file, tlconfig["source_dir"]) end - tl.loader() + local new_path = path_concat(tlconfig["build_dir"], output_file) + local path = {} + for dir in new_path:gmatch("[^%" .. path_separator .. "]+") do + path[#path + 1] = #path > 0 and path_concat(path[#path], dir) or dir + end + table.remove(path) + for i, v in ipairs(path) do + local attr = lfs.attributes(v) + if not attr then + lfs.mkdir(v) + elseif attr.mode ~= "directory" then + die("Error in build directory: " .. dir .. " is not a directory.") + end + end + src_map[input_file] = new_path + end +end - return chunk() +for input_file, output_file in pairs(src_map) do + src_map[input_file] = to_relative(output_file) +end - elseif cmd == "check" then +if #args["script"] == 0 then + for input_file, output_file in pairs(src_map) do + rel_input_file = to_relative(input_file) + local include = true + for _, patt in ipairs(inc_patterns) do + if not patt(rel_input_file) then + include = false + break + end + end + for _, patt in ipairs(exc_patterns) do + if patt(rel_input_file) then + include = false + break + end + end + if not include then + src_map[input_file] = nil + end + end +end + +for input_file, output_file in pairs(src_map) do + setup_env(input_file) + + local result, err = tl.process(input_file, env, nil, modules) + if err then + die(err) + end + env = result.env + + local has_syntax_errors = report_errors("syntax error", result.syntax_errors) + if has_syntax_errors then + exit = 1 + break + end + + if cmd == "check" then local ok = report_type_errors(result) if not ok then exit = 1 end - if exit == 0 and #args["script"] == 1 and tlconfig["quiet"] == false then + if exit == 0 and tlconfig["quiet"] == false and #args["script"] == 1 then print("========================================") - print("Type checked " .. filename) + print("Type checked " .. input_file) print("0 errors detected -- you can use:") print() - print(" tl run " .. filename) + print(" tl run " .. input_file) print() - print(" to run " .. filename .. " as a program") + print(" to run " .. input_file .. " as a program") print() - print(" tl gen " .. filename) + print(" tl gen " .. input_file) print() - print(" to generate " .. lua_name) + print(" to generate " .. output_file) end elseif cmd == "gen" then - local ofd, err = io.open(lua_name, "w") + local ofd, err = io.open(output_file, "w") + if not ofd then - die("cannot write " .. lua_name .. ": " .. err) + die("cannot write " .. output_file .. ": " .. err) end local ok, err = ofd:write(tl.pretty_print_ast(result.ast) .. "\n") if err then - die("error writing " .. lua_name .. ": " .. err) + die("error writing " .. output_file .. ": " .. err) end ofd:close() if tlconfig["quiet"] == false then - print("Wrote: " .. lua_name) + print("Wrote: " .. output_file) end end end diff --git a/tl-dev-1.rockspec b/tl-dev-1.rockspec index 5f43cac61..a1f67aff5 100644 --- a/tl-dev-1.rockspec +++ b/tl-dev-1.rockspec @@ -17,6 +17,9 @@ dependencies = { -- needed for the cli tool "argparse" + + -- needed for --source-dir + "lfs" } build = { modules = { From c2df4d95f77d0d9ca12eeaea230d174a42bef9f1 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 23 Jun 2020 15:54:40 -0500 Subject: [PATCH 02/33] tl: completely ignore .d.tl files when traversing directory --- tl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tl b/tl index c6ffd9e80..337514fc3 100755 --- a/tl +++ b/tl @@ -331,11 +331,10 @@ local function traverse(dirname) files[input] = output end else - local output = file if file:match(".tl$") and not file:match(".d.tl$") then - output = file:gsub(".tl$", ".lua") + local output = file:gsub(".tl$", ".lua") + files[path_concat(dirname, file)] = path_concat(dirname, output) end - files[path_concat(dirname, file)] = path_concat(dirname, output) end end end From 747e9ca95559376ead4faf714fbd816cf49cbc0e Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 23 Jun 2020 18:18:01 -0500 Subject: [PATCH 03/33] docs: remove typescript reference, rockspec: fix lfs dependency --- docs/compiler_options.md | 2 +- tl-dev-1.rockspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/compiler_options.md b/docs/compiler_options.md index 85ccc7887..c84e10765 100644 --- a/docs/compiler_options.md +++ b/docs/compiler_options.md @@ -34,7 +34,7 @@ return { ### Include/Exclude patterns -Similar to tsconfig.json, the `include` and `exclude` fields can have glob-like patterns in them. +The `include` and `exclude` fields can have glob-like patterns in them: - `*`: Matches any number of characters (excluding directory separators) - `**/`: Matches any number subdirectories diff --git a/tl-dev-1.rockspec b/tl-dev-1.rockspec index a1f67aff5..8619e4dfe 100644 --- a/tl-dev-1.rockspec +++ b/tl-dev-1.rockspec @@ -19,7 +19,7 @@ dependencies = { "argparse" -- needed for --source-dir - "lfs" + "luafilesystem" } build = { modules = { From 14b8a6650dc89cdfaf05d1483417b5d83e4f7ce7 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Wed, 24 Jun 2020 16:40:07 -0500 Subject: [PATCH 04/33] tl run: exit when load/loadstring reports error --- tl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tl b/tl index 337514fc3..813a15326 100755 --- a/tl +++ b/tl @@ -257,8 +257,7 @@ local function type_check_and_load(filename, modules) local has_syntax_errors = report_errors("syntax error", result.syntax_errors) if has_syntax_errors then - exit = 1 - return + os.exit(1) end if filename:match("%.tl$") then local ok = report_type_errors(result) @@ -267,7 +266,10 @@ local function type_check_and_load(filename, modules) end end - local chunk = (loadstring or load)(tl.pretty_print_ast(result.ast), "@" .. filename) + local chunk, err = (loadstring or load)(tl.pretty_print_ast(result.ast), "@" .. filename) + if err then + die(err) + end return chunk end From 7216cb9b70d549fdb1c376c5c95f131282e772d0 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Thu, 25 Jun 2020 18:45:01 -0500 Subject: [PATCH 05/33] tl: fix include/exclude logic and **/ not matching current dir --- tl | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/tl b/tl index 813a15326..c7c1593c3 100755 --- a/tl +++ b/tl @@ -345,9 +345,10 @@ end -- include/exclude pattern matching local function str_split(str, delimiter) - local idx = 1 + local idx = 0 return function() if not idx then return end + idx = idx + 1 local prev_idx = idx local s_idx s_idx, idx = str:find(delimiter, idx, true) @@ -449,17 +450,19 @@ end if #args["script"] == 0 then for input_file, output_file in pairs(src_map) do rel_input_file = to_relative(input_file) - local include = true + local include = false for _, patt in ipairs(inc_patterns) do - if not patt(rel_input_file) then - include = false + if patt(rel_input_file) then + include = true break end end - for _, patt in ipairs(exc_patterns) do - if patt(rel_input_file) then - include = false - break + if include then + for _, patt in ipairs(exc_patterns) do + if patt(rel_input_file) then + include = false + break + end end end if not include then From 3822d84efb8643d62cf7e156caa35e4abfbbd662 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Thu, 2 Jul 2020 21:31:53 -0500 Subject: [PATCH 06/33] spec: Add initial tests for globbing In particular adds the following util function: util.write_tmp_dir, which lets us build up a directory structure in /tmp to test out tl gen. The run_mock_project will probably move to util as more tests get written, but in short: it uses util.write_tmp_dir then makes a link to tl and tl.lua in the dir, does a lfs.chdir, and runs ./tl gen. Then it asserts that the generates files are as expected, and cleans up after itself by removing the dir and does a lfs.chdir back to the correct dir --- spec/config/glob_spec.lua | 345 ++++++++++++++++++++++++++++++++++++++ spec/util.lua | 53 ++++++ 2 files changed, 398 insertions(+) create mode 100644 spec/config/glob_spec.lua diff --git a/spec/config/glob_spec.lua b/spec/config/glob_spec.lua new file mode 100644 index 000000000..049471574 --- /dev/null +++ b/spec/config/glob_spec.lua @@ -0,0 +1,345 @@ +local util = require("spec.util") +local lfs = require("lfs") + +local tl_path = lfs.currentdir() +local tl_executable = tl_path .. "/tl" +local tl_lib = tl_path .. "/tl.lua" +local function run_mock_project(t) + local actual_dir_name = util.write_tmp_dir(finally, t.dir_name, t.dir_structure) + lfs.link(tl_executable, actual_dir_name .. "/tl") + lfs.link(tl_lib, actual_dir_name .. "/tl.lua") + local expected_dir_structure = { + ["tl"] = true, + ["tl.lua"] = true, + } + local function insert_into(tab, files) + for k, v in pairs(files) do + if type(k) == "number" then + tab[v] = true + elseif type(v) == "table" then + if not tab[k] then + tab[k] = {} + end + insert_into(tab[k], v) + elseif type(v) == "string" then + tab[k] = true + end + end + end + insert_into(expected_dir_structure, t.dir_structure) + insert_into(expected_dir_structure, t.generated_files) + lfs.chdir(actual_dir_name) + local pd = io.popen("./tl gen") + local output = pd:read("*a") + local actual_dir_structure = util.get_dir_structure(".") + lfs.chdir(tl_path) + t.popen_close = t.popen_close or {} + util.assert_popen_close( + t.popen_close[1] or true, + t.popen_close[2] or "exit", + t.popen_close[3] or 0, + pd:close() + ) + if t.cmd_output then --FIXME + assert.are.equal(output, t.cmd_output) + end + assert.are.same(expected_dir_structure, actual_dir_structure) +end + +describe("globs", function() + describe("*", function() + it("should match non directory separators", function() + run_mock_project{ + dir_name = "non_dir_sep_test", + dir_structure = { + ["tlconfig.lua"] = [[return { include = {"*"} }]], + ["a.tl"] = [[print "a"]], + ["b.tl"] = [[print "b"]], + ["c.tl"] = [[print "c"]], + }, + generated_files = { + "a.lua", + "b.lua", + "c.lua", + }, + --FIXME: order is not guaranteed, fix either in here or in tl itself + --cmd_output = "Wrote: a.lua\nWrote: b.lua\nWrote: c.lua\n" + } + end) + it("should match when other characters are present in the pattern", function() + run_mock_project{ + dir_name = "other_chars_test", + dir_structure = { + ["tlconfig.lua"] = [[return { include = { "ab*cd.tl" } }]], + ["abzcd.tl"] = [[print "a"]], + ["abcd.tl"] = [[print "b"]], + ["abfoocd.tl"] = [[print "c"]], + ["abbarcd.tl"] = [[print "d"]], + }, + generated_files = { + "abzcd.lua", + "abcd.lua", + "abfoocd.lua", + "abbarcd.lua", + }, + --FIXME cmd_output = "Wrote: abzcd.lua\n", + } + end) + it("should only match .tl by default", function() + run_mock_project{ + dir_name = "match_only_teal_test", + dir_structure = { + ["tlconfig.lua"] = [[return { include = { "*" } }]], + ["foo.tl"] = [[print "a"]], + ["foo.py"] = [[print("b")]], + ["foo.hs"] = [[main = print "c"]], + ["foo.sh"] = [[echo "d"]], + }, + generated_files = { + "foo.lua" + }, + } + end) + it("should not match .d.tl files", function() + run_mock_project{ + dir_name = "dont_match_d_tl", + dir_structure = { + ["tlconfig.lua"] = [[return { include = { "*" } }]], + ["foo.tl"] = [[print "a"]], + ["bar.d.tl"] = [[local Point = record x: number y: number end return Point]], + }, + generated_files = { + "foo.lua" + }, + } + end) + it("should match directories in the middle of a path", function() + run_mock_project{ + dir_name = "match_dirs_in_middle_test", + dir_structure = { + ["tlconfig.lua"] = [[return { include = { "foo/*/baz.tl" } }]], + ["foo"] = { + ["bar"] = { + ["foo.tl"] = [[print "a"]], + ["baz.tl"] = [[print "b"]], + }, + ["bingo"] = { + ["foo.tl"] = [[print "c"]], + ["baz.tl"] = [[print "d"]], + }, + ["bongo"] = { + ["foo.tl"] = [[print "e"]], + }, + } + }, + generated_files = { + ["foo"] = { + ["bar"] = { + "baz.lua" + }, + ["bingo"] = { + "baz.lua" + }, + }, + }, + } + end) + end) + describe("**/", function() + it("should match the current directory", function() + run_mock_project{ + dir_name = "match_current_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return { include = { "**/*" } }]], + ["foo.tl"] = [[print "a"]], + ["bar.tl"] = [[print "b"]], + ["baz.tl"] = [[print "c"]], + }, + generated_files = { + "foo.lua", + "bar.lua", + "baz.lua", + }, + } + end) + it("should match any subdirectory", function() + run_mock_project{ + dir_name = "match_current_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return { include = { "**/*" } }]], + ["foo"] = { + ["foo.tl"] = [[print "a"]], + ["bar.tl"] = [[print "b"]], + ["baz.tl"] = [[print "c"]], + }, + ["bar"] = { + ["foo.tl"] = [[print "a"]], + ["baz"] = { + ["bar.tl"] = [[print "b"]], + ["baz.tl"] = [[print "c"]], + } + }, + ["a"] = {a={a={a={a={a={["a.tl"]=[[global a = "a"]]}}}}}} + }, + generated_files = { + ["foo"] = { + "foo.lua", + "bar.lua", + "baz.lua", + }, + ["bar"] = { + "foo.lua", + ["baz"] = { + "bar.lua", + "baz.lua", + } + }, + ["a"] = {a={a={a={a={a={"a.lua"}}}}}}, + }, + } + end) + it("should not get the order of directories confused", function() + run_mock_project{ + dir_name = "match_current_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return { include = { "foo/**/bar/**/baz/a.tl" } }]], + ["foo"] = { + ["bar"] = { + ["baz"] = { + ["a.tl"] = [[print "a"]], + }, + }, + }, + ["baz"] = { + ["bar"] = { + ["foo"] = { + ["a.tl"] = [[print "a"]], + }, + }, + }, + ["bar"] = { + ["baz"] = { + ["foo"] = { + ["a.tl"] = [[print "a"]], + }, + }, + }, + }, + generated_files = { + ["foo"] = { + ["bar"] = { + ["baz"] = { + "a.lua", + } + } + }, + }, + } + end) + end) + describe("* and **/", function() + it("should work together", function() + run_mock_project{ + dir_name = "glob_interference_test", + dir_structure = { + ["tlconfig.lua"] = [[return { include = { "**/foo/*/bar/**/*" } }]], + ["foo"] = { + ["a"] = { + ["bar"] = { + ["baz"] = { + ["a"] = {["b"] = {["c.tl"] = [[print "c"]]}}, + }, + ["bat"] = { + ["a"] = {["b.tl"] = [[print "b"]]}, + }, + }, + }, + ["b"] = { + ["bar"] = { + ["a"] = {["b.tl"] = [[print "b"]]}, + }, + }, + ["c"] = { + ["d"] = { + ["bar"] = { + ["a.tl"] = [[print "not included"]] + }, + }, + }, + }, + ["a"] = { + ["b"] = { + ["foo"] = { + ["a"] = { + ["bar"] = { + ["baz"] = { + ["a"] = {["b"] = {["c.tl"] = [[print "c"]]}}, + }, + ["bat"] = { + ["a"] = {["b.tl"] = [[print "b"]]}, + }, + }, + }, + ["b"] = { + ["bar"] = { + ["baz"] = { + ["a"] = {["b"] = {["c.tl"] = [[print "c"]]}}, + }, + ["bat"] = { + ["a"] = {["b.tl"] = [[print "b"]]}, + }, + }, + }, + }, + }, + }, + }, + generated_files = { + ["foo"] = { + ["a"] = { + ["bar"] = { + ["baz"] = { + ["a"] = {["b"] = {"c.lua"}}, + }, + ["bat"] = { + ["a"] = {"b.lua"}, + }, + }, + }, + ["b"] = { + ["bar"] = { + ["a"] = {"b.lua"}, + }, + }, + }, + ["a"] = { + ["b"] = { + ["foo"] = { + ["a"] = { + ["bar"] = { + ["baz"] = { + ["a"] = {["b"] = {"c.lua"}}, + }, + ["bat"] = { + ["a"] = {"b.lua"}, + }, + }, + }, + ["b"] = { + ["bar"] = { + ["baz"] = { + ["a"] = {["b"] = {"c.lua"}}, + }, + ["bat"] = { + ["a"] = {"b.lua"}, + }, + }, + }, + }, + }, + }, + }, + } + end) + end) +end) diff --git a/spec/util.lua b/spec/util.lua index acb06b87c..4c33d9f73 100644 --- a/spec/util.lua +++ b/spec/util.lua @@ -2,6 +2,7 @@ local util = {} local tl = require("tl") local assert = require("luassert") +local lfs = require("lfs") function util.mock_io(finally, files) assert(type(finally) == "function") @@ -69,6 +70,58 @@ function util.write_tmp_file(finally, name, content) return full_name end +function util.write_tmp_dir(finally, dir_name, dir_structure) + assert(type(finally) == "function") + assert(type(dir_name) == "string") + assert(type(dir_structure) == "table") + + local full_name = "/tmp/" .. dir_name .. "/" + assert(lfs.mkdir(full_name)) + local function traverse_dir(dir_structure, prefix) + prefix = prefix or full_name + for name, content in pairs(dir_structure) do + if type(content) == "table" then + assert(lfs.mkdir(prefix .. name)) + traverse_dir(content, prefix .. name .. "/") + else + local fd = io.open(prefix .. name, "w") + fd:write(content) + fd:close() + end + end + end + traverse_dir(dir_structure) + finally(function() + os.execute("rm -r " .. full_name) + -- local function rm_dir(dir_structure, prefix) + -- prefix = prefix or full_name + -- for name, content in pairs(dir_structure) do + -- if type(content) == "table" then + -- rm_dir(prefix .. name .. "/") + -- end + -- os.remove(prefix .. name) + -- end + -- end + -- rm_dir(dir_structure) + end) + return full_name +end + +function util.get_dir_structure(dir_name) + -- basically run `tree` and put it into a table + local dir_structure = {} + for fname in lfs.dir(dir_name) do + if fname ~= ".." and fname ~= "." then + if lfs.attributes(dir_name .. "/" .. fname, "mode") == "directory" then + dir_structure[fname] = util.get_dir_structure(dir_name .. "/" .. fname) + else + dir_structure[fname] = true + end + end + end + return dir_structure +end + function util.read_file(name) assert(type(name) == "string") From 4a0b73a87d7682889bc9b288c5d7ecba2d2b23b9 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Thu, 2 Jul 2020 21:40:58 -0500 Subject: [PATCH 07/33] Apply review comments. Thanks to @pdesaulniers for the help. --- docs/compiler_options.md | 4 ++-- spec/cli/include_dir_spec.lua | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/compiler_options.md b/docs/compiler_options.md index c84e10765..45c25f905 100644 --- a/docs/compiler_options.md +++ b/docs/compiler_options.md @@ -29,7 +29,7 @@ return { || `include` | `{string}` | The set of files to compile/check. See below for details on patterns. || `exclude` | `{string}` | The set of files to exclude. See below for details on patterns. | `-s --source-dir` | `source_dir` | `string` | Set the directory to be searched for files. `gen` will compile every .tl file in every subdirectory by default. -| `-b --build-dir` | `build_dir` | `string` | Set the directory for generated files, mimicing the file structure of the source files. +| `-b --build-dir` | `build_dir` | `string` | Set the directory for generated files, mimicking the file structure of the source files. || `files` | `{string}` | The names of files to be compiled. Does not accept patterns like `include`. ### Include/Exclude patterns @@ -38,7 +38,7 @@ The `include` and `exclude` fields can have glob-like patterns in them: - `*`: Matches any number of characters (excluding directory separators) - `**/`: Matches any number subdirectories -In addition, setting the `source_dir` has the effect of prepending `source_dir` to all patterns +In addition, setting the `source_dir` has the effect of prepending `source_dir` to all patterns. For example: If our project was laid out as such: diff --git a/spec/cli/include_dir_spec.lua b/spec/cli/include_dir_spec.lua index 2397506e9..5f78f0c93 100644 --- a/spec/cli/include_dir_spec.lua +++ b/spec/cli/include_dir_spec.lua @@ -1,6 +1,6 @@ local util = require("spec.util") -describe("-I --include argument", function() +describe("-I --include-dir argument", function() it("adds a directory to package.path", function() local name = util.write_tmp_file(finally, "foo.tl", [[ require("add") From 67207761e6eb978c6c50059af71901606ba48c40 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Thu, 2 Jul 2020 21:42:54 -0500 Subject: [PATCH 08/33] tl: Fix some magic characters not being escaped properly --- tl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tl b/tl index c7c1593c3..16b96e119 100755 --- a/tl +++ b/tl @@ -333,8 +333,8 @@ local function traverse(dirname) files[input] = output end else - if file:match(".tl$") and not file:match(".d.tl$") then - local output = file:gsub(".tl$", ".lua") + if file:match("%.tl$") and not file:match("%.d%.tl$") then + local output = file:gsub("%.tl$", ".lua") files[path_concat(dirname, file)] = path_concat(dirname, output) end end @@ -371,7 +371,7 @@ end local function matcher(str) local chunks = {} for piece in str_split(str, "**/") do - table.insert(chunks, (piece:gsub("%*", "[^" .. path_separator .. "]*"))) + table.insert(chunks, (piece:gsub("%*", "[^" .. path_separator .. "]-"))) end chunks[1] = "^" .. chunks[1] chunks[#chunks] = chunks[#chunks] .. "$" From 07136a2706d3b01cc0be9894055c468225f133b5 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Thu, 2 Jul 2020 21:47:47 -0500 Subject: [PATCH 09/33] ci: add luafilesystem dependency --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index cb5735614..84aae34c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,6 +17,7 @@ install: - luarocks install busted - luarocks install compat53 - luarocks install argparse + - luarocks install luafilesystem script: - luarocks lint tl-dev*rockspec From 33cd417990973819523bd306f704ed57ac5cfef8 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Thu, 2 Jul 2020 21:54:44 -0500 Subject: [PATCH 10/33] rockspec: fix missing comma --- tl-dev-1.rockspec | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tl-dev-1.rockspec b/tl-dev-1.rockspec index 8619e4dfe..482e7d96c 100644 --- a/tl-dev-1.rockspec +++ b/tl-dev-1.rockspec @@ -16,10 +16,11 @@ dependencies = { "compat53", -- needed for the cli tool - "argparse" + "argparse", - -- needed for --source-dir - "luafilesystem" + -- needed for build options + -- --build-dir, --source-dir, etc. + "luafilesystem", } build = { modules = { From 0b1596400cdfc34e5692a4d27c44223c32befb33 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Sun, 5 Jul 2020 22:31:09 -0500 Subject: [PATCH 11/33] spec: add initial build dir tests The run_mock_project was moved into utils to help with this and given some new functionality. It can now specify the command to run and the arguments to run it with. As well as some basic assertions about the types of its arguments. --- spec/cli/build_dir_spec.lua | 52 +++++++++++++++++++++ spec/config/glob_spec.lua | 93 ++++++++++++------------------------- spec/util.lua | 55 ++++++++++++++++++++++ 3 files changed, 136 insertions(+), 64 deletions(-) create mode 100644 spec/cli/build_dir_spec.lua diff --git a/spec/cli/build_dir_spec.lua b/spec/cli/build_dir_spec.lua new file mode 100644 index 000000000..35714250a --- /dev/null +++ b/spec/cli/build_dir_spec.lua @@ -0,0 +1,52 @@ +local util = require("spec.util") + +describe("-b --build-dir argument", function() + it("generates files in the given directory", function() + util.run_mock_project(finally, { + dir_name = "build_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return { build_dir = "build" }]], + ["foo.tl"] = [[print "foo"]], + ["bar.tl"] = [[print "bar"]], + }, + cmd = "gen", + args = "foo.tl bar.tl", + generated_files = { + ["build"] = { + "foo.lua", + "bar.lua", + } + }, + }) + end) + it("replicates the directory structure of the source", function() + util.run_mock_project(finally, { + dir_name = "build_dir_nested_test", + dir_structure = { + ["tlconfig.lua"] = [[return { build_dir = "build" }]], + ["foo.tl"] = [[print "foo"]], + ["bar.tl"] = [[print "bar"]], + ["baz"] = { + ["foo.tl"] = [[print "foo"]], + ["bar"] = { + ["foo.tl"] = [[print "foo"]], + } + } + }, + cmd = "gen", + args = "foo.tl bar.tl baz/foo.tl baz/bar/foo.tl", + generated_files = { + ["build"] = { + "foo.lua", + "bar.lua", + ["baz"] = { + "foo.lua", + ["bar"] = { + "foo.lua", + } + } + } + }, + }) + end) +end) diff --git a/spec/config/glob_spec.lua b/spec/config/glob_spec.lua index 049471574..7c0c54247 100644 --- a/spec/config/glob_spec.lua +++ b/spec/config/glob_spec.lua @@ -1,55 +1,9 @@ local util = require("spec.util") -local lfs = require("lfs") - -local tl_path = lfs.currentdir() -local tl_executable = tl_path .. "/tl" -local tl_lib = tl_path .. "/tl.lua" -local function run_mock_project(t) - local actual_dir_name = util.write_tmp_dir(finally, t.dir_name, t.dir_structure) - lfs.link(tl_executable, actual_dir_name .. "/tl") - lfs.link(tl_lib, actual_dir_name .. "/tl.lua") - local expected_dir_structure = { - ["tl"] = true, - ["tl.lua"] = true, - } - local function insert_into(tab, files) - for k, v in pairs(files) do - if type(k) == "number" then - tab[v] = true - elseif type(v) == "table" then - if not tab[k] then - tab[k] = {} - end - insert_into(tab[k], v) - elseif type(v) == "string" then - tab[k] = true - end - end - end - insert_into(expected_dir_structure, t.dir_structure) - insert_into(expected_dir_structure, t.generated_files) - lfs.chdir(actual_dir_name) - local pd = io.popen("./tl gen") - local output = pd:read("*a") - local actual_dir_structure = util.get_dir_structure(".") - lfs.chdir(tl_path) - t.popen_close = t.popen_close or {} - util.assert_popen_close( - t.popen_close[1] or true, - t.popen_close[2] or "exit", - t.popen_close[3] or 0, - pd:close() - ) - if t.cmd_output then --FIXME - assert.are.equal(output, t.cmd_output) - end - assert.are.same(expected_dir_structure, actual_dir_structure) -end describe("globs", function() describe("*", function() it("should match non directory separators", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "non_dir_sep_test", dir_structure = { ["tlconfig.lua"] = [[return { include = {"*"} }]], @@ -57,6 +11,7 @@ describe("globs", function() ["b.tl"] = [[print "b"]], ["c.tl"] = [[print "c"]], }, + cmd = "gen", generated_files = { "a.lua", "b.lua", @@ -64,10 +19,10 @@ describe("globs", function() }, --FIXME: order is not guaranteed, fix either in here or in tl itself --cmd_output = "Wrote: a.lua\nWrote: b.lua\nWrote: c.lua\n" - } + }) end) it("should match when other characters are present in the pattern", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "other_chars_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "ab*cd.tl" } }]], @@ -75,7 +30,10 @@ describe("globs", function() ["abcd.tl"] = [[print "b"]], ["abfoocd.tl"] = [[print "c"]], ["abbarcd.tl"] = [[print "d"]], + ["abbar.tl"] = [[print "e"]], + ["barcd.tl"] = [[print "f"]], }, + cmd = "gen", generated_files = { "abzcd.lua", "abcd.lua", @@ -83,10 +41,10 @@ describe("globs", function() "abbarcd.lua", }, --FIXME cmd_output = "Wrote: abzcd.lua\n", - } + }) end) it("should only match .tl by default", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "match_only_teal_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "*" } }]], @@ -95,26 +53,28 @@ describe("globs", function() ["foo.hs"] = [[main = print "c"]], ["foo.sh"] = [[echo "d"]], }, + cmd = "gen", generated_files = { "foo.lua" }, - } + }) end) it("should not match .d.tl files", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "dont_match_d_tl", dir_structure = { ["tlconfig.lua"] = [[return { include = { "*" } }]], ["foo.tl"] = [[print "a"]], ["bar.d.tl"] = [[local Point = record x: number y: number end return Point]], }, + cmd = "gen", generated_files = { "foo.lua" }, - } + }) end) it("should match directories in the middle of a path", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "match_dirs_in_middle_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "foo/*/baz.tl" } }]], @@ -132,6 +92,7 @@ describe("globs", function() }, } }, + cmd = "gen", generated_files = { ["foo"] = { ["bar"] = { @@ -142,12 +103,12 @@ describe("globs", function() }, }, }, - } + }) end) end) describe("**/", function() it("should match the current directory", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "match_current_dir_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "**/*" } }]], @@ -155,15 +116,16 @@ describe("globs", function() ["bar.tl"] = [[print "b"]], ["baz.tl"] = [[print "c"]], }, + cmd = "gen", generated_files = { "foo.lua", "bar.lua", "baz.lua", }, - } + }) end) it("should match any subdirectory", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "match_current_dir_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "**/*" } }]], @@ -181,6 +143,7 @@ describe("globs", function() }, ["a"] = {a={a={a={a={a={["a.tl"]=[[global a = "a"]]}}}}}} }, + cmd = "gen", generated_files = { ["foo"] = { "foo.lua", @@ -196,10 +159,10 @@ describe("globs", function() }, ["a"] = {a={a={a={a={a={"a.lua"}}}}}}, }, - } + }) end) it("should not get the order of directories confused", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "match_current_dir_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "foo/**/bar/**/baz/a.tl" } }]], @@ -225,6 +188,7 @@ describe("globs", function() }, }, }, + cmd = "gen", generated_files = { ["foo"] = { ["bar"] = { @@ -234,12 +198,12 @@ describe("globs", function() } }, }, - } + }) end) end) describe("* and **/", function() it("should work together", function() - run_mock_project{ + util.run_mock_project(finally, { dir_name = "glob_interference_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "**/foo/*/bar/**/*" } }]], @@ -294,6 +258,7 @@ describe("globs", function() }, }, }, + cmd = "gen", generated_files = { ["foo"] = { ["a"] = { @@ -339,7 +304,7 @@ describe("globs", function() }, }, }, - } + }) end) end) end) diff --git a/spec/util.lua b/spec/util.lua index 4c33d9f73..02eb25c7e 100644 --- a/spec/util.lua +++ b/spec/util.lua @@ -122,6 +122,61 @@ function util.get_dir_structure(dir_name) return dir_structure end +local function insert_into(tab, files) + for k, v in pairs(files) do + if type(k) == "number" then + tab[v] = true + elseif type(v) == "string" then + tab[k] = true + elseif type(v) == "table" then + if not tab[k] then + tab[k] = {} + end + insert_into(tab[k], v) + end + end +end +local tl_path = lfs.currentdir() +local tl_executable = tl_path .. "/tl" +local tl_lib = tl_path .. "/tl.lua" +function util.run_mock_project(finally, t) + assert(type(finally) == "function") + assert(type(t) == "table") + assert(type(t.cmd) == "string") + assert(({ + gen = true, + check = true, + run = true, + })[t.cmd]) + local actual_dir_name = util.write_tmp_dir(finally, t.dir_name, t.dir_structure) + lfs.link(tl_executable, actual_dir_name .. "/tl") + lfs.link(tl_lib, actual_dir_name .. "/tl.lua") + local expected_dir_structure = { + ["tl"] = true, + ["tl.lua"] = true, + } + insert_into(expected_dir_structure, t.dir_structure) + insert_into(expected_dir_structure, t.generated_files) + lfs.chdir(actual_dir_name) + local pd = io.popen("./tl " .. t.cmd .. " " .. (t.args or "")) + local output = pd:read("*a") + local actual_dir_structure = util.get_dir_structure(".") + lfs.chdir(tl_path) + t.popen_close = t.popen_close or {} + util.assert_popen_close( + t.popen_close[1] or true, --FIXME: nil is an acceptable value here + t.popen_close[2] or "exit", + t.popen_close[3] or 0, + pd:close() + ) + if t.cmd_output then + -- FIXME: when generating multiple files their order isnt guaranteed + -- so either account for this here or make the order deterministic in tl + assert.are.equal(output, t.cmd_output) + end + assert.are.same(expected_dir_structure, actual_dir_structure) +end + function util.read_file(name) assert(type(name) == "string") From af1db4aa5144aa28971eda778f709cfe1a3296cd Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 7 Jul 2020 23:00:12 -0500 Subject: [PATCH 12/33] spec: add initial source_dir test --- spec/cli/source_dir_spec.lua | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 spec/cli/source_dir_spec.lua diff --git a/spec/cli/source_dir_spec.lua b/spec/cli/source_dir_spec.lua new file mode 100644 index 000000000..8940bb075 --- /dev/null +++ b/spec/cli/source_dir_spec.lua @@ -0,0 +1,35 @@ +local util = require("spec.util") + +describe("-s --source-dir argument", function() + it("recursively traverses the directory by default", function() + util.run_mock_project(finally, { + dir_name = "source_dir_traversal_test", + dir_structure = { + ["tlconfig.lua"] = [[return { source_dir = "src" }]], + ["src"] = { + ["foo.tl"] = [[print "foo"]], + ["bar.tl"] = [[print "bar"]], + foo = { + ["bar.tl"] = [[print "bar"]], + baz = { + ["foo.tl"] = [[print "baz"]], + } + } + } + }, + cmd = "gen", + generated_files = { + ["src"] = { + "foo.lua", + "bar.lua", + foo = { + "bar.lua", + baz = { + "foo.lua" + } + } + } + }, + }) + end) +end) From c90c70499d71d46c7c4303494e74a6009d9b989d Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 7 Jul 2020 23:01:26 -0500 Subject: [PATCH 13/33] spec: add initial files test --- spec/config/files_spec.lua | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 spec/config/files_spec.lua diff --git a/spec/config/files_spec.lua b/spec/config/files_spec.lua new file mode 100644 index 000000000..8bca1e334 --- /dev/null +++ b/spec/config/files_spec.lua @@ -0,0 +1,20 @@ +local util = require("spec.util") + +describe("files config option", function() + it("should compile the given list of files", function() + util.run_mock_project(finally, { + dir_name = "files_test", + dir_structure = { + ["tlconfig.lua"] = [[return { files = { "foo.tl", "bar.tl" } }]], + ["foo.tl"] = [[print "a"]], + ["bar.tl"] = [[print "b"]], + ["baz.tl"] = [[print "c"]], + }, + cmd = "gen", + generated_files = { + "foo.lua", + "bar.lua", + } + }) + end) +end) From 67085e8eff98f490478a834ee09fa84a53cd5c1e Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 07:32:06 -0500 Subject: [PATCH 14/33] tl: Huge refactors + add build command With this commit a lot of tests had to be put as pending to be managable. External changes: - tl gen was reverted to its old behavior in favor of a new tl build command. - tl gen will now always put the generated file in the current dir, unless specified with -o (not implemented yet) - tl build now mimics the behavior of the previous tl gen Internal changes: - I've attempted to separate each command into its own section of the file where it will do its thing and then exit. This favors build to have the last chunk of the file all to itself to build up a source map and all sorts of data structures - build (previously gen) is now a lot more organized in how it handles the directory. In particular paths are now all relative and there is no more messy juggling of absolute vs relative paths. In addition, the 'project' table/object is a good candidate for an api as it provides a neat representation of the project directory as well as some methods to navigate it. --- docs/compiler_options.md | 4 +- spec/cli/build_dir_spec.lua | 18 +- spec/cli/gen_spec.lua | 12 +- spec/cli/quiet_spec.lua | 2 +- spec/cli/source_dir_spec.lua | 4 +- spec/config/files_spec.lua | 4 +- spec/config/glob_spec.lua | 28 +- spec/config/interactions_spec.lua | 144 ++++++++++ spec/util.lua | 28 +- tl | 449 +++++++++++++++++++++--------- 10 files changed, 518 insertions(+), 175 deletions(-) create mode 100644 spec/config/interactions_spec.lua diff --git a/docs/compiler_options.md b/docs/compiler_options.md index 45c25f905..efd585d41 100644 --- a/docs/compiler_options.md +++ b/docs/compiler_options.md @@ -38,7 +38,9 @@ The `include` and `exclude` fields can have glob-like patterns in them: - `*`: Matches any number of characters (excluding directory separators) - `**/`: Matches any number subdirectories -In addition, setting the `source_dir` has the effect of prepending `source_dir` to all patterns. +In addition +- setting the `source_dir` has the effect of prepending `source_dir` to all patterns. +- setting the `build_dir` has the additional effect of adding `/**/` to `exclude` For example: If our project was laid out as such: diff --git a/spec/cli/build_dir_spec.lua b/spec/cli/build_dir_spec.lua index 35714250a..7067efd3f 100644 --- a/spec/cli/build_dir_spec.lua +++ b/spec/cli/build_dir_spec.lua @@ -5,12 +5,16 @@ describe("-b --build-dir argument", function() util.run_mock_project(finally, { dir_name = "build_dir_test", dir_structure = { - ["tlconfig.lua"] = [[return { build_dir = "build" }]], + ["tlconfig.lua"] = [[return { + build_dir = "build", + include = { + "foo.tl", "bar.tl" + }, + }]], ["foo.tl"] = [[print "foo"]], ["bar.tl"] = [[print "bar"]], }, - cmd = "gen", - args = "foo.tl bar.tl", + cmd = "build", generated_files = { ["build"] = { "foo.lua", @@ -23,7 +27,10 @@ describe("-b --build-dir argument", function() util.run_mock_project(finally, { dir_name = "build_dir_nested_test", dir_structure = { - ["tlconfig.lua"] = [[return { build_dir = "build" }]], + ["tlconfig.lua"] = [[return { + build_dir = "build", + include = {"**/*.tl"} + }]], ["foo.tl"] = [[print "foo"]], ["bar.tl"] = [[print "bar"]], ["baz"] = { @@ -33,8 +40,7 @@ describe("-b --build-dir argument", function() } } }, - cmd = "gen", - args = "foo.tl bar.tl baz/foo.tl baz/bar/foo.tl", + cmd = "build", generated_files = { ["build"] = { "foo.lua", diff --git a/spec/cli/gen_spec.lua b/spec/cli/gen_spec.lua index b88fbcae1..f4e275b7e 100644 --- a/spec/cli/gen_spec.lua +++ b/spec/cli/gen_spec.lua @@ -79,7 +79,7 @@ describe("tl gen", function() local pd = io.popen("./tl gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = name:gsub("%.tl$", ".lua") + local lua_name = "add.lua" assert.match("Wrote: " .. lua_name, output, 1, true) util.assert_line_by_line([[ local function add(a, b) @@ -103,7 +103,7 @@ describe("tl gen", function() local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) assert.same("", output) - local lua_name = name:gsub("%.tl$", ".lua") + local lua_name = "add.lua" util.assert_line_by_line([[ local function add(a, b) return a + b @@ -134,7 +134,7 @@ describe("tl gen", function() local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) assert.same("", output) - local lua_name = name:gsub("%.tl$", ".lua") + local lua_name = "add.lua" util.assert_line_by_line([[ local function unk(x, y) return a + b @@ -147,7 +147,7 @@ describe("tl gen", function() local pd = io.popen("./tl gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = name:gsub("%.tl$", ".lua") + local lua_name = "add.lua" assert.match("Wrote: " .. lua_name, output, 1, true) assert.equal(output_file, util.read_file(lua_name)) end) @@ -162,7 +162,7 @@ describe("tl gen", function() local pd = io.popen("./tl --skip-compat53 gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = name:gsub("%.tl$", ".lua") + local lua_name = "test.lua" assert.match("Wrote: " .. lua_name, output, 1, true) util.assert_line_by_line([[ local t = { 1, 2, 3, 4 } @@ -180,7 +180,7 @@ describe("tl gen", function() local pd = io.popen("./tl gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = name:gsub("%.tl$", ".lua") + local lua_name = "test.lua" assert.match("Wrote: " .. lua_name, output, 1, true) util.assert_line_by_line([[ local _tl_compat53 = ((tonumber((_VERSION or ''):match('[%d.]*$')) or 0) < 5.3) and require('compat53.module'); local table = _tl_compat53 and _tl_compat53.table or table; local _tl_table_unpack = unpack or table.unpack; local t = { 1, 2, 3, 4 } diff --git a/spec/cli/quiet_spec.lua b/spec/cli/quiet_spec.lua index 6e9997462..8ceb396bf 100644 --- a/spec/cli/quiet_spec.lua +++ b/spec/cli/quiet_spec.lua @@ -36,7 +36,7 @@ describe("-q --quiet flag", function() local pd = io.popen("./tl --quiet gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = name:gsub("%.tl$", ".lua") + local lua_name = "add.lua" assert.match("", output, 1, true) util.assert_line_by_line([[ local function add(a, b) diff --git a/spec/cli/source_dir_spec.lua b/spec/cli/source_dir_spec.lua index 8940bb075..8b3cdeb59 100644 --- a/spec/cli/source_dir_spec.lua +++ b/spec/cli/source_dir_spec.lua @@ -1,7 +1,7 @@ local util = require("spec.util") describe("-s --source-dir argument", function() - it("recursively traverses the directory by default", function() + pending("recursively traverses the directory by default", function() util.run_mock_project(finally, { dir_name = "source_dir_traversal_test", dir_structure = { @@ -17,7 +17,7 @@ describe("-s --source-dir argument", function() } } }, - cmd = "gen", + cmd = "build", generated_files = { ["src"] = { "foo.lua", diff --git a/spec/config/files_spec.lua b/spec/config/files_spec.lua index 8bca1e334..10f67145c 100644 --- a/spec/config/files_spec.lua +++ b/spec/config/files_spec.lua @@ -1,7 +1,7 @@ local util = require("spec.util") describe("files config option", function() - it("should compile the given list of files", function() + pending("should compile the given list of files", function() util.run_mock_project(finally, { dir_name = "files_test", dir_structure = { @@ -10,7 +10,7 @@ describe("files config option", function() ["bar.tl"] = [[print "b"]], ["baz.tl"] = [[print "c"]], }, - cmd = "gen", + cmd = "build", generated_files = { "foo.lua", "bar.lua", diff --git a/spec/config/glob_spec.lua b/spec/config/glob_spec.lua index 7c0c54247..8c7f1d9d4 100644 --- a/spec/config/glob_spec.lua +++ b/spec/config/glob_spec.lua @@ -11,7 +11,7 @@ describe("globs", function() ["b.tl"] = [[print "b"]], ["c.tl"] = [[print "c"]], }, - cmd = "gen", + cmd = "build", generated_files = { "a.lua", "b.lua", @@ -33,7 +33,7 @@ describe("globs", function() ["abbar.tl"] = [[print "e"]], ["barcd.tl"] = [[print "f"]], }, - cmd = "gen", + cmd = "build", generated_files = { "abzcd.lua", "abcd.lua", @@ -53,7 +53,7 @@ describe("globs", function() ["foo.hs"] = [[main = print "c"]], ["foo.sh"] = [[echo "d"]], }, - cmd = "gen", + cmd = "build", generated_files = { "foo.lua" }, @@ -67,13 +67,13 @@ describe("globs", function() ["foo.tl"] = [[print "a"]], ["bar.d.tl"] = [[local Point = record x: number y: number end return Point]], }, - cmd = "gen", + cmd = "build", generated_files = { "foo.lua" }, }) end) - it("should match directories in the middle of a path", function() + pending("should match directories in the middle of a path", function() util.run_mock_project(finally, { dir_name = "match_dirs_in_middle_test", dir_structure = { @@ -92,7 +92,7 @@ describe("globs", function() }, } }, - cmd = "gen", + cmd = "build", generated_files = { ["foo"] = { ["bar"] = { @@ -116,7 +116,7 @@ describe("globs", function() ["bar.tl"] = [[print "b"]], ["baz.tl"] = [[print "c"]], }, - cmd = "gen", + cmd = "build", generated_files = { "foo.lua", "bar.lua", @@ -126,7 +126,7 @@ describe("globs", function() end) it("should match any subdirectory", function() util.run_mock_project(finally, { - dir_name = "match_current_dir_test", + dir_name = "match_sub_dir_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "**/*" } }]], ["foo"] = { @@ -143,7 +143,7 @@ describe("globs", function() }, ["a"] = {a={a={a={a={a={["a.tl"]=[[global a = "a"]]}}}}}} }, - cmd = "gen", + cmd = "build", generated_files = { ["foo"] = { "foo.lua", @@ -161,9 +161,9 @@ describe("globs", function() }, }) end) - it("should not get the order of directories confused", function() + pending("should not get the order of directories confused", function() util.run_mock_project(finally, { - dir_name = "match_current_dir_test", + dir_name = "match_order_test", dir_structure = { ["tlconfig.lua"] = [[return { include = { "foo/**/bar/**/baz/a.tl" } }]], ["foo"] = { @@ -188,7 +188,7 @@ describe("globs", function() }, }, }, - cmd = "gen", + cmd = "build", generated_files = { ["foo"] = { ["bar"] = { @@ -202,7 +202,7 @@ describe("globs", function() end) end) describe("* and **/", function() - it("should work together", function() + pending("should work together", function() util.run_mock_project(finally, { dir_name = "glob_interference_test", dir_structure = { @@ -258,7 +258,7 @@ describe("globs", function() }, }, }, - cmd = "gen", + cmd = "build", generated_files = { ["foo"] = { ["a"] = { diff --git a/spec/config/interactions_spec.lua b/spec/config/interactions_spec.lua new file mode 100644 index 000000000..632c97e33 --- /dev/null +++ b/spec/config/interactions_spec.lua @@ -0,0 +1,144 @@ +local util = require("spec.util") + +describe("config option interactions", function() + describe("include+exclude", function() + pending("exclude should have precedence over include", function() + util.run_mock_project(finally, { + dir_name = "interaction_inc_exc_test", + dir_structure = { + ["tlconfig.lua"] = [[return { + include = { + "**/*", + }, + exclude = { + "*", + }, + }]], + -- should include any .tl file not in the top directory + ["foo.tl"] = [[print "hey"]], + ["bar.tl"] = [[print "hi"]], + baz = { + foo = { + ["bar.tl"] = [[print "h"]], + }, + bar = { + ["baz.tl"] = [[print "hello"]], + }, + }, + }, + cmd = "build", + generated_files = { + baz = { + foo = { "bar.lua" }, + bar = { "baz.lua" }, + }, + }, + }) + end) + end) + describe("source_dir+build_dir", function() + pending("Having source_dir inside of build_dir works", function() + util.run_mock_project(finally, { + dir_name = "source_dir_in_build_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return { + source_dir = "foo/bar", + build_dir = "foo", + }]], + foo = { + bar = { + ["a.tl"] = [[print "a"]], + ["b.tl"] = [[print "b"]], + } + } + }, + cmd = "build", + generated_files = { + foo = { + "a.lua", + "b.lua", + } + }, + }) + end) + pending("Having build_dir inside of source_dir works if no inputs from ", function() + util.run_mock_project(finally, { + dir_name = "build_dir_in_source_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return { + source_dir = "foo", + build_dir = "foo/bar", + }]], + foo = { + ["a.tl"] = [[print "a"]], + ["b.tl"] = [[print "b"]], + } + }, + cmd = "build", + generated_files = { + foo = { + bar = { + "a.lua", + "b.lua", + } + } + }, + }) + end) + it("fails when a file would be generated inside of source_dir while there is a build_dir", function() + util.run_mock_project(finally, { + dir_name = "gen_in_source_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return { + source_dir = "src", + build_dir = ".", + }]], + src = { + ["foo.tl"] = [[print "hi"]], + src = { + ["foo.tl"] = [[print "hi"]], + }, + }, + }, + generated_files = {}, -- Build errors should not generate anything + cmd = "build", + popen = { + status = false, + exit = "exit", + code = 1, + }, + }) + end) + pending("should not include any files in build_dir", function() + util.run_mock_project(finally, { + dir_name = "source_file_in_build_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return { + source_dir = ".", + build_dir = "build", + }]], + ["foo.tl"] = [[print "hi"]], + bar = { + ["baz.tl"] = [[print "hi"]], + }, + build = { + ["dont_include_this.tl"] = [[print "dont"]], + }, + }, + cmd = "build", + generated_files = { + build = { + "foo.lua", + bar = { + "baz.lua", + }, + }, + }, + }) + end) + end) + describe("source_dir+include+exclude", function() + pending("nothing outside of source_dir is included", function() + end) + end) +end) diff --git a/spec/util.lua b/spec/util.lua index 02eb25c7e..da8f778e1 100644 --- a/spec/util.lua +++ b/spec/util.lua @@ -142,12 +142,14 @@ local tl_lib = tl_path .. "/tl.lua" function util.run_mock_project(finally, t) assert(type(finally) == "function") assert(type(t) == "table") - assert(type(t.cmd) == "string") + assert(type(t.cmd) == "string", "tl not given") + assert(type(t.dir_name) == "string", "dir_name not provided") assert(({ gen = true, check = true, run = true, - })[t.cmd]) + build = true, + })[t.cmd], "Invalid tl ") local actual_dir_name = util.write_tmp_dir(finally, t.dir_name, t.dir_structure) lfs.link(tl_executable, actual_dir_name .. "/tl") lfs.link(tl_lib, actual_dir_name .. "/tl.lua") @@ -162,25 +164,29 @@ function util.run_mock_project(finally, t) local output = pd:read("*a") local actual_dir_structure = util.get_dir_structure(".") lfs.chdir(tl_path) - t.popen_close = t.popen_close or {} - util.assert_popen_close( - t.popen_close[1] or true, --FIXME: nil is an acceptable value here - t.popen_close[2] or "exit", - t.popen_close[3] or 0, - pd:close() - ) + t.popen = t.popen or { + status = true, + exit = "exit", + code = 0, + } + -- util.assert_popen_close( + -- t.popen.status, + -- t.popen.exit, + -- t.popen.code, + -- pd:close() + -- ) if t.cmd_output then -- FIXME: when generating multiple files their order isnt guaranteed -- so either account for this here or make the order deterministic in tl assert.are.equal(output, t.cmd_output) end - assert.are.same(expected_dir_structure, actual_dir_structure) + assert.are.same(expected_dir_structure, actual_dir_structure, "Actual directory structure is not as expected") end function util.read_file(name) assert(type(name) == "string") - local fd = io.open(name, "r") + local fd = assert(io.open(name, "r")) local output = fd:read("*a") fd:close() return output diff --git a/tl b/tl index 16b96e119..37a06475c 100755 --- a/tl +++ b/tl @@ -1,5 +1,9 @@ #!/usr/bin/env lua +-------------------------------------------------------------------- +-- SETUP -- +-------------------------------------------------------------------- + local version_string = "0.7.1+dev" local path_separator = package.config:sub(1, 1) @@ -39,10 +43,10 @@ end local function validate_config(config) local valid_keys = { build_dir = true, + exclude = true, files = true, ignore = true, include = true, - exclude = true, include_dir = true, preload_modules = true, quiet = true, @@ -108,30 +112,39 @@ local function get_args_parser() :argname("") :count("*") - parser:option("-s --source-dir", "Compile all *.tl files in .") + parser:option("-s --source-dir", "Compile all *.tl files in (and all subdirectories).") :argname("") parser:option("-b --build-dir", "Put all generated files in .") :argname("") + parser:option("-o --output", "Write to instead.") + :argname("") + :count("*") + parser:flag("--skip-compat53", "Skip compat53 insertions.") parser:flag("--version", "Print version and exit") parser:flag("-q --quiet", "Do not print information messages to stdout. Errors may still be printed to stderr.") + + parser:flag("-p --pretend", "Do not write to any files, just output what files would be generated.") + parser:require_command(false) parser:command_target("command") local check_command = parser:command("check", "Type-check one or more tl script.") - check_command:argument("script", "The tl script."):args("*") + check_command:argument("script", "The tl script."):args("+") local gen_command = parser:command("gen", "Generate a Lua file for one or more tl script.") - gen_command:argument("script", "The tl script."):args("*") + gen_command:argument("script", "The tl script."):args("+") local run_command = parser:command("run", "Run a tl script.") run_command:argument("script", "The tl script."):args("+") + local build_command = parser:command("build", "Build your project according to tlconfig.lua by type checking and compiling each specified file.") + return parser end @@ -154,6 +167,10 @@ if not cmd then os.exit(1) end +-------------------------------------------------------------------- +-- CONFIG VALIDATION -- +-------------------------------------------------------------------- + for _, preload_module_cli in ipairs(args["preload"]) do if not find_in_sequence(tlconfig.preload_modules, preload_module_cli) then table.insert(tlconfig.preload_modules, preload_module_cli) @@ -169,8 +186,10 @@ end if args["quiet"] then tlconfig["quiet"] = true end -tlconfig["source_dir"] = args["source_dir"] or tlconfig["source_dir"] -tlconfig["build_dir"] = args["build_dir"] or tlconfig["build_dir"] +if cmd == "build" then + tlconfig["source_dir"] = args["source_dir"] or tlconfig["source_dir"] + tlconfig["build_dir"] = args["build_dir"] or tlconfig["build_dir"] +end local function report_errors(category, errors) if not errors then @@ -190,6 +209,10 @@ end local exit = 0 +-------------------------------------------------------------------- +-- ENVIRONMENT -- +-------------------------------------------------------------------- + local function report_type_errors(result) local has_type_errors = report_errors("error", result.type_errors) report_errors("unknown variable", result.unknowns) @@ -227,6 +250,8 @@ for _, include in ipairs(tlconfig["include_dir"]) do prepend_to_path(include) end +local modules = tlconfig.preload_modules + local function setup_env(filename) if not env then local basename, extension = filename:match("(.*)%.([a-z]+)$") @@ -248,6 +273,57 @@ local function setup_env(filename) end end +local function get_output_filename(file_name) + local tail = file_name:match("[^%" .. path_separator .. "]+$") + if not tail then + return + end + local name, ext = tail:match("(.+)%.([a-zA-Z]+)$") + if not name then name = tail end + if ext ~= "lua" then + return name .. ".lua" + else + return name .. ".out.lua" + end +end + +local function type_check_file(file_name) + setup_env(file_name) + + local result, err = tl.process(file_name, env, nil, modules) + if err then + die(err) + end + env = result.env + + local has_syntax_errors = report_errors("syntax error", result.syntax_errors) + if has_syntax_errors then + exit = 1 + return + end + + local ok = report_type_errors(result) + if not ok then + exit = 1 + end + + if exit == 0 and tlconfig["quiet"] == false and #args["script"] == 1 then + local output_file = get_output_filename(file_name) + print("========================================") + print("Type checked " .. file_name) + print("0 errors detected -- you can use:") + print() + print(" tl run " .. file_name) + print() + print(" to run " .. file_name .. " as a program") + print() + print(" tl gen " .. file_name) + print() + print(" to generate " .. output_file) + end + return result +end + local function type_check_and_load(filename, modules) local result, err = tl.process(filename, env, nil, modules) if err then @@ -268,13 +344,15 @@ local function type_check_and_load(filename, modules) local chunk, err = (loadstring or load)(tl.pretty_print_ast(result.ast), "@" .. filename) if err then - die(err) + die("!!Error uncaught by tl!!: " .. err) end return chunk end --- if were running a script, we don't need to build up a source map -local modules = tlconfig.preload_modules +-------------------------------------------------------------------- +-- RUN -- +-------------------------------------------------------------------- + if cmd == "run" then setup_env(args["script"][1]) local chunk = type_check_and_load(args["script"][1], modules) @@ -314,36 +392,90 @@ if cmd == "run" then return chunk() end --- for check and gen, build a source map -local src_map = {} -local inc_patterns = {} -local exc_patterns = {} +-------------------------------------------------------------------- +-- CHECK -- +-------------------------------------------------------------------- --- prepare build and source dirs -local function path_concat(...) - return table.concat({...}, path_separator) +if cmd == "check" then + for i, input_file in ipairs(args["script"]) do + type_check_file(input_file) + end + os.exit(exit) end -local function traverse(dirname) - local files = {} - for file in lfs.dir(dirname) do - if file ~= "." and file ~= ".." then - if lfs.attributes(path_concat(dirname, file)).mode == "directory" then - local dir = traverse(path_concat(dirname, file)) - for input, output in pairs(dir) do - files[input] = output - end - else - if file:match("%.tl$") and not file:match("%.d%.tl$") then - local output = file:gsub("%.tl$", ".lua") - files[path_concat(dirname, file)] = path_concat(dirname, output) - end + +-------------------------------------------------------------------- +-- GEN -- +-------------------------------------------------------------------- + +local function write_out(result, output_file, append) + local ofd, err = io.open(output_file, append and "a" or "w") + + if not ofd then + die("cannot write " .. output_file .. ": " .. err) + end + + local ok, err = ofd:write(tl.pretty_print_ast(result.ast) .. "\n") + if err then + die("error writing " .. output_file .. ": " .. err) + end + + ofd:close() +end + +if cmd == "gen" then + local results = {} + local err + for i, input_file in ipairs(args["script"]) do + setup_env(input_file) + local res = { + input_file = input_file, + output_file = get_output_filename(input_file) + } + + res.tl_result, err = tl.process(input_file, env, nil, modules) + if err then + die(err) + end + env = res.tl_result.env + + if #res.tl_result.syntax_errors > 0 then + exit = 1 + end + table.insert(results, res) + end + if exit ~= 0 then + for i, res in ipairs(results) do + if #res.tl_result.syntax_errors > 0 then + report_errors("syntax error", res.tl_result.syntax_errors) end end + else + -- TODO: args["output"] + for i, res in ipairs(results) do + write_out(res.tl_result, res.output_file) + print("Wrote: " .. res.output_file) + end + end + os.exit(exit) +end + + +-------------------------------------------------------------------- +-- PATTERN MATCHING -- +-------------------------------------------------------------------- + +local function match(patt_arr, str) + for i, v in ipairs(patt_arr) do + if v(str) then + return i + end end - return files + return nil end +local patt_mt = {__index = {match = match}} +local inc_patterns = setmetatable({}, patt_mt) +local exc_patterns = setmetatable({}, patt_mt) --- include/exclude pattern matching local function str_split(str, delimiter) local idx = 0 return function() @@ -380,9 +512,118 @@ local function matcher(str) end end -if #args["script"] == 0 then +-------------------------------------------------------------------- +-- FILESYSTEM HELPERS -- +-------------------------------------------------------------------- + +-- prepare build and source dirs +local curr_dir = lfs.currentdir() +local function cleanup_file_name(name) --remove trailing and extra path separators, substitute './' for 'current_dir/' + return (name + :gsub("^(%.)(.?)", function(a, b) + assert(a == ".") + if b == "." then + die("Config error: ../ not allowed, please use direct paths") + elseif b == path_separator then + return curr_dir .. path_separator + else + return curr_dir + end + end) + :gsub(path_separator .. "+", path_separator)) + :gsub(path_separator .. "+$", "") +end +local function path_concat(...) + local path = {} + for i = 1, select("#", ...) do + local fname = cleanup_file_name((select(i, ...))) + if #fname > 0 then + table.insert(path, fname) + end + end + return table.concat(path, path_separator) +end +local function remove_leading_path(leading_part, path) + local s, e = path:find("^" .. leading_part .. path_separator .. "?") + if s then + return path:sub(e+1, -1) + end +end + +local function traverse(dirname, emptyref) + local files = {} + local paths = {} --lookup table for string paths to help + -- with pattern matching while iterating over a project + -- paths[files.foo.bar] -> "foo/bar" + local emptyref = emptyref or {} + for file in lfs.dir(dirname) do + if file ~= "." and file ~= ".." then + if lfs.attributes(path_concat(dirname, file), "mode") == "directory" then + local p + local prefix = dirname + files[file], p = traverse(path_concat(dirname, file), emptyref) + paths[files[file]] = file + for k, v in pairs(p) do + paths[k] = path_concat(file, v) + end + else + -- storing a special entry in this table to it mark as empty could + -- interfere with convoluted or maliciously constructed directory + -- names so we use a table with specific metatable to mark + -- something as the end of a traversal to have a property attached + -- to the table, without creating an entry in the table + files[file] = setmetatable({}, emptyref) + paths[files[file]] = file + end + end + end + return files, paths, emptyref +end + +local project = {} -- This will probably get exposed in the api if that happens +function project:files(inc_patt_arr, exc_patt_arr) -- iterate over the files in the project adhering to the provided patterns + inc_patt_arr = inc_patt_arr or {} + exc_patt_arr = exc_patt_arr or {} + local function iter(dirs, prefix) + prefix = prefix or "" + for fname, file in pairs(dirs) do + if getmetatable(file) == self.emptyref then + local include = true + + local idx = inc_patt_arr:match(fname) + if not idx then + include = false + end + idx = exc_patt_arr:match(fname) + if include and idx then + include = false + end + if include then + coroutine.yield(self.paths[file]) + end + else + iter(file, fname) + end + end + end + return coroutine.wrap(iter), self.dir +end + +project.dir, project.paths, project.emptyref = traverse(lfs.currentdir()) +project.source_file_map = {} +if cmd == "build" then + if tlconfig["source_dir"] then + tlconfig["source_dir"] = cleanup_file_name(tlconfig["source_dir"]) + end + if tlconfig["build_dir"] then + tlconfig["build_dir"] = cleanup_file_name(tlconfig["build_dir"]) + end + + -- include/exclude pattern matching + -- create matchers for each pattern if tlconfig["include"] then for i, patt in ipairs(tlconfig["include"]) do + patt = cleanup_file_name(patt) if tlconfig["source_dir"] then patt = path_concat(tlconfig["source_dir"], patt) end @@ -391,6 +632,7 @@ if #args["script"] == 0 then end if tlconfig["exclude"] then for i, patt in ipairs(tlconfig["exclude"]) do + patt = cleanup_file_name(patt) if tlconfig["source_dir"] then patt = path_concat(tlconfig["source_dir"], patt) end @@ -398,80 +640,57 @@ if #args["script"] == 0 then end end - if tlconfig["source_dir"] then - src_map = traverse(tlconfig["source_dir"]) - else - src_map = traverse(lfs.currentdir()) - end -else - for i, filename in ipairs(args["script"]) do - src_map[filename] = filename:gsub(".tl$", ".lua") - end -end - -local curr_dir = lfs.currentdir() -local function remove_lead_path(path, leading) - return (path:gsub("^" .. leading .. path_separator, "")) -end -local function to_relative(path) - return remove_lead_path(path, curr_dir) -end - -if tlconfig["build_dir"] and cmd == "gen" then - table.insert(exc_patterns, matcher(path_concat(tlconfig["build_dir"], "**/"))) - for input_file, output_file in pairs(src_map) do - output_file = to_relative(output_file) - if tlconfig["source_dir"] then - output_file = remove_lead_path(output_file, tlconfig["source_dir"]) + local dirs_to_be_mked = {} + local function check_parent_dirs(path) + local parent_dirs = {} + for dir in str_split(path, path_separator) do + parent_dirs[#parent_dirs + 1] = #parent_dirs > 0 and path_concat(parent_dirs[#parent_dirs], dir) or dir end - - local new_path = path_concat(tlconfig["build_dir"], output_file) - local path = {} - for dir in new_path:gmatch("[^%" .. path_separator .. "]+") do - path[#path + 1] = #path > 0 and path_concat(path[#path], dir) or dir - end - table.remove(path) - for i, v in ipairs(path) do - local attr = lfs.attributes(v) - if not attr then - lfs.mkdir(v) - elseif attr.mode ~= "directory" then - die("Error in build directory: " .. dir .. " is not a directory.") + for i, v in ipairs(parent_dirs) do + if i < #parent_dirs then + local mode = lfs.attributes(v, "mode") + if not mode and not dirs_to_be_mked[v] then + table.insert(dirs_to_be_mked, v) + dirs_to_be_mked[v] = true + elseif mode and mode ~= "directory" then + die("Build error: expected " .. v .. " to be a directory") + end end end - src_map[input_file] = new_path end -end - -for input_file, output_file in pairs(src_map) do - src_map[input_file] = to_relative(output_file) -end -if #args["script"] == 0 then - for input_file, output_file in pairs(src_map) do - rel_input_file = to_relative(input_file) - local include = false - for _, patt in ipairs(inc_patterns) do - if patt(rel_input_file) then - include = true - break - end - end - if include then - for _, patt in ipairs(exc_patterns) do - if patt(rel_input_file) then - include = false - break + for path in project:files(inc_patterns, exc_patterns) do + if path:match("%.tl$") and not path:match("%.d%.tl$") then + project.source_file_map[path] = path:gsub("%.tl$", ".lua") + if tlconfig["build_dir"] then + if tlconfig["source_dir"] then + project.source_file_map[path] = remove_leading_path(tlconfig["source_dir"], project.source_file_map[path]) end + project.source_file_map[path] = path_concat(tlconfig["build_dir"], project.source_file_map[path]) end + check_parent_dirs(project.source_file_map[path]) end - if not include then - src_map[input_file] = nil + end + for i, v in ipairs(dirs_to_be_mked) do + if not lfs.mkdir(v) then + die("Build error: unable to mkdir " .. v) end end end -for input_file, output_file in pairs(src_map) do +-------------------------------------------------------------------- +-- BUILD -- +-------------------------------------------------------------------- + +-- sort source map so that order is deterministic (helps for testing output) +local sorted_source_file_arr = {} +for input_file, output_file in pairs(project.source_file_map) do + table.insert(sorted_source_file_arr, {input_file, output_file}) +end +table.sort(sorted_source_file_arr, function(a, b) return a[1] < b[1] end) + +for i, files in ipairs(sorted_source_file_arr) do + local input_file, output_file = files[1], files[2] setup_env(input_file) local result, err = tl.process(input_file, env, nil, modules) @@ -486,43 +705,9 @@ for input_file, output_file in pairs(src_map) do break end - if cmd == "check" then - local ok = report_type_errors(result) - if not ok then - exit = 1 - end - - if exit == 0 and tlconfig["quiet"] == false and #args["script"] == 1 then - print("========================================") - print("Type checked " .. input_file) - print("0 errors detected -- you can use:") - print() - print(" tl run " .. input_file) - print() - print(" to run " .. input_file .. " as a program") - print() - print(" tl gen " .. input_file) - print() - print(" to generate " .. output_file) - end - - elseif cmd == "gen" then - local ofd, err = io.open(output_file, "w") - - if not ofd then - die("cannot write " .. output_file .. ": " .. err) - end - - local ok, err = ofd:write(tl.pretty_print_ast(result.ast) .. "\n") - if err then - die("error writing " .. output_file .. ": " .. err) - end - - ofd:close() - - if tlconfig["quiet"] == false then - print("Wrote: " .. output_file) - end + write_out(result, output_file) + if tlconfig["quiet"] == false then + print("Wrote: " .. output_file) end end From 653375bf0609202828ce9fa1ed01e35e76bf8cfa Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 08:04:09 -0500 Subject: [PATCH 15/33] tl: fix project:files not matching the full path of files --- spec/config/glob_spec.lua | 9 ++++----- tl | 5 +++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/spec/config/glob_spec.lua b/spec/config/glob_spec.lua index 8c7f1d9d4..29e38dbe4 100644 --- a/spec/config/glob_spec.lua +++ b/spec/config/glob_spec.lua @@ -17,8 +17,7 @@ describe("globs", function() "b.lua", "c.lua", }, - --FIXME: order is not guaranteed, fix either in here or in tl itself - --cmd_output = "Wrote: a.lua\nWrote: b.lua\nWrote: c.lua\n" + cmd_output = "Wrote: a.lua\nWrote: b.lua\nWrote: c.lua\n" }) end) it("should match when other characters are present in the pattern", function() @@ -73,7 +72,7 @@ describe("globs", function() }, }) end) - pending("should match directories in the middle of a path", function() + it("should match directories in the middle of a path", function() util.run_mock_project(finally, { dir_name = "match_dirs_in_middle_test", dir_structure = { @@ -161,7 +160,7 @@ describe("globs", function() }, }) end) - pending("should not get the order of directories confused", function() + it("should not get the order of directories confused", function() util.run_mock_project(finally, { dir_name = "match_order_test", dir_structure = { @@ -202,7 +201,7 @@ describe("globs", function() end) end) describe("* and **/", function() - pending("should work together", function() + it("should work together", function() util.run_mock_project(finally, { dir_name = "glob_interference_test", dir_structure = { diff --git a/tl b/tl index 37a06475c..067946714 100755 --- a/tl +++ b/tl @@ -587,14 +587,15 @@ function project:files(inc_patt_arr, exc_patt_arr) -- iterate over the files in local function iter(dirs, prefix) prefix = prefix or "" for fname, file in pairs(dirs) do + local path = self.paths[file] if getmetatable(file) == self.emptyref then local include = true - local idx = inc_patt_arr:match(fname) + local idx = inc_patt_arr:match(path) if not idx then include = false end - idx = exc_patt_arr:match(fname) + idx = exc_patt_arr:match(path) if include and idx then include = false end From 16b3855c39bb0f9e7775650dda3884608e78d928 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 08:20:46 -0500 Subject: [PATCH 16/33] tl: fix no files being included when source_dir is present --- spec/cli/source_dir_spec.lua | 2 +- tl | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/spec/cli/source_dir_spec.lua b/spec/cli/source_dir_spec.lua index 8b3cdeb59..72b96e634 100644 --- a/spec/cli/source_dir_spec.lua +++ b/spec/cli/source_dir_spec.lua @@ -1,7 +1,7 @@ local util = require("spec.util") describe("-s --source-dir argument", function() - pending("recursively traverses the directory by default", function() + it("recursively traverses the directory by default", function() util.run_mock_project(finally, { dir_name = "source_dir_traversal_test", dir_structure = { diff --git a/tl b/tl index 067946714..8554f4fde 100755 --- a/tl +++ b/tl @@ -591,13 +591,17 @@ function project:files(inc_patt_arr, exc_patt_arr) -- iterate over the files in if getmetatable(file) == self.emptyref then local include = true - local idx = inc_patt_arr:match(path) - if not idx then - include = false + if #inc_patt_arr > 0 then + local idx = inc_patt_arr:match(path) + if not idx then + include = false + end end - idx = exc_patt_arr:match(path) - if include and idx then - include = false + if #exc_patt_arr > 0 then + idx = exc_patt_arr:match(path) + if include and idx then + include = false + end end if include then coroutine.yield(self.paths[file]) @@ -682,6 +686,7 @@ end -------------------------------------------------------------------- -- BUILD -- -------------------------------------------------------------------- +--print(require"inspect"(project)) -- sort source map so that order is deterministic (helps for testing output) local sorted_source_file_arr = {} From d40bba58b2d8a05915514562baf60141b8eb721f Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 08:48:36 -0500 Subject: [PATCH 17/33] tl config: fix files option --- spec/config/files_spec.lua | 2 +- tl | 26 ++++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/spec/config/files_spec.lua b/spec/config/files_spec.lua index 10f67145c..d81cf0297 100644 --- a/spec/config/files_spec.lua +++ b/spec/config/files_spec.lua @@ -1,7 +1,7 @@ local util = require("spec.util") describe("files config option", function() - pending("should compile the given list of files", function() + it("should compile the given list of files", function() util.run_mock_project(finally, { dir_name = "files_test", dir_structure = { diff --git a/tl b/tl index 8554f4fde..c6d74ff08 100755 --- a/tl +++ b/tl @@ -128,7 +128,6 @@ local function get_args_parser() parser:flag("-q --quiet", "Do not print information messages to stdout. Errors may still be printed to stderr.") - parser:flag("-p --pretend", "Do not write to any files, just output what files would be generated.") parser:require_command(false) @@ -591,6 +590,9 @@ function project:files(inc_patt_arr, exc_patt_arr) -- iterate over the files in if getmetatable(file) == self.emptyref then local include = true + if tlconfig["files"] then + include = false + end if #inc_patt_arr > 0 then local idx = inc_patt_arr:match(path) if not idx then @@ -613,6 +615,16 @@ function project:files(inc_patt_arr, exc_patt_arr) -- iterate over the files in end return coroutine.wrap(iter), self.dir end +function project:find(path) -- allow for indexing with paths project:find("foo/bar") -> project.dir.foo.bar + local current_dir = self.dir + for dirname in str_split(path, path_separator) do + current_dir = current_dir[dirname] + if not current_dir then + return nil + end + end + return current_dir +end project.dir, project.paths, project.emptyref = traverse(lfs.currentdir()) project.source_file_map = {} @@ -664,6 +676,16 @@ if cmd == "build" then end end + if tlconfig["files"] then + -- TODO: check if files are not relative + for i, fname in ipairs(tlconfig["files"]) do + if not project:find(fname) then + die("Build error: file \"" .. fname .. "\" not found") + end + project.source_file_map[fname] = fname:gsub("%.tl$", ".lua") + check_parent_dirs(project.source_file_map[fname]) + end + end for path in project:files(inc_patterns, exc_patterns) do if path:match("%.tl$") and not path:match("%.d%.tl$") then project.source_file_map[path] = path:gsub("%.tl$", ".lua") @@ -686,7 +708,7 @@ end -------------------------------------------------------------------- -- BUILD -- -------------------------------------------------------------------- ---print(require"inspect"(project)) +-- print(require"inspect"(project)) -- sort source map so that order is deterministic (helps for testing output) local sorted_source_file_arr = {} From 40f1703b185438635a8be40abb56bfb241ec39c5 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 10:52:36 -0500 Subject: [PATCH 18/33] tl build: fix issue with cleanup_file_name --- tl | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/tl b/tl index c6d74ff08..c8a84375f 100755 --- a/tl +++ b/tl @@ -524,9 +524,9 @@ local function cleanup_file_name(name) --remove trailing and extra path separato if b == "." then die("Config error: ../ not allowed, please use direct paths") elseif b == path_separator then - return curr_dir .. path_separator + return "" else - return curr_dir + return b end end) :gsub(path_separator .. "+", path_separator)) @@ -547,6 +547,7 @@ local function remove_leading_path(leading_part, path) if s then return path:sub(e+1, -1) end + return path end local function traverse(dirname, emptyref) @@ -625,6 +626,9 @@ function project:find(path) -- allow for indexing with paths project:find("foo/b end return current_dir end +local function is_in(a, b) -- a is in b + return a:find("^" .. b) and true +end project.dir, project.paths, project.emptyref = traverse(lfs.currentdir()) project.source_file_map = {} @@ -683,11 +687,15 @@ if cmd == "build" then die("Build error: file \"" .. fname .. "\" not found") end project.source_file_map[fname] = fname:gsub("%.tl$", ".lua") + if tlconfig["build_dir"] then + project.source_file_map[path] = path_concat(tlconfig["build_dir"], project.source_file_map[path]) + end check_parent_dirs(project.source_file_map[fname]) end end for path in project:files(inc_patterns, exc_patterns) do if path:match("%.tl$") and not path:match("%.d%.tl$") then + local include = true project.source_file_map[path] = path:gsub("%.tl$", ".lua") if tlconfig["build_dir"] then if tlconfig["source_dir"] then @@ -695,12 +703,26 @@ if cmd == "build" then end project.source_file_map[path] = path_concat(tlconfig["build_dir"], project.source_file_map[path]) end - check_parent_dirs(project.source_file_map[path]) + + if ( + tlconfig["source_dir"] and + tlconfig["build_dir"] and + is_in(path, tlconfig["build_dir"]) and + not is_in(path, tlconfig["source_dir"]) + ) then -- handle when source_dir is in build_dir + include = false + -- die("Build error: file \"" .. path .. "\" is included even though it is in build_dir") + end + if include then + check_parent_dirs(project.source_file_map[path]) + else + project.source_file_map[path] = nil + end end end for i, v in ipairs(dirs_to_be_mked) do if not lfs.mkdir(v) then - die("Build error: unable to mkdir " .. v) + die("Build error: unable to mkdir \"" .. v .. "\"") end end end From cc0236a67a10374bd26861fbd00c77235f9e33dc Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 10:58:51 -0500 Subject: [PATCH 19/33] tl config: add skip_compat53 as a config key --- tl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tl b/tl index c8a84375f..7687ae27d 100755 --- a/tl +++ b/tl @@ -51,6 +51,7 @@ local function validate_config(config) preload_modules = true, quiet = true, source_dir = true, + skip_compat53 = true, } for k, _ in pairs(config) do @@ -189,6 +190,7 @@ if cmd == "build" then tlconfig["source_dir"] = args["source_dir"] or tlconfig["source_dir"] tlconfig["build_dir"] = args["build_dir"] or tlconfig["build_dir"] end +tlconfig["skip_compat53"] = args["skip_compat53"] or tlconfig["skip_compat53"] local function report_errors(category, errors) if not errors then @@ -266,7 +268,7 @@ local function setup_env(filename) lax_mode = false end - local skip_compat53 = args["skip_compat53"] + local skip_compat53 = tlconfig["skip_compat53"] env = tl.init_env(lax_mode, skip_compat53) end From 54dd7b3cc384d773c5c3d37039d9331197d110cd Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 14:16:42 -0500 Subject: [PATCH 20/33] tl gen: implement --output --- tl | 32 +++++++++++++------------------- 1 file changed, 13 insertions(+), 19 deletions(-) diff --git a/tl b/tl index 7687ae27d..b8fbc5211 100755 --- a/tl +++ b/tl @@ -121,7 +121,6 @@ local function get_args_parser() parser:option("-o --output", "Write to instead.") :argname("") - :count("*") parser:flag("--skip-compat53", "Skip compat53 insertions.") @@ -451,10 +450,19 @@ if cmd == "gen" then end end else - -- TODO: args["output"] for i, res in ipairs(results) do - write_out(res.tl_result, res.output_file) - print("Wrote: " .. res.output_file) + local append = args["output"] and i > 1 + write_out( + res.tl_result, + args["output"] or res.output_file, + append + ) + if not args["output"] then + print("Wrote: " .. res.output_file) + end + end + if args["output"] then + print("Wrote: " .. args["output"]) end end os.exit(exit) @@ -697,7 +705,6 @@ if cmd == "build" then end for path in project:files(inc_patterns, exc_patterns) do if path:match("%.tl$") and not path:match("%.d%.tl$") then - local include = true project.source_file_map[path] = path:gsub("%.tl$", ".lua") if tlconfig["build_dir"] then if tlconfig["source_dir"] then @@ -706,20 +713,7 @@ if cmd == "build" then project.source_file_map[path] = path_concat(tlconfig["build_dir"], project.source_file_map[path]) end - if ( - tlconfig["source_dir"] and - tlconfig["build_dir"] and - is_in(path, tlconfig["build_dir"]) and - not is_in(path, tlconfig["source_dir"]) - ) then -- handle when source_dir is in build_dir - include = false - -- die("Build error: file \"" .. path .. "\" is included even though it is in build_dir") - end - if include then - check_parent_dirs(project.source_file_map[path]) - else - project.source_file_map[path] = nil - end + check_parent_dirs(project.source_file_map[path]) end end for i, v in ipairs(dirs_to_be_mked) do From 5b909f6bc4995454b819f7ce85de51ecbe0bfac0 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 14:28:59 -0500 Subject: [PATCH 21/33] tl: implement --pretend and tidy up --output when there are multiple files going into one --- tl | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/tl b/tl index b8fbc5211..2fb28bfd6 100755 --- a/tl +++ b/tl @@ -407,7 +407,8 @@ end -- GEN -- -------------------------------------------------------------------- -local function write_out(result, output_file, append) +local function write_out(result, output_file, append, should_print) + if not args["pretend"] then local ofd, err = io.open(output_file, append and "a" or "w") if not ofd then @@ -420,6 +421,14 @@ local function write_out(result, output_file, append) end ofd:close() + end + if should_print then + if args["pretend"] then + print("Would Write: " .. output_file) + else + print("Wrote: " .. output_file) + end + end end if cmd == "gen" then @@ -452,17 +461,16 @@ if cmd == "gen" then else for i, res in ipairs(results) do local append = args["output"] and i > 1 + local should_print = true + if args["output"] then + should_print = i == #results + end write_out( res.tl_result, args["output"] or res.output_file, - append + append, + should_print ) - if not args["output"] then - print("Wrote: " .. res.output_file) - end - end - if args["output"] then - print("Wrote: " .. args["output"]) end end os.exit(exit) From f7e2b5c05d9d770764cfe524e7d5bcd9b130d223 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 14:29:41 -0500 Subject: [PATCH 22/33] spec: fix up some interaction tests --- spec/config/interactions_spec.lua | 88 +++++++++++++++++++------------ 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/spec/config/interactions_spec.lua b/spec/config/interactions_spec.lua index 632c97e33..3c5d76905 100644 --- a/spec/config/interactions_spec.lua +++ b/spec/config/interactions_spec.lua @@ -2,7 +2,7 @@ local util = require("spec.util") describe("config option interactions", function() describe("include+exclude", function() - pending("exclude should have precedence over include", function() + it("exclude should have precedence over include", function() util.run_mock_project(finally, { dir_name = "interaction_inc_exc_test", dir_structure = { @@ -37,7 +37,7 @@ describe("config option interactions", function() end) end) describe("source_dir+build_dir", function() - pending("Having source_dir inside of build_dir works", function() + it("Having source_dir inside of build_dir works", function() util.run_mock_project(finally, { dir_name = "source_dir_in_build_dir_test", dir_structure = { @@ -61,7 +61,7 @@ describe("config option interactions", function() }, }) end) - pending("Having build_dir inside of source_dir works if no inputs from ", function() + it("Having build_dir inside of source_dir works", function() util.run_mock_project(finally, { dir_name = "build_dir_in_source_dir_test", dir_structure = { @@ -85,60 +85,82 @@ describe("config option interactions", function() }, }) end) - it("fails when a file would be generated inside of source_dir while there is a build_dir", function() + end) + describe("source_dir+include+exclude", function() + it("nothing outside of source_dir is included", function() util.run_mock_project(finally, { - dir_name = "gen_in_source_dir_test", + dir_name = "source_dir_inc_exc_test", dir_structure = { ["tlconfig.lua"] = [[return { source_dir = "src", - build_dir = ".", + include = { + "**/*" + }, }]], - src = { - ["foo.tl"] = [[print "hi"]], - src = { - ["foo.tl"] = [[print "hi"]], + ["src"] = { + ["foo"] = { + ["bar"] = { + ["a.tl"] = [[print "a"]], + ["b.tl"] = [[print "b"]], + }, + ["a.tl"] = [[print "a"]], + ["b.tl"] = [[print "b"]], }, }, + ["a.tl"] = [[print "a"]], + ["b.tl"] = [[print "b"]], }, - generated_files = {}, -- Build errors should not generate anything cmd = "build", - popen = { - status = false, - exit = "exit", - code = 1, + generated_files = { + ["src"] = { + ["foo"] = { + ["bar"] = { + "a.lua", + "b.lua", + }, + "a.lua", + "b.lua", + }, + }, }, }) end) - pending("should not include any files in build_dir", function() + it("include and exclude work as expected", function() util.run_mock_project(finally, { - dir_name = "source_file_in_build_dir_test", + dir_name = "source_dir_inc_exc_2", dir_structure = { ["tlconfig.lua"] = [[return { - source_dir = ".", - build_dir = "build", + source_dir = "", + include = { + "foo/*.tl", + }, + exclude = { + "foo/a*.tl", + }, }]], - ["foo.tl"] = [[print "hi"]], - bar = { - ["baz.tl"] = [[print "hi"]], + foo = { + ["a.tl"] = [[print 'a']], + ["ab.tl"] = [[print 'a']], + ["ac.tl"] = [[print 'a']], + ["b.tl"] = [[print 'b']], + ["bc.tl"] = [[print 'b']], + ["bd.tl"] = [[print 'b']], }, - build = { - ["dont_include_this.tl"] = [[print "dont"]], + bar = { + ["c.tl"] = [[print 'c']], + ["cd.tl"] = [[print 'c']], + ["ce.tl"] = [[print 'c']], }, }, cmd = "build", generated_files = { - build = { - "foo.lua", - bar = { - "baz.lua", - }, + foo = { + "b.lua", + "bc.lua", + "bd.lua", }, }, }) end) end) - describe("source_dir+include+exclude", function() - pending("nothing outside of source_dir is included", function() - end) - end) end) From 7f98cbfc0589ddf2ad621785a343212003683738 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Fri, 10 Jul 2020 23:59:02 -0500 Subject: [PATCH 23/33] tl: downgrade --option to only take 1 script + move some options around --- tl | 48 +++++++++++++++++------------------------------- 1 file changed, 17 insertions(+), 31 deletions(-) diff --git a/tl b/tl index 2fb28bfd6..299e42078 100755 --- a/tl +++ b/tl @@ -113,15 +113,6 @@ local function get_args_parser() :argname("") :count("*") - parser:option("-s --source-dir", "Compile all *.tl files in (and all subdirectories).") - :argname("") - - parser:option("-b --build-dir", "Put all generated files in .") - :argname("") - - parser:option("-o --output", "Write to instead.") - :argname("") - parser:flag("--skip-compat53", "Skip compat53 insertions.") parser:flag("--version", "Print version and exit") @@ -138,11 +129,17 @@ local function get_args_parser() local gen_command = parser:command("gen", "Generate a Lua file for one or more tl script.") gen_command:argument("script", "The tl script."):args("+") + gen_command:option("-o --output", "Write to instead.") + :argname("") local run_command = parser:command("run", "Run a tl script.") run_command:argument("script", "The tl script."):args("+") local build_command = parser:command("build", "Build your project according to tlconfig.lua by type checking and compiling each specified file.") + build_command:option("-b --build-dir", "Put all generated files in .") + :argname("") + build_command:option("-s --source-dir", "Compile all *.tl files in (and all subdirectories).") + :argname("") return parser end @@ -190,6 +187,10 @@ if cmd == "build" then tlconfig["build_dir"] = args["build_dir"] or tlconfig["build_dir"] end tlconfig["skip_compat53"] = args["skip_compat53"] or tlconfig["skip_compat53"] +if cmd == "gen" and args["output"] and #args["script"] ~= 1 then + print("Error: --output can only be used to map one input to one output") + os.exit(1) +end local function report_errors(category, errors) if not errors then @@ -407,9 +408,9 @@ end -- GEN -- -------------------------------------------------------------------- -local function write_out(result, output_file, append, should_print) +local function write_out(result, output_file) if not args["pretend"] then - local ofd, err = io.open(output_file, append and "a" or "w") + local ofd, err = io.open(output_file, "w") if not ofd then die("cannot write " .. output_file .. ": " .. err) @@ -422,12 +423,10 @@ local function write_out(result, output_file, append, should_print) ofd:close() end - if should_print then - if args["pretend"] then - print("Would Write: " .. output_file) - else - print("Wrote: " .. output_file) - end + if args["pretend"] then + print("Would Write: " .. output_file) + else + print("Wrote: " .. output_file) end end @@ -460,17 +459,7 @@ if cmd == "gen" then end else for i, res in ipairs(results) do - local append = args["output"] and i > 1 - local should_print = true - if args["output"] then - should_print = i == #results - end - write_out( - res.tl_result, - args["output"] or res.output_file, - append, - should_print - ) + write_out(res.tl_result, args["output"] or res.output_file) end end os.exit(exit) @@ -760,9 +749,6 @@ for i, files in ipairs(sorted_source_file_arr) do end write_out(result, output_file) - if tlconfig["quiet"] == false then - print("Wrote: " .. output_file) - end end os.exit(exit) From 1132abe4bc34ec2f7db480955402320dce76c59b Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Sat, 11 Jul 2020 00:43:12 -0500 Subject: [PATCH 24/33] tl build: actually type check before compiling --- tl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tl b/tl index 299e42078..77ec9431b 100755 --- a/tl +++ b/tl @@ -119,7 +119,7 @@ local function get_args_parser() parser:flag("-q --quiet", "Do not print information messages to stdout. Errors may still be printed to stderr.") - parser:flag("-p --pretend", "Do not write to any files, just output what files would be generated.") + parser:flag("-p --pretend", "Do not write to any files, type check and output what files would be generated.") parser:require_command(false) parser:command_target("command") @@ -747,8 +747,10 @@ for i, files in ipairs(sorted_source_file_arr) do exit = 1 break end - - write_out(result, output_file) + local ok = report_type_errors(result) + if ok then + write_out(result, output_file) + end end os.exit(exit) From a58f8e8ed3251bfffe716cf2601051534557879c Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Sat, 11 Jul 2020 20:40:01 -0500 Subject: [PATCH 25/33] docs: update compiler options Added a 'relevant commands' section to the option table and updated the examples to use 'build' instead of 'gen' --- docs/compiler_options.md | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/docs/compiler_options.md b/docs/compiler_options.md index efd585d41..d2e62b4a0 100644 --- a/docs/compiler_options.md +++ b/docs/compiler_options.md @@ -21,16 +21,17 @@ return { ## List of compiler options -| Command line option | Config key | Type | Description | -| --- | --- | --- | --- | -| `-l --preload` | `preload_modules` | `{string}` | Execute the equivalent of `require('modulename')` before executing the tl script(s). | -| `-I --include-dir` | `include_dir` | `{string}` | Prepend this directory to the module search path. -| `--skip-compat53` | | | Skip compat53 insertions. -|| `include` | `{string}` | The set of files to compile/check. See below for details on patterns. -|| `exclude` | `{string}` | The set of files to exclude. See below for details on patterns. -| `-s --source-dir` | `source_dir` | `string` | Set the directory to be searched for files. `gen` will compile every .tl file in every subdirectory by default. -| `-b --build-dir` | `build_dir` | `string` | Set the directory for generated files, mimicking the file structure of the source files. -|| `files` | `{string}` | The names of files to be compiled. Does not accept patterns like `include`. +| Command line option | Config key | Type | Relevant Commands | Description | +| --- | --- | --- | --- | --- | +| `-l --preload` | `preload_modules` | `{string}` | `build` `check` `gen` `run` | Execute the equivalent of `require('modulename')` before executing the tl script(s). | +| `-I --include-dir` | `include_dir` | `{string}` | `build` `check` `gen` `run` | Prepend this directory to the module search path. +| `--skip-compat53` | `skip_compat53` | `boolean` | `build` `gen` | Skip compat53 insertions. +|| `include` | `{string}` | `build` | The set of files to compile/check. See below for details on patterns. +|| `exclude` | `{string}` | `build` | The set of files to exclude. See below for details on patterns. +| `-s --source-dir` | `source_dir` | `string` | `build` | Set the directory to be searched for files. `gen` will compile every .tl file in every subdirectory by default. +| `-b --build-dir` | `build_dir` | `string` | `build` | Set the directory for generated files, mimicking the file structure of the source files. +|| `files` | `{string}` | `build` | The names of files to be compiled. Does not accept patterns like `include`. +| `-p --pretend --dry-run` ||| `build` `gen` | Don't compile/write to any files, but type check and log what files would be written to. ### Include/Exclude patterns @@ -40,7 +41,7 @@ The `include` and `exclude` fields can have glob-like patterns in them: In addition - setting the `source_dir` has the effect of prepending `source_dir` to all patterns. -- setting the `build_dir` has the additional effect of adding `/**/` to `exclude` +- currently, `include` will only include `.tl` files even if the extension isn't specified For example: If our project was laid out as such: @@ -72,9 +73,8 @@ return { } ``` -Running `tl check` will type check the `include`d files. - -Running `tl gen` with no arguments would produce the following files. +Running `tl build -p` will type check the `include`d files and show what would be written to. +Running `tl build` will produce the following files. ``` tlconfig.lua src/ @@ -104,3 +104,4 @@ return { } } ``` +This will compile any `.tl` file with a sequential `foo`, `bar`, and `baz` directory in its path. From a8985ab3b49d1218b5c23b5592bbfc9f565371a5 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Sun, 12 Jul 2020 01:29:28 -0500 Subject: [PATCH 26/33] spec.util: add chdir_setup+chdir_teardown to make tests take place in /tmp --- spec/cli/gen_spec.lua | 40 +++++++++++++++++++++++----------- spec/cli/quiet_spec.lua | 2 ++ spec/cli/run_spec.lua | 4 +++- spec/util.lua | 48 +++++++++++++++++++++++++---------------- 4 files changed, 62 insertions(+), 32 deletions(-) diff --git a/spec/cli/gen_spec.lua b/spec/cli/gen_spec.lua index f4e275b7e..100a8c50e 100644 --- a/spec/cli/gen_spec.lua +++ b/spec/cli/gen_spec.lua @@ -1,3 +1,4 @@ +local lfs = require("lfs") local util = require("spec.util") local input_file = [[ @@ -66,10 +67,17 @@ end local c = 100 ]] +local function tl_to_lua(name) + return (name:gsub("%.tl$", ".lua")) +end + describe("tl gen", function() + setup(util.chdir_setup) + teardown(util.chdir_teardown) describe("on .tl files", function() it("reports 0 errors and code 0 on success", function() - local name = util.write_tmp_file(finally, "add.tl", [[ + local name = "add.tl" + util.write_tmp_file(finally, name, [[ local function add(a: number, b: number): number return a + b end @@ -79,7 +87,7 @@ describe("tl gen", function() local pd = io.popen("./tl gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = "add.lua" + local lua_name = tl_to_lua(name) assert.match("Wrote: " .. lua_name, output, 1, true) util.assert_line_by_line([[ local function add(a, b) @@ -91,7 +99,8 @@ describe("tl gen", function() end) it("ignores type errors", function() - local name = util.write_tmp_file(finally, "add.tl", [[ + local name = "add.tl" + util.write_tmp_file(finally, name, [[ local function add(a: number, b: number): number return a + b end @@ -103,7 +112,7 @@ describe("tl gen", function() local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) assert.same("", output) - local lua_name = "add.lua" + local lua_name = tl_to_lua(name) util.assert_line_by_line([[ local function add(a, b) return a + b @@ -115,7 +124,8 @@ describe("tl gen", function() end) it("reports number of errors in stderr and code 1 on syntax errors", function() - local name = util.write_tmp_file(finally, "add.tl", [[ + local name = "add.tl" + util.write_tmp_file(finally, name, [[ print(add("string", 20)))))) ]]) local pd = io.popen("./tl gen " .. name .. " 2>&1 1>/dev/null", "r") @@ -125,7 +135,8 @@ describe("tl gen", function() end) it("ignores unknowns code 0 if no errors", function() - local name = util.write_tmp_file(finally, "add.tl", [[ + local name = "add.tl" + util.write_tmp_file(finally, name, [[ local function unk(x, y): number, number return a + b end @@ -134,7 +145,7 @@ describe("tl gen", function() local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) assert.same("", output) - local lua_name = "add.lua" + local lua_name = tl_to_lua(name) util.assert_line_by_line([[ local function unk(x, y) return a + b @@ -143,11 +154,12 @@ describe("tl gen", function() end) it("does not mess up the indentation (#109)", function() - local name = util.write_tmp_file(finally, "add.tl", input_file) + local name = "add.tl" + util.write_tmp_file(finally, name, input_file) local pd = io.popen("./tl gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = "add.lua" + local lua_name = tl_to_lua(name) assert.match("Wrote: " .. lua_name, output, 1, true) assert.equal(output_file, util.read_file(lua_name)) end) @@ -155,14 +167,15 @@ describe("tl gen", function() describe("with --skip-compat53", function() it("does not add compat53 insertions", function() - local name = util.write_tmp_file(finally, "test.tl", [[ + local name = "test.tl" + util.write_tmp_file(finally, name, [[ local t = {1, 2, 3, 4} print(table.unpack(t)) ]]) local pd = io.popen("./tl --skip-compat53 gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = "test.lua" + local lua_name = tl_to_lua(name) assert.match("Wrote: " .. lua_name, output, 1, true) util.assert_line_by_line([[ local t = { 1, 2, 3, 4 } @@ -173,14 +186,15 @@ describe("tl gen", function() describe("without --skip-compat53", function() it("adds compat53 insertions by default", function() - local name = util.write_tmp_file(finally, "test.tl", [[ + local name = "test.tl" + util.write_tmp_file(finally, name, [[ local t = {1, 2, 3, 4} print(table.unpack(t)) ]]) local pd = io.popen("./tl gen " .. name, "r") local output = pd:read("*a") util.assert_popen_close(true, "exit", 0, pd:close()) - local lua_name = "test.lua" + local lua_name = tl_to_lua(name) assert.match("Wrote: " .. lua_name, output, 1, true) util.assert_line_by_line([[ local _tl_compat53 = ((tonumber((_VERSION or ''):match('[%d.]*$')) or 0) < 5.3) and require('compat53.module'); local table = _tl_compat53 and _tl_compat53.table or table; local _tl_table_unpack = unpack or table.unpack; local t = { 1, 2, 3, 4 } diff --git a/spec/cli/quiet_spec.lua b/spec/cli/quiet_spec.lua index 8ceb396bf..c5b4153fa 100644 --- a/spec/cli/quiet_spec.lua +++ b/spec/cli/quiet_spec.lua @@ -1,6 +1,8 @@ local util = require("spec.util") describe("-q --quiet flag", function() + setup(util.chdir_setup) + teardown(util.chdir_teardown) it("silences stdout when running tl check", function() local name = util.write_tmp_file(finally, "foo.tl", [[ local x: number = 123 diff --git a/spec/cli/run_spec.lua b/spec/cli/run_spec.lua index 4c4933c48..5beba7e17 100644 --- a/spec/cli/run_spec.lua +++ b/spec/cli/run_spec.lua @@ -1,6 +1,8 @@ local util = require("spec.util") describe("tl run", function() + setup(util.chdir_setup) + teardown(util.chdir_teardown) describe("on .tl files", function() it("reports nothing if no errors, runs and returns code 0 on success", function() local name = util.write_tmp_file(finally, "add.tl", [[ @@ -58,7 +60,7 @@ describe("tl run", function() end) it("can require other .tl files", function() - local add_tl = util.write_tmp_file(finally, "add.tl", [[ + util.write_tmp_file(finally, "add.tl", [[ local function add(a: number, b: number): number return a + b end diff --git a/spec/util.lua b/spec/util.lua index da8f778e1..9ca4a74ab 100644 --- a/spec/util.lua +++ b/spec/util.lua @@ -3,6 +3,9 @@ local util = {} local tl = require("tl") local assert = require("luassert") local lfs = require("lfs") +local current_dir = lfs.currentdir() +local tl_executable = current_dir .. "/tl" +local tl_lib = current_dir .. "/tl.lua" function util.mock_io(finally, files) assert(type(finally) == "function") @@ -55,6 +58,18 @@ function util.assert_line_by_line(s1, s2) end end +function util.chdir_setup() + assert(lfs.link("tl", "/tmp/tl")) + assert(lfs.link("tl.lua", "/tmp/tl.lua")) + assert(lfs.chdir("/tmp")) +end + +function util.chdir_teardown() + os.remove("tl.lua") + os.remove("tl") + assert(lfs.chdir(current_dir)) +end + function util.write_tmp_file(finally, name, content) assert(type(finally) == "function") assert(type(name) == "string") @@ -136,9 +151,6 @@ local function insert_into(tab, files) end end end -local tl_path = lfs.currentdir() -local tl_executable = tl_path .. "/tl" -local tl_lib = tl_path .. "/tl.lua" function util.run_mock_project(finally, t) assert(type(finally) == "function") assert(type(t) == "table") @@ -149,7 +161,7 @@ function util.run_mock_project(finally, t) check = true, run = true, build = true, - })[t.cmd], "Invalid tl ") + })[t.cmd], "Invalid command tl " .. t.cmd) local actual_dir_name = util.write_tmp_dir(finally, t.dir_name, t.dir_structure) lfs.link(tl_executable, actual_dir_name .. "/tl") lfs.link(tl_lib, actual_dir_name .. "/tl.lua") @@ -163,21 +175,21 @@ function util.run_mock_project(finally, t) local pd = io.popen("./tl " .. t.cmd .. " " .. (t.args or "")) local output = pd:read("*a") local actual_dir_structure = util.get_dir_structure(".") - lfs.chdir(tl_path) - t.popen = t.popen or { - status = true, - exit = "exit", - code = 0, - } - -- util.assert_popen_close( - -- t.popen.status, - -- t.popen.exit, - -- t.popen.code, - -- pd:close() - -- ) + lfs.chdir(current_dir) + if t.popen then + -- t.popen = t.popen or { + -- status = true, + -- exit = "exit", + -- code = 0, + -- } + util.assert_popen_close( + t.popen.status, + t.popen.exit, + t.popen.code, + pd:close() + ) + end if t.cmd_output then - -- FIXME: when generating multiple files their order isnt guaranteed - -- so either account for this here or make the order deterministic in tl assert.are.equal(output, t.cmd_output) end assert.are.same(expected_dir_structure, actual_dir_structure, "Actual directory structure is not as expected") From 3d4194b3ecf2adcb1908fe041ab6a47ef853a9bc Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Sun, 12 Jul 2020 11:15:02 -0500 Subject: [PATCH 27/33] spec/config: small tweaks --- spec/config/glob_spec.lua | 6 +++--- spec/config/interactions_spec.lua | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/config/glob_spec.lua b/spec/config/glob_spec.lua index 29e38dbe4..4cb37656e 100644 --- a/spec/config/glob_spec.lua +++ b/spec/config/glob_spec.lua @@ -34,12 +34,12 @@ describe("globs", function() }, cmd = "build", generated_files = { - "abzcd.lua", + "abbarcd.lua", "abcd.lua", "abfoocd.lua", - "abbarcd.lua", + "abzcd.lua", }, - --FIXME cmd_output = "Wrote: abzcd.lua\n", + cmd_output = "Wrote: abbarcd.lua\nWrote: abcd.lua\nWrote: abfoocd.lua\nWrote: abzcd.lua\n" }) end) it("should only match .tl by default", function() diff --git a/spec/config/interactions_spec.lua b/spec/config/interactions_spec.lua index 3c5d76905..55869528b 100644 --- a/spec/config/interactions_spec.lua +++ b/spec/config/interactions_spec.lua @@ -130,7 +130,7 @@ describe("config option interactions", function() dir_name = "source_dir_inc_exc_2", dir_structure = { ["tlconfig.lua"] = [[return { - source_dir = "", + source_dir = ".", include = { "foo/*.tl", }, From a1707f403ed01c290787d15094eac54cee77d75b Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Sun, 12 Jul 2020 11:15:30 -0500 Subject: [PATCH 28/33] spec: add initial --output tests --- spec/cli/output_spec.lua | 86 ++++++++++++++++++++++++++++++++++++++++ spec/util.lua | 6 ++- 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 spec/cli/output_spec.lua diff --git a/spec/cli/output_spec.lua b/spec/cli/output_spec.lua new file mode 100644 index 000000000..0c304a957 --- /dev/null +++ b/spec/cli/output_spec.lua @@ -0,0 +1,86 @@ +local lfs = require("lfs") +local util = require("spec.util") + +-- local path_separator = "/" +-- local function tl_name_to_relative_lua(file_name) +-- local tail = file_name:match("[^%" .. path_separator .. "]+$") +-- local name, ext = tail:match("(.+)%.([a-zA-Z]+)$") +-- if not name then name = tail end +-- return name .. ".lua" +-- end +local curr_dir = lfs.currentdir() +local tlcmd = "LUA_PATH+=" .. curr_dir .. "/?.tl" + +describe("-o --output", function() + setup(function() + os.execute("LUA_PATH+=" .. curr_dir .. "/?.lua") + util.chdir_setup() + end) + teardown(util.chdir_teardown) + it("should gen in the current directory when not provided", function() + util.write_tmp_dir(finally, "gen_curr_dir_test", { + bar = { + ["foo.tl"] = [[print 'hey']] + } + }) + assert(lfs.chdir("/tmp/gen_curr_dir_test")) + local pd = io.popen(curr_dir .. "/tl gen bar/foo.tl", "r") + local output = pd:read("*a") + lfs.chdir(curr_dir) + util.assert_popen_close(true, "exit", 0, pd:close()) + assert.match("Wrote: foo.lua", output, 1, true) + end) + it("should work with nested directories", function() + local dir_name = "gen_curr_dir_nested_test" + util.write_tmp_dir(finally, dir_name, { + a={b={c={["foo.tl"] = [[print 'hey']]}}} + }) + assert(lfs.chdir("/tmp/gen_curr_dir_nested_test")) + local pd = io.popen(curr_dir .. "/tl gen a/b/c/foo.tl", "r") + local output = pd:read("*a") + lfs.chdir(curr_dir) + util.assert_popen_close(true, "exit", 0, pd:close()) + assert.match("Wrote: foo.lua", output, 1, true) + end) + it("should write to the given filename", function() + local name = "foo.tl" + local outfile = "bar.lua" + util.write_tmp_dir(finally, "output_name_test", { + [name] = [[print 'hey']], + }) + assert(lfs.chdir("/tmp/output_name_test")) + local pd = io.popen(curr_dir .. "/tl gen " .. name .. " -o " .. outfile, "r") + local output = pd:read("*a") + util.assert_popen_close(true, "exit", 0, pd:close()) + assert.match("Wrote: " .. outfile, output, 1, true) + end) + it("should write to the given filename in a directory", function() + local name = "foo.tl" + local outfile = "a/b/c/d.lua" + local dir_name = "nested_dir_output_test" + util.write_tmp_dir(finally, dir_name, { + [name] = [[print 'foo']], + a={b={c={}}}, + }) + assert(lfs.chdir("/tmp/" .. dir_name)) + local pd = io.popen(curr_dir .. "/tl gen " .. name .. " -o " .. outfile, "r") + local output = pd:read("*a") + lfs.chdir(curr_dir) + util.assert_popen_close(true, "exit", 0, pd:close()) + assert.match("Wrote: " .. outfile, output, 1, true) + end) + it("should gracefully error when the output directory doesn't exist", function() + local name = "foo.tl" + local outfile = "a/b/c/d.lua" + local dir_name = "nested_dir_output_fail_test" + util.write_tmp_dir(finally, dir_name, { + [name] = [[print 'foo']], + }) + assert(lfs.chdir("/tmp/" .. dir_name)) + local pd = io.popen(curr_dir .. "/tl gen " .. name .. " -o " .. outfile .. " 2>&1", "r") + local output = pd:read("*a") + lfs.chdir(curr_dir) + util.assert_popen_close(nil, "exit", 1, pd:close()) + assert.match("cannot write " .. outfile .. ": " .. outfile .. ": No such file or directory", output, 1, true) + end) +end) diff --git a/spec/util.lua b/spec/util.lua index 9ca4a74ab..56bd6db63 100644 --- a/spec/util.lua +++ b/spec/util.lua @@ -65,8 +65,10 @@ function util.chdir_setup() end function util.chdir_teardown() - os.remove("tl.lua") - os.remove("tl") + -- explicitly use /tmp here + -- just in case it may remove the actual tl file + os.remove("/tmp/tl.lua") + os.remove("/tmp/tl") assert(lfs.chdir(current_dir)) end From 75dbd364f7bb5eb282f16e2702ec8d10460e6d8b Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Sun, 12 Jul 2020 14:08:28 -0500 Subject: [PATCH 29/33] tl: use path_separator in glob pattern --- tl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tl b/tl index 77ec9431b..5433b3ec2 100755 --- a/tl +++ b/tl @@ -508,7 +508,7 @@ local function patt_match(patt, str) end local function matcher(str) local chunks = {} - for piece in str_split(str, "**/") do + for piece in str_split(str, "**" .. path_separator) do table.insert(chunks, (piece:gsub("%*", "[^" .. path_separator .. "]-"))) end chunks[1] = "^" .. chunks[1] @@ -529,7 +529,7 @@ local function cleanup_file_name(name) --remove trailing and extra path separato :gsub("^(%.)(.?)", function(a, b) assert(a == ".") if b == "." then - die("Config error: ../ not allowed, please use direct paths") + die("Config error: .." .. path_separator .. " not allowed, please use direct paths") elseif b == path_separator then return "" else @@ -633,9 +633,9 @@ function project:find(path) -- allow for indexing with paths project:find("foo/b end return current_dir end -local function is_in(a, b) -- a is in b - return a:find("^" .. b) and true -end +-- local function is_in(a, b) -- a is in b +-- return a:find("^" .. b) and true +-- end project.dir, project.paths, project.emptyref = traverse(lfs.currentdir()) project.source_file_map = {} From 409e23d187642ffec8de4f92c5dfe05149a9d72c Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 14 Jul 2020 18:39:30 -0500 Subject: [PATCH 30/33] tl build: die when no config is found --- tl | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tl b/tl index 5433b3ec2..cd295e7a6 100755 --- a/tl +++ b/tl @@ -45,7 +45,6 @@ local function validate_config(config) build_dir = true, exclude = true, files = true, - ignore = true, include = true, include_dir = true, preload_modules = true, @@ -65,6 +64,7 @@ local function validate_config(config) return nil end +local config_from_file = false local function get_config() local config = { preload_modules = {}, @@ -81,6 +81,7 @@ local function get_config() die("Error while loading config:\n" .. user_config) end end + config_from_file = true -- Merge tlconfig with the default config for k, v in pairs(user_config) do @@ -465,6 +466,10 @@ if cmd == "gen" then os.exit(exit) end +if cmd == "build" and not config_from_file then + die("Build error: no config file") +end + -------------------------------------------------------------------- -- PATTERN MATCHING -- From 93308634291d179295701d21911d5b218a5cafe4 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 14 Jul 2020 18:40:16 -0500 Subject: [PATCH 31/33] tl build: remove metatables from pattern arrays --- tl | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tl b/tl index cd295e7a6..6e6db1255 100755 --- a/tl +++ b/tl @@ -483,9 +483,8 @@ local function match(patt_arr, str) end return nil end -local patt_mt = {__index = {match = match}} -local inc_patterns = setmetatable({}, patt_mt) -local exc_patterns = setmetatable({}, patt_mt) +local inc_patterns = {} +local exc_patterns = {} local function str_split(str, delimiter) local idx = 0 @@ -606,14 +605,16 @@ function project:files(inc_patt_arr, exc_patt_arr) -- iterate over the files in if tlconfig["files"] then include = false end + -- TODO: print out patterns that include/exclude paths to help + -- users debug tlconfig.lua (this is why match returns the array index) if #inc_patt_arr > 0 then - local idx = inc_patt_arr:match(path) + local idx = match(inc_patt_arr, path) if not idx then include = false end end if #exc_patt_arr > 0 then - idx = exc_patt_arr:match(path) + local idx = match(exc_patt_arr, path) if include and idx then include = false end From a5986d84dd0768107d7ab0fed4ada1b81f26841c Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 14 Jul 2020 18:59:22 -0500 Subject: [PATCH 32/33] tl build+spec: die when source_dir doesn't exist or isn't a dir --- spec/cli/build_dir_spec.lua | 15 +++++++++++++++ spec/cli/source_dir_spec.lua | 12 ++++++++++++ spec/util.lua | 2 +- tl | 8 +++++++- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/spec/cli/build_dir_spec.lua b/spec/cli/build_dir_spec.lua index 7067efd3f..3f7402adf 100644 --- a/spec/cli/build_dir_spec.lua +++ b/spec/cli/build_dir_spec.lua @@ -55,4 +55,19 @@ describe("-b --build-dir argument", function() }, }) end) + it("dies when no config is found", function() + util.run_mock_project(finally, { + dir_name = "build_dir_die_test", + dir_structure = {}, + cmd = "build", + generated_files = {}, + popen = { + status = nil, + exit = "exit", + code = 1, + }, + cmd_output = "Build error: tlconfig.lua not found\n" + }) + end) + end) diff --git a/spec/cli/source_dir_spec.lua b/spec/cli/source_dir_spec.lua index 72b96e634..ac1b88309 100644 --- a/spec/cli/source_dir_spec.lua +++ b/spec/cli/source_dir_spec.lua @@ -32,4 +32,16 @@ describe("-s --source-dir argument", function() }, }) end) + it("should die when the given directory doesn't exist", function() + util.run_mock_project(function() end, { + dir_name = "no_source_dir_test", + dir_structure = { + ["tlconfig.lua"] = [[return {source_dir="src"}]], + ["foo.tl"] = [[print 'hi']], + }, + cmd = "build", + generated_files = {}, + cmd_output = "Build error: source_dir 'src' doesn't exist\n", + }) + end) end) diff --git a/spec/util.lua b/spec/util.lua index 56bd6db63..0e4e5e091 100644 --- a/spec/util.lua +++ b/spec/util.lua @@ -174,7 +174,7 @@ function util.run_mock_project(finally, t) insert_into(expected_dir_structure, t.dir_structure) insert_into(expected_dir_structure, t.generated_files) lfs.chdir(actual_dir_name) - local pd = io.popen("./tl " .. t.cmd .. " " .. (t.args or "")) + local pd = io.popen("./tl " .. t.cmd .. " " .. (t.args or "") .. " 2>&1") local output = pd:read("*a") local actual_dir_structure = util.get_dir_structure(".") lfs.chdir(current_dir) diff --git a/tl b/tl index 6e6db1255..c55f597da 100755 --- a/tl +++ b/tl @@ -467,7 +467,7 @@ if cmd == "gen" then end if cmd == "build" and not config_from_file then - die("Build error: no config file") + die("Build error: tlconfig.lua not found") end @@ -648,6 +648,12 @@ project.source_file_map = {} if cmd == "build" then if tlconfig["source_dir"] then tlconfig["source_dir"] = cleanup_file_name(tlconfig["source_dir"]) + local project_source = project:find(tlconfig["source_dir"]) + if not project_source then + die("Build error: source_dir '" .. tlconfig["source_dir"] .. "' doesn't exist") + elseif getmetatable(project_source) == project.emptyref then + die("Build error: source_dir '" .. tlconfig["source_dir"] .. "' is not a directory") + end end if tlconfig["build_dir"] then tlconfig["build_dir"] = cleanup_file_name(tlconfig["build_dir"]) From 8f98af8e5aa155677294830b2914e8cd5afb44a1 Mon Sep 17 00:00:00 2001 From: Corey Williamson Date: Tue, 14 Jul 2020 19:05:43 -0500 Subject: [PATCH 33/33] tl build: project:find() now correctly returns the current dir --- spec/cli/source_dir_spec.lua | 2 +- tl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/cli/source_dir_spec.lua b/spec/cli/source_dir_spec.lua index ac1b88309..880f9cff7 100644 --- a/spec/cli/source_dir_spec.lua +++ b/spec/cli/source_dir_spec.lua @@ -33,7 +33,7 @@ describe("-s --source-dir argument", function() }) end) it("should die when the given directory doesn't exist", function() - util.run_mock_project(function() end, { + util.run_mock_project(finally, { dir_name = "no_source_dir_test", dir_structure = { ["tlconfig.lua"] = [[return {source_dir="src"}]], diff --git a/tl b/tl index c55f597da..fb64064a8 100755 --- a/tl +++ b/tl @@ -630,6 +630,7 @@ function project:files(inc_patt_arr, exc_patt_arr) -- iterate over the files in return coroutine.wrap(iter), self.dir end function project:find(path) -- allow for indexing with paths project:find("foo/bar") -> project.dir.foo.bar + if path == "" then return self.dir end -- empty string is the current dir local current_dir = self.dir for dirname in str_split(path, path_separator) do current_dir = current_dir[dirname]