Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optional delimiter #37

Closed
wants to merge 2 commits into from
Closed

Optional delimiter #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
10d4fdd
Change delimiter arg to be optional
timothymtorres Mar 17, 2023
5db31e4
Fix unit tests for delimiters
timothymtorres Mar 17, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions ftcsv.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -397,9 +397,6 @@ local function initializeInputFromStringOrFile(inputFile, options, amount)
end

local function parseOptions(delimiter, options, fromParseLine)
-- delimiter MUST be one character
assert(#delimiter == 1 and type(delimiter) == "string", "the delimiter must be of string type and exactly one character")

local fieldsToKeep = nil

if options then
Expand Down Expand Up @@ -538,7 +535,10 @@ local function parseHeadersAndSetupArgs(inputString, delimiter, options, fieldsT
end

-- runs the show!
function ftcsv.parse(inputFile, delimiter, options)
function ftcsv.parse(inputFile, options)
local delimiter = options.delimiter or "," -- delimiter MUST be one character
assert(#delimiter == 1 and type(delimiter) == "string", "the delimiter must be of string type and exactly one character")

local options, fieldsToKeep = parseOptions(delimiter, options, false)

local inputString = initializeInputFromStringOrFile(inputFile, options, "*all")
Expand Down Expand Up @@ -572,7 +572,10 @@ local function initializeInputFile(inputString, options)
return initializeInputFromStringOrFile(inputString, options, options.bufferSize)
end

function ftcsv.parseLine(inputFile, delimiter, userOptions)
function ftcsv.parseLine(inputFile, userOptions)
local delimiter = userOptions.delimiter or "," -- delimiter MUST be one character
assert(#delimiter == 1 and type(delimiter) == "string", "the delimiter must be of string type and exactly one character")

local options, fieldsToKeep = parseOptions(delimiter, userOptions, true)
local inputString, file = initializeInputFile(inputFile, options)

Expand Down Expand Up @@ -765,9 +768,6 @@ local function getHeadersFromOptions(options)
end

local function initializeGenerator(inputTable, delimiter, options)
-- delimiter MUST be one character
assert(#delimiter == 1 and type(delimiter) == "string", "the delimiter must be of string type and exactly one character")

local headers = getHeadersFromOptions(options)
if headers == nil then
headers = extractHeadersFromTable(inputTable)
Expand All @@ -780,7 +780,10 @@ local function initializeGenerator(inputTable, delimiter, options)
end

-- works really quickly with luajit-2.1, because table.concat life
function ftcsv.encode(inputTable, delimiter, options)
function ftcsv.encode(inputTable, options)
local delimiter = options.delimiter or "," -- delimiter MUST be one character
assert(#delimiter == 1 and type(delimiter) == "string", "the delimiter must be of string type and exactly one character")

local output, headers = initializeGenerator(inputTable, delimiter, options)

for i, line in csvLineGenerator(inputTable, delimiter, headers, options) do
Expand Down
32 changes: 16 additions & 16 deletions spec/dynamic_features_spec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ describe("csv features", function()
end

local options = {loadFromString=true}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -61,7 +61,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "f"}}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -92,7 +92,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, rename={["a"] = "d", ["b"] = "e", ["c"] = "e"}}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -123,7 +123,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, fieldsToKeep={"a", "b"}}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -154,7 +154,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, rename={["c"] = "b"}, fieldsToKeep={"a","b"}}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -185,7 +185,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, rename={["c"] = "f"}, fieldsToKeep={"a","f"}}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -217,7 +217,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, headerFunc=string.upper}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -248,7 +248,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, rename={["c"] = "f"}, fieldsToKeep={"A","F"}, headerFunc=string.upper}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -278,7 +278,7 @@ describe("csv features", function()
end

local options = {loadFromString=true}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -315,7 +315,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, headers=false}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -347,7 +347,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, headers=false}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -383,7 +383,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, headers=false, rename={"a","b","c"}}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -417,7 +417,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, headers=false, rename={"a","b","c"}, fieldsToKeep={"a","b"}}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -451,7 +451,7 @@ describe("csv features", function()
end

local options = {loadFromString=true, headers=false, rename={"a","b"}, fieldsToKeep={"a","b"}}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
Expand Down Expand Up @@ -481,11 +481,11 @@ describe("csv features", function()
end

local options = {loadFromString=true, ignoreQuotes=true}
local actual, actualHeaders = ftcsv.parse(defaultString, ",", options)
local actual, actualHeaders = ftcsv.parse(defaultString, options)
assert.are.same(expected, actual)
assert.are.same(expectedHeaders, actualHeaders)
end)
end
end
end
end)
end)
14 changes: 7 additions & 7 deletions spec/error_spec.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ describe("csv decode error", function()
for _, value in ipairs(files) do
it("should error out " .. value[1], function()
local test = function()
ftcsv.parse("spec/bad_csvs/" .. value[1] .. ".csv", ",")
ftcsv.parse("spec/bad_csvs/" .. value[1] .. ".csv")
end
assert.has_error(test, value[2])
end)
Expand All @@ -23,8 +23,8 @@ end)

it("should error out for fieldsToKeep if no headers and no renaming takes place", function()
local test = function()
local options = {loadFromString=true, headers=false, fieldsToKeep={1, 2}}
ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", ">", options)
local options = {loadFromString=true, headers=false, fieldsToKeep={1, 2}, delimiter=">"}
ftcsv.parse("apple>banana>carrot\ndiamond>emerald>pearl", options)
end
assert.has_error(test, "ftcsv: fieldsToKeep only works with header-less files when using the 'rename' functionality")
end)
Expand All @@ -37,7 +37,7 @@ it("should error out when you want to encode a table and specify a field that do
}

local test = function()
ftcsv.encode(encodeThis, ">", {fieldsToKeep={"c"}})
ftcsv.encode(encodeThis, {fieldsToKeep={"c"}, delimiter=">"})
end

assert.has_error(test, "ftcsv: the field 'c' doesn't exist in the inputTable")
Expand All @@ -47,7 +47,7 @@ describe("parseLine features small, nonworking buffer size", function()
it("should error out when trying to load from string", function()
local test = function()
local parse = {}
for i, line in ftcsv.parseLine("a,b,c\n1,2,3", ",", {loadFromString=true}) do
for i, line in ftcsv.parseLine("a,b,c\n1,2,3", {loadFromString=true}) do
parse[i] = line
end
return parse
Expand All @@ -58,14 +58,14 @@ end)

it("should error when dealing with quotes", function()
local test = function()
local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', ",", {loadFromString=true})
local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', {loadFromString=true})
end
assert.has_error(test, "ftcsv: can't find closing quote in row 1. Try running with the option ignoreQuotes=true if the source incorrectly uses quotes.")
end)

it("should error if bufferSize is set when parsing entire files", function()
local test = function()
local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', ",", {loadFromString=true, bufferSize=34})
local actual = ftcsv.parse('a,b,c\n"apple,banana,carrot', {loadFromString=true, bufferSize=34})
end
assert.has_error(test, "ftcsv: bufferSize can only be specified using 'parseLine'. When using 'parse', the entire file is read into memory")
end)
Loading