Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

wrong behavior of limit operator #166

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

wrong behavior of limit operator #166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions include/s3select.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2267,7 +2267,10 @@ class base_s3object
size_t m_error_count;
bool m_is_limit_on;
unsigned long m_limit;
//number of rows processed by the query
unsigned long m_processed_rows;
//number of rows returned by the query
unsigned long m_returned_rows;
size_t m_returned_bytes_size;
std::function<void(const char*)> m_fp_ext_debug_mesg=nullptr;//dispache debug message into external system
std::vector<std::string> m_projection_keys{};
Expand Down Expand Up @@ -2395,6 +2398,7 @@ class base_s3object
}

m_processed_rows = 0;
m_returned_rows = 0;
}

void set_continue_mesg_interval(std::chrono::seconds interval)
Expand Down Expand Up @@ -2472,6 +2476,9 @@ class base_s3object
std::string output_delimiter(1,m_csv_defintion.output_column_delimiter);
std::string output_row_delimiter(1,m_csv_defintion.output_row_delimiter);

//each call to this function is a new row
m_returned_rows++;

if(m_csv_defintion.output_json_format && projections_resuls.values.size()) {
json_result_format(projections_resuls, result, output_delimiter);
result.append(output_row_delimiter);
Expand Down Expand Up @@ -2551,7 +2558,7 @@ class base_s3object
{
multi_values projections_resuls;

if (m_is_limit_on && m_processed_rows == m_limit)
if (m_is_limit_on && m_returned_rows == m_limit)
{
return m_sql_processing_status = Status::LIMIT_REACHED;
}
Expand Down Expand Up @@ -2605,7 +2612,7 @@ class base_s3object
}
}

if(m_is_limit_on && m_processed_rows == m_limit)
if(m_is_limit_on && m_returned_rows == m_limit)
{
for (auto& i : m_projections)
{
Expand Down Expand Up @@ -2645,14 +2652,14 @@ class base_s3object
}

}
while (multiple_row_processing() && m_where_clause && !(where_clause_result = m_where_clause->eval().is_true()) && !(m_is_limit_on && m_processed_rows == m_limit));
while (multiple_row_processing() && m_where_clause && !(where_clause_result = m_where_clause->eval().is_true()) && !(m_is_limit_on && m_returned_rows == m_limit));

// in the of JSON it needs to evaluate the where-clause(for the first time)
if(!multiple_row_processing() && m_where_clause){
where_clause_result = m_where_clause->eval().is_true();
}

if(m_where_clause && ! where_clause_result && m_is_limit_on && m_processed_rows == m_limit)
if(m_where_clause && ! where_clause_result && m_is_limit_on && m_returned_rows == m_limit)
{
return m_sql_processing_status = Status::LIMIT_REACHED;
}
Expand All @@ -2672,11 +2679,11 @@ class base_s3object
{
projections_resuls.push_value( &(i->eval()) );
}
result_values_to_string(projections_resuls,result);
if(m_sql_processing_status == Status::SQL_ERROR)
{
result_values_to_string(projections_resuls,result);
if(m_sql_processing_status == Status::SQL_ERROR)
{
return m_sql_processing_status;
}
}
}
}

Expand Down
28 changes: 14 additions & 14 deletions test/s3select_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2648,58 +2648,58 @@ TEST(TestS3selectFunctions, limit)
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

input_query = "select _1 from stdin where _2 > _3 limit 8;";
input_query = "select _1 from stdin where _2 > _3 limit 1;";
expected_res = "7\n";
std::cout << "Running query: 3 (non-aggregate_query, where + limit clause)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

input_query = "select _1 from stdin where _2 > _3 limit 7;";
input_query = "select _1 from stdin where _2 > _3 limit 1;";
expected_res = "7\n";
std::cout << "Running query: 4 (non-aggregate_query, where + limit clause)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

input_query = "select _1 from stdin where _2 > _3 limit 6;";
expected_res = "";
input_query = "select _1 from stdin where _2 < _3 limit 3;";
expected_res = "1\n2\n3\n";
std::cout << "Running query: 5 (non-aggregate_query, where + limit clause)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

input_query = "select count(0) from stdin limit 9;";
expected_res = "9";
input_query = "select count(0) from stdin limit 9;";//limit X has no affect because it is aggregate query
expected_res = "20";
std::cout << "Running query: 6 (aggregate query, limit clause only)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

input_query = "select count(0) from stdin where _2 > _3 limit 8;";
expected_res = "1";
expected_res = "2";
std::cout << "Running query: 7 (aggregate_query, where + limit clause)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

input_query = "select count(0) from stdin where _2 > _3 limit 7;";
expected_res = "1";
expected_res = "2";
std::cout << "Running query: 8 (aggregate_query, where + limit clause)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

input_query = "select count(0) from stdin where _2 > _3 limit 6;";
expected_res = "0";
expected_res = "2";
std::cout << "Running query: 9 (aggregate_query, where + limit clause)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

generate_csv_multirow(input_csv, 10000);

input_query = "select count(0) from stdin limit 90000;";
expected_res = "90000";
expected_res = "100000";
std::cout << "Running query: 10 (aggregate_query, limit clause only, with Large input)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);

input_query = "select count(0) from stdin where _2 > _3 limit 90000;";
expected_res = "9000";
expected_res = "10000";
std::cout << "Running query: 11 (aggregate_query, where + limit clause, with Large input)" << std::endl;
s3select_res = run_s3select(input_query, input_csv);
EXPECT_EQ(s3select_res, expected_res);
Expand Down Expand Up @@ -2757,13 +2757,13 @@ TEST(TestS3selectFunctions, limit)
EXPECT_EQ(s3select_res, expected_res);

input_query_json = "select _1.addr from s3object[*].phoneNumbers where _1.type like \"%1%\" limit 12;";
expected_res = "11\n100\n101\n102\n";
expected_res = "11\n100\n101\n102\n103\n104\n105\n";
std::cout << "Running query: 14 (json, non-aggregate query, where + limit clause)" << std::endl;
run_json_query(input_query_json, json_input, s3select_res);
EXPECT_EQ(s3select_res, expected_res);

input_query_json = "select count(0) from s3object[*].phoneNumbers limit 9;";
expected_res = "9";
expected_res = "15";
std::cout << "Running query: 15 (json, aggregate query, limit clause only, limit reached)" << std::endl;
run_json_query(input_query_json, json_input, s3select_res);
EXPECT_EQ(s3select_res, expected_res);
Expand All @@ -2775,7 +2775,7 @@ TEST(TestS3selectFunctions, limit)
EXPECT_EQ(s3select_res, expected_res);

input_query_json = "select count(0) from s3object[*].phoneNumbers where _1.type like \"%1_\" limit 10;";
expected_res = "1";
expected_res = "6";
std::cout << "Running query: 17 (json, aggregate query, where + limit clause)" << std::endl;
run_json_query(input_query_json, json_input, s3select_res);
EXPECT_EQ(s3select_res, expected_res);
Expand Down
Loading