[DBMON-3057] create Oracle test suite (#26)

* create Oracle test suite

* more oracle specific syntax tests

* update invoke procedure test case

dbms_test.go

@@ -35,6 +35,7 @@ type testcase struct {
 func TestQueriesPerDBMS(t *testing.T) {
 	dbmsTypes := []DBMSType{
 		DBMSPostgres,
+		DBMSOracle,
 	}
 
 	for _, dbms := range dbmsTypes {

sqllexer_utils.go

@@ -137,6 +137,7 @@ var keywords = map[string]bool{
 	"RETURNING":  true,
 	"OFFSET":     true,
 	"OF":         true,
+	"SKIP":       true,
 }
 
 func isWhitespace(ch rune) bool {

testdata/oracle/complex/bulk-operations.json

@@ -0,0 +1,27 @@
+{
+    "input": "DECLARE TYPE EmpTabTyp IS TABLE OF employees%ROWTYPE INDEX BY PLS_INTEGER; emp_tab EmpTabTyp; BEGIN SELECT * BULK COLLECT INTO emp_tab FROM employees; FORALL i IN emp_tab.FIRST .. emp_tab.LAST SAVE EXCEPTIONS UPDATE employees SET test = test * 1.05 WHERE employee_id = emp_tab(i).employee_id; END;",
+    "outputs": [
+      {
+        "expected": "DECLARE TYPE EmpTabTyp IS TABLE OF employees % ROWTYPE INDEX BY PLS_INTEGER; emp_tab EmpTabTyp; BEGIN SELECT * BULK COLLECT INTO emp_tab FROM employees; FORALL i IN emp_tab.FIRST . . emp_tab.LAST SAVE EXCEPTIONS UPDATE employees SET test = test * ? WHERE employee_id = emp_tab(i) . employee_id; END;",
+        "statement_metadata": {
+          "size": 33,
+          "tables": ["emp_tab", "employees"],
+          "commands": ["BEGIN", "SELECT", "UPDATE"],
+          "comments": [],
+          "procedures": []
+        },
+        "obfuscator_config": {
+            "replace_digits": true
+        },
+        "normalizer_config": {
+            "collect_tables": true,
+            "collect_commands": true,
+            "collect_comments": true,
+            "collect_procedure": true,
+            "keep_trailing_semicolon": true,
+            "remove_space_between_parentheses": true
+        }
+      }
+    ]
+  }
+

testdata/oracle/complex/complex-multi-table-delete.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM orders o WHERE o.customer_id IN (SELECT c.id FROM customers c WHERE NOT EXISTS (SELECT 1 FROM customer_orders co WHERE co.customer_id = c.id AND co.order_date > SYSDATE - 365)) AND EXISTS (SELECT 1 FROM order_items oi WHERE oi.order_id = o.id AND oi.product_id IN (SELECT p.id FROM products p WHERE p.category = 'Obsolete'));",
+    "outputs": [
+      {
+        "expected": "DELETE FROM orders o WHERE o.customer_id IN ( SELECT c.id FROM customers c WHERE NOT EXISTS ( SELECT ? FROM customer_orders co WHERE co.customer_id = c.id AND co.order_date > SYSDATE - ? ) ) AND EXISTS ( SELECT ? FROM order_items oi WHERE oi.order_id = o.id AND oi.product_id IN ( SELECT p.id FROM products p WHERE p.category = ? ) )",
+        "statement_metadata": {
+          "size": 61,
+          "tables": ["orders", "customers", "customer_orders", "order_items", "products"],
+          "commands": ["DELETE", "SELECT"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/complex/complex-nested-subqueries.json

@@ -0,0 +1,16 @@
+{
+    "input": "SELECT e.employee_id, (SELECT MAX(s.yoe) FROM employees s WHERE s.department_id = e.department_id) AS max_dept_yoe FROM employees e WHERE EXISTS (SELECT 1 FROM departments d WHERE d.id = e.department_id AND d.budget > (SELECT AVG(budget) FROM departments)) ORDER BY e.department_id, e.employee_id;",
+    "outputs": [
+      {
+        "expected": "SELECT e.employee_id, ( SELECT MAX ( s.yoe ) FROM employees s WHERE s.department_id = e.department_id ) FROM employees e WHERE EXISTS ( SELECT ? FROM departments d WHERE d.id = e.department_id AND d.budget > ( SELECT AVG ( budget ) FROM departments ) ) ORDER BY e.department_id, e.employee_id",
+        "statement_metadata": {
+          "size": 26,
+          "tables": ["employees", "departments"],
+          "commands": ["SELECT"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/complex/complex-select-aggregates-joins.json

@@ -0,0 +1,23 @@
+{
+    "input": "SELECT u.id, u.name, COUNT(o.id) AS order_count, AVG(o.total) AS average_order FROM users u JOIN orders o ON u.id = o.user_id WHERE u.status = 'active' GROUP BY u.id, u.name HAVING COUNT(o.id) > 5;",
+    "outputs": [
+      {
+        "expected": "SELECT u.id, u.name, COUNT ( o.id ), AVG ( o.total ) FROM users u JOIN orders o ON u.id = o.user_id WHERE u.status = ? GROUP BY u.id, u.name HAVING COUNT ( o.id ) > ?",
+        "statement_metadata": {
+          "size": 21,
+          "tables": ["users", "orders"],
+          "commands": ["SELECT", "JOIN"],
+          "comments": [],
+          "procedures": []
+        }
+      },
+      {
+        "expected": "SELECT u.id, u.name, COUNT(o.id), AVG(o.total) FROM users u JOIN orders o ON u.id = o.user_id WHERE u.status = ? GROUP BY u.id, u.name HAVING COUNT(o.id) > ?;",
+        "normalizer_config": {
+            "keep_trailing_semicolon": true,
+            "remove_space_between_parentheses": true
+        }
+      }
+    ]
+  }
+

testdata/oracle/complex/extremely-complex-oracle-query.json

@@ -0,0 +1,16 @@
+{
+    "input": "WITH RECURSIVE sales_cte (product_id, total_sales, sales_rank) AS (SELECT product_id, SUM(amount), RANK() OVER (ORDER BY SUM(amount) DESC) FROM sales GROUP BY product_id UNION ALL SELECT s.product_id, s.total_sales, s.sales_rank FROM sales s JOIN sales_cte sc ON s.product_id = sc.product_id WHERE s.amount > 1000), complex_view AS (SELECT e.employee_id, e.department_id, e.test_amt, AVG(e.test_amt) OVER (PARTITION BY e.department_id) AS avg_dept_test_amt, d.department_name, d.manager_id, (SELECT MAX(p.price) FROM products p WHERE p.department_id = e.department_id) AS max_product_price FROM employees e JOIN departments d ON e.department_id = d.id WHERE e.hire_date > SYSDATE - INTERVAL '10' YEAR) SELECT cv.*, sc.total_sales, sc.sales_rank FROM complex_view cv LEFT JOIN sales_cte sc ON cv.department_id = sc.product_id WHERE cv.avg_dept_test_amt > (SELECT AVG(total_sal) FROM (SELECT department_id, SUM(test_amt) AS total_sal FROM employees GROUP BY department_id)) AND EXISTS (SELECT 1 FROM customer_orders co WHERE co.employee_id = cv.employee_id AND co.order_status = 'Completed') ORDER BY cv.department_id, cv.test_amt DESC;",
+    "outputs": [
+      {
+        "expected": "WITH RECURSIVE sales_cte ( product_id, total_sales, sales_rank ) AS ( SELECT product_id, SUM ( amount ), RANK ( ) OVER ( ORDER BY SUM ( amount ) DESC ) FROM sales GROUP BY product_id UNION ALL SELECT s.product_id, s.total_sales, s.sales_rank FROM sales s JOIN sales_cte sc ON s.product_id = sc.product_id WHERE s.amount > ? ), complex_view AS ( SELECT e.employee_id, e.department_id, e.test_amt, AVG ( e.test_amt ) OVER ( PARTITION BY e.department_id ), d.department_name, d.manager_id, ( SELECT MAX ( p.price ) FROM products p WHERE p.department_id = e.department_id ) FROM employees e JOIN departments d ON e.department_id = d.id WHERE e.hire_date > SYSDATE - INTERVAL ? YEAR ) SELECT cv. *, sc.total_sales, sc.sales_rank FROM complex_view cv LEFT JOIN sales_cte sc ON cv.department_id = sc.product_id WHERE cv.avg_dept_test_amt > ( SELECT AVG ( total_sal ) FROM ( SELECT department_id, SUM ( test_amt ) FROM employees GROUP BY department_id ) ) AND EXISTS ( SELECT ? FROM customer_orders co WHERE co.employee_id = cv.employee_id AND co.order_status = ? ) ORDER BY cv.department_id, cv.test_amt DESC",
+        "statement_metadata": {
+          "size": 79,
+          "tables": ["sales", "sales_cte", "products", "employees", "departments", "complex_view", "customer_orders"],
+          "commands": ["SELECT", "JOIN"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/complex/extremely-complex-stored-procedure.json

@@ -0,0 +1,16 @@
+{
+    "input": "CREATE OR REPLACE PROCEDURE /* test comments \n\tsecond line \n*/ complex_data_audit AS CURSOR emp_cursor IS SELECT employee_id FROM employees; /* another comment */ v_employee_id employees.employee_id%TYPE; BEGIN FOR emp_record IN emp_cursor LOOP v_employee_id := emp_record.employee_id; INSERT INTO audit_log (message) VALUES ('Auditing employee with ID: ' || v_employee_id); FOR c IN (SELECT * FROM customer_orders WHERE employee_id = v_employee_id) LOOP IF c.order_status = 'Pending' THEN UPDATE customer_orders SET order_status = 'Under Review' WHERE order_id = c.order_id; ELSE INSERT INTO audit_log (message) VALUES ('Order ' || c.order_id || ' already processed'); END IF; END LOOP; END LOOP; EXCEPTION WHEN OTHERS THEN RAISE_APPLICATION_ERROR(-20002, 'Error in complex_data_audit'); END complex_data_audit;",
+    "outputs": [
+      {
+        "expected": "CREATE OR REPLACE PROCEDURE complex_data_audit emp_cursor IS SELECT employee_id FROM employees; v_employee_id employees.employee_id % TYPE; BEGIN FOR emp_record IN emp_cursor LOOP v_employee_id := emp_record.employee_id; INSERT INTO audit_log ( message ) VALUES ( ? || v_employee_id ); FOR c IN ( SELECT * FROM customer_orders WHERE employee_id = v_employee_id ) LOOP IF c.order_status = ? THEN UPDATE customer_orders SET order_status = ? WHERE order_id = c.order_id; ELSE INSERT INTO audit_log ( message ) VALUES ( ? || c.order_id || ? ); END IF; END LOOP; END LOOP; EXCEPTION WHEN OTHERS THEN RAISE_APPLICATION_ERROR ( ? ); END complex_data_audit",
+        "statement_metadata": {
+          "size": 135,
+          "tables": ["employees", "audit_log", "customer_orders"],
+          "commands": ["CREATE", "SELECT", "BEGIN", "INSERT", "UPDATE"],
+          "comments": ["/* test comments \n\tsecond line \n*/", "/* another comment */"],
+          "procedures": ["complex_data_audit"]
+        }
+      }
+    ]
+  }
+

testdata/oracle/complex/plsql-blocks.json

@@ -0,0 +1,16 @@
+{
+    "input": "DECLARE x NUMBER; BEGIN SELECT COUNT(*) INTO x FROM employees; DBMS_OUTPUT.PUT_LINE('Count: ' || x); END;",
+    "outputs": [
+      {
+        "expected": "DECLARE x NUMBER; BEGIN SELECT COUNT ( * ) INTO x FROM employees; DBMS_OUTPUT.PUT_LINE ( ? || x ); END",
+        "statement_metadata": {
+          "size": 21,
+          "tables": ["x", "employees"],
+          "commands": ["BEGIN", "SELECT"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/complex/super-complex-oracle-query.json

@@ -0,0 +1,16 @@
+{
+    "input": "WITH ranked_sales AS (SELECT product_id, SUM(amount) AS total_sales, RANK() OVER (ORDER BY SUM(amount) DESC) sales_rank FROM sales GROUP BY product_id), dept_costs AS (SELECT department_id, SUM(test_amt) AS total_sal FROM employees GROUP BY department_id), latest_transactions AS (SELECT t.account_id, t.amount, ROW_NUMBER() OVER (PARTITION BY t.account_id ORDER BY t.transaction_date DESC) rn FROM transactions t WHERE t.transaction_date >= ADD_MONTHS(SYSDATE, -6)) SELECT e.employee_id, e.last_name, e.test_amt, d.department_name, d.location_id, rs.total_sales, rs.sales_rank, lt.amount AS latest_transaction_amount FROM employees e INNER JOIN departments d ON e.department_id = d.id LEFT JOIN ranked_sales rs ON e.product_id = rs.product_id LEFT JOIN latest_transactions lt ON e.account_id = lt.account_id AND lt.rn = 1 WHERE e.hire_date > '2010-01-01' AND (d.budget > (SELECT AVG(total_sal) FROM dept_costs) OR e.test_amt > (SELECT AVG(test_amt) FROM employees WHERE department_id = e.department_id)) AND EXISTS (SELECT 1 FROM customer_orders co WHERE co.employee_id = e.employee_id AND co.order_status = 'Completed') ORDER BY e.department_id, e.test_amt DESC;",
+    "outputs": [
+      {
+        "expected": "WITH ranked_sales AS ( SELECT product_id, SUM ( amount ), RANK ( ) OVER ( ORDER BY SUM ( amount ) DESC ) sales_rank FROM sales GROUP BY product_id ), dept_costs AS ( SELECT department_id, SUM ( test_amt ) FROM employees GROUP BY department_id ), latest_transactions AS ( SELECT t.account_id, t.amount, ROW_NUMBER ( ) OVER ( PARTITION BY t.account_id ORDER BY t.transaction_date DESC ) rn FROM transactions t WHERE t.transaction_date >= ADD_MONTHS ( SYSDATE, ? ) ) SELECT e.employee_id, e.last_name, e.test_amt, d.department_name, d.location_id, rs.total_sales, rs.sales_rank, lt.amount FROM employees e INNER JOIN departments d ON e.department_id = d.id LEFT JOIN ranked_sales rs ON e.product_id = rs.product_id LEFT JOIN latest_transactions lt ON e.account_id = lt.account_id AND lt.rn = ? WHERE e.hire_date > ? AND ( d.budget > ( SELECT AVG ( total_sal ) FROM dept_costs ) OR e.test_amt > ( SELECT AVG ( test_amt ) FROM employees WHERE department_id = e.department_id ) ) AND EXISTS ( SELECT ? FROM customer_orders co WHERE co.employee_id = e.employee_id AND co.order_status = ? ) ORDER BY e.department_id, e.test_amt DESC",
+        "statement_metadata": {
+          "size": 103,
+          "tables": ["sales", "employees", "transactions", "departments", "ranked_sales", "latest_transactions", "dept_costs", "customer_orders"],
+          "commands": ["SELECT", "JOIN"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/conditional-delete-with-case.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM user_notifications WHERE id IN (SELECT id FROM notifications WHERE recipient_id = 123 AND status = CASE WHEN urgency = 'High' THEN 'Unread' ELSE 'Read' END);",
+    "outputs": [
+      {
+        "expected": "DELETE FROM user_notifications WHERE id IN ( SELECT id FROM notifications WHERE recipient_id = ? AND status = CASE WHEN urgency = ? THEN ? ELSE ? END )",
+        "statement_metadata": {
+          "size": 43,
+          "tables": ["user_notifications", "notifications"],
+          "commands": ["DELETE", "SELECT"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-basic.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM customers WHERE last_purchase_date < ADD_MONTHS(SYSDATE, -12);",
+    "outputs": [
+      {
+        "expected": "DELETE FROM customers WHERE last_purchase_date < ADD_MONTHS ( SYSDATE, ? )",
+        "statement_metadata": {
+          "size": 15,
+          "tables": ["customers"],
+          "commands": ["DELETE"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-cascade.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM orders WHERE customer_id = 456 CASCADE;",
+    "outputs": [
+      {
+        "expected": "DELETE FROM orders WHERE customer_id = ? CASCADE",
+        "statement_metadata": {
+          "size": 12,
+          "tables": ["orders"],
+          "commands": ["DELETE"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-using-rowid.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM employees WHERE rowid = (SELECT max(rowid) FROM employees WHERE department_id = 20);",
+    "outputs": [
+      {
+        "expected": "DELETE FROM employees WHERE rowid = ( SELECT max ( rowid ) FROM employees WHERE department_id = ? )",
+        "statement_metadata": {
+          "size": 21,
+          "tables": ["employees"],
+          "commands": ["DELETE", "SELECT"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-where-current-of.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM employees WHERE CURRENT OF emp_cursor;",
+    "outputs": [
+      {
+        "expected": "DELETE FROM employees WHERE CURRENT OF emp_cursor",
+        "statement_metadata": {
+          "size": 15,
+          "tables": ["employees"],
+          "commands": ["DELETE"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-with-complex-subqueries.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM products WHERE id IN (SELECT p.id FROM products p JOIN inventory i ON p.id = i.product_id WHERE i.quantity = 0);",
+    "outputs": [
+      {
+        "expected": "DELETE FROM products WHERE id IN ( SELECT p.id FROM products p JOIN inventory i ON p.id = i.product_id WHERE i.quantity = ? )",
+        "statement_metadata": {
+          "size": 33,
+          "tables": ["products", "inventory"],
+          "commands": ["DELETE", "SELECT", "JOIN"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-with-flashback-query.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM orders AS OF TIMESTAMP TO_TIMESTAMP('2023-03-15 08:30:00', 'YYYY-MM-DD HH24:MI:SS') WHERE order_date < '2023-01-01';",
+    "outputs": [
+      {
+        "expected": "DELETE FROM orders AS OF TIMESTAMP TO_TIMESTAMP ( ? ) WHERE order_date < ?",
+        "statement_metadata": {
+          "size": 12,
+          "tables": ["orders"],
+          "commands": ["DELETE"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-with-join-syntax.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM orders o WHERE EXISTS (SELECT 1 FROM customers c WHERE o.customer_id = c.id AND c.status = 'Inactive');",
+    "outputs": [
+      {
+        "expected": "DELETE FROM orders o WHERE EXISTS ( SELECT ? FROM customers c WHERE o.customer_id = c.id AND c.status = ? )",
+        "statement_metadata": {
+          "size": 27,
+          "tables": ["orders", "customers"],
+          "commands": ["DELETE", "SELECT"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-with-pseudocolumns.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM session_logs WHERE ROWNUM <= 10;",
+    "outputs": [
+      {
+        "expected": "DELETE FROM session_logs WHERE ROWNUM <= ?",
+        "statement_metadata": {
+          "size": 18,
+          "tables": ["session_logs"],
+          "commands": ["DELETE"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-with-returning-clause.json

@@ -0,0 +1,16 @@
+{
+    "input": "DELETE FROM logs WHERE entry_date < SYSDATE RETURNING id INTO :deleted_ids;",
+    "outputs": [
+      {
+        "expected": "DELETE FROM logs WHERE entry_date < SYSDATE RETURNING id INTO :deleted_ids",
+        "statement_metadata": {
+          "size": 10,
+          "tables": ["logs"],
+          "commands": ["DELETE"],
+          "comments": [],
+          "procedures": []
+        }
+      }
+    ]
+  }
+

testdata/oracle/delete/delete-with-subquery.json

@@ -0,0 +1,23 @@
+{
+    "input": "DELETE FROM logs WHERE entry_date < (SELECT MIN(order_date) FROM orders);",
+    "outputs": [
+      {
+        "expected": "DELETE FROM logs WHERE entry_date < ( SELECT MIN ( order_date ) FROM orders )",
+        "statement_metadata": {
+          "size": 22,
+          "tables": ["logs", "orders"],
+          "commands": ["DELETE", "SELECT"],
+          "comments": [],
+          "procedures": []
+        }
+      },
+      {
+        "expected": "DELETE FROM logs WHERE entry_date < (SELECT MIN(order_date) FROM orders);",
+        "normalizer_config": {
+            "keep_trailing_semicolon": true,
+            "remove_space_between_parentheses": true
+        }
+      }
+    ]
+  }
+