Skip to content

Commit

Permalink
address review comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@alex-dukhno
alex-dukhno committed Sep 27, 2020
1 parent 0df8011 commit 1f2b8d3
Show file tree
Hide file tree
Showing 6 changed files with 83 additions and 184 deletions.
10 changes: 3 additions & 7 deletions src/ast/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,11 +191,7 @@ pub enum Expr {
right: Box<Expr>,
},
/// Unary operation e.g. `NOT foo`
UnaryOp {
op: UnaryOperator,
expr: Box<Expr>,
infix: bool,
},
UnaryOp { op: UnaryOperator, expr: Box<Expr> },
/// CAST an expression to a different data type e.g. `CAST(foo AS VARCHAR(123))`
Cast {
expr: Box<Expr>,
Expand Down Expand Up @@ -286,8 +282,8 @@ impl fmt::Display for Expr {
high
),
Expr::BinaryOp { left, op, right } => write!(f, "{} {} {}", left, op, right),
Expr::UnaryOp { op, expr, infix } => {
if *infix {
Expr::UnaryOp { op, expr } => {
if op == &UnaryOperator::PGPostfixFactorial {
write!(f, "{}{}", expr, op)
} else {
write!(f, "{} {}", op, expr)
Expand Down
22 changes: 14 additions & 8 deletions src/ast/operator.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,17 @@ pub enum UnaryOperator {
Plus,
Minus,
Not,
/// Bitwise Not, e.g. `~9` (PostgreSQL-specific)
PGBitwiseNot,
PGSqrt,
PGCbrt,
PGFactorial,
PGInfixFactorial,
/// Square root, e.g. `|/9` (PostgreSQL-specific)
PGSquareRoot,
/// Cube root, e.g. `||/27` (PostgreSQL-specific)
PGCubeRoot,
/// Factorial, e.g. `9!` (PostgreSQL-specific)
PGPostfixFactorial,
/// Factorial, e.g. `!!9` (PostgreSQL-specific)
PGPrefixFactorial,
/// Absolute value, e.g. `@ -9` (PostgreSQL-specific)
PGAbs,
}

Expand All @@ -36,10 +42,10 @@ impl fmt::Display for UnaryOperator {
UnaryOperator::Minus => "-",
UnaryOperator::Not => "NOT",
UnaryOperator::PGBitwiseNot => "~",
UnaryOperator::PGSqrt => "|/",
UnaryOperator::PGCbrt => "||/",
UnaryOperator::PGFactorial => "!",
UnaryOperator::PGInfixFactorial => "!!",
UnaryOperator::PGSquareRoot => "|/",
UnaryOperator::PGCubeRoot => "||/",
UnaryOperator::PGPostfixFactorial => "!",
UnaryOperator::PGPrefixFactorial => "!!",
UnaryOperator::PGAbs => "@",
})
}
Expand Down
71 changes: 34 additions & 37 deletions src/parser.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,6 @@ impl<'a> Parser<'a> {
Keyword::NOT => Ok(Expr::UnaryOp {
op: UnaryOperator::Not,
expr: Box::new(self.parse_subexpr(Self::UNARY_NOT_PREC)?),
infix: false,
}),
// Here `w` is a word, check if it's a part of a multi-part
// identifier, a function call, or a simple identifier:
Expand Down Expand Up @@ -284,41 +283,34 @@ impl<'a> Parser<'a> {
},
}, // End of Token::Word
Token::Mult => Ok(Expr::Wildcard),
Token::Tilde => Ok(Expr::UnaryOp {
op: UnaryOperator::PGBitwiseNot,
expr: Box::new(self.parse_subexpr(0)?),
infix: false,
}),
Token::DoubleExclamationMark => Ok(Expr::UnaryOp {
op: UnaryOperator::PGInfixFactorial,
expr: Box::new(self.parse_subexpr(0)?),
infix: false,
}),
Token::SquareRoot => Ok(Expr::UnaryOp {
op: UnaryOperator::PGSqrt,
expr: Box::new(self.parse_subexpr(0)?),
infix: false,
}),
Token::CubeRoot => Ok(Expr::UnaryOp {
op: UnaryOperator::PGCbrt,
expr: Box::new(self.parse_subexpr(0)?),
infix: false,
}),
Token::Ampersat => Ok(Expr::UnaryOp {
op: UnaryOperator::PGAbs,
expr: Box::new(self.parse_subexpr(0)?),
infix: false,
}),
tok @ Token::Minus | tok @ Token::Plus => {
let op = if tok == Token::Plus {
UnaryOperator::Plus
} else {
UnaryOperator::Minus
tok @ Token::Plus
| tok @ Token::Minus
| tok @ Token::DoubleExclamationMark
| tok @ Token::PGSquareRoot
| tok @ Token::PGCubeRoot
| tok @ Token::AtSign
| tok @ Token::Tilde => {
let op = match tok {
Token::Plus => UnaryOperator::Plus,
Token::Minus => UnaryOperator::Minus,
Token::DoubleExclamationMark if dialect_of!(self is PostgreSqlDialect) => {
UnaryOperator::PGPrefixFactorial
}
Token::PGSquareRoot if dialect_of!(self is PostgreSqlDialect) => {
UnaryOperator::PGSquareRoot
}
Token::PGCubeRoot if dialect_of!(self is PostgreSqlDialect) => {
UnaryOperator::PGCubeRoot
}
Token::AtSign if dialect_of!(self is PostgreSqlDialect) => UnaryOperator::PGAbs,
Token::Tilde if dialect_of!(self is PostgreSqlDialect) => {
UnaryOperator::PGBitwiseNot
}
_ => unreachable!(),
};
Ok(Expr::UnaryOp {
op,
expr: Box::new(self.parse_subexpr(Self::PLUS_MINUS_PREC)?),
infix: false,
})
}
Token::Number(_)
Expand Down Expand Up @@ -685,9 +677,15 @@ impl<'a> Parser<'a> {
Token::Caret => Some(BinaryOperator::BitwiseXor),
Token::Ampersand => Some(BinaryOperator::BitwiseAnd),
Token::Div => Some(BinaryOperator::Divide),
Token::ShiftLeft => Some(BinaryOperator::PGBitwiseShiftLeft),
Token::ShiftRight => Some(BinaryOperator::PGBitwiseShiftRight),
Token::Sharp => Some(BinaryOperator::PGBitwiseXor),
Token::ShiftLeft if dialect_of!(self is PostgreSqlDialect) => {
Some(BinaryOperator::PGBitwiseShiftLeft)
}
Token::ShiftRight if dialect_of!(self is PostgreSqlDialect) => {
Some(BinaryOperator::PGBitwiseShiftRight)
}
Token::Sharp if dialect_of!(self is PostgreSqlDialect) => {
Some(BinaryOperator::PGBitwiseXor)
}
Token::Word(w) => match w.keyword {
Keyword::AND => Some(BinaryOperator::And),
Keyword::OR => Some(BinaryOperator::Or),
Expand Down Expand Up @@ -740,9 +738,8 @@ impl<'a> Parser<'a> {
} else if Token::ExclamationMark == tok {
// PostgreSQL factorial operation
Ok(Expr::UnaryOp {
op: UnaryOperator::PGFactorial,
op: UnaryOperator::PGPostfixFactorial,
expr: Box::new(expr),
infix: true,
})
} else {
// Can only happen if `get_next_precedence` got out of sync with this function
Expand Down
119 changes: 22 additions & 97 deletions src/tokenizer.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ use std::str::Chars;

use super::dialect::keywords::{Keyword, ALL_KEYWORDS, ALL_KEYWORDS_INDEX};
use super::dialect::Dialect;
use super::dialect::PostgreSqlDialect;
use super::dialect::SnowflakeDialect;
#[cfg(feature = "serde")]
use serde::{Deserialize, Serialize};
Expand Down Expand Up @@ -103,24 +102,24 @@ pub enum Token {
RBrace,
/// Right Arrow `=>`
RArrow,
/// Sharp `#` use for PostgreSQL Bitwise XOR operator
/// Sharp `#` used for PostgreSQL Bitwise XOR operator
Sharp,
/// Tilde `~` use for PostgreSQL Bitwise NOT operator
/// Tilde `~` used for PostgreSQL Bitwise NOT operator
Tilde,
/// Bitwise left operator `<<` use for PostgreSQL
/// `<<`, a bitwise shift left operator in PostgreSQL
ShiftLeft,
/// Bitwise right operator `>>` use for PostgreSQL
/// `>>`, a bitwise shift right operator in PostgreSQL
ShiftRight,
/// Exclamation Mark `!` use for PostgreSQL factorial operator
/// Exclamation Mark `!` used for PostgreSQL factorial operator
ExclamationMark,
/// Exclamation Mark `!!` use for PostgreSQL prefix factorial operator
/// Double Exclamation Mark `!!` used for PostgreSQL prefix factorial operator
DoubleExclamationMark,
/// Ampersat `@` use for PostgreSQL abs operator
Ampersat,
/// PostgreSQL square root math operator
SquareRoot,
/// PostgreSQL cube root math operator
CubeRoot,
/// AtSign `@` used for PostgreSQL abs operator
AtSign,
/// `|/`, a square root math operator in PostgreSQL
PGSquareRoot,
/// `||/` , a cube root math operator in PostgreSQL
PGCubeRoot,
}

impl fmt::Display for Token {
Expand Down Expand Up @@ -166,11 +165,11 @@ impl fmt::Display for Token {
Token::ExclamationMark => f.write_str("!"),
Token::DoubleExclamationMark => f.write_str("!!"),
Token::Tilde => f.write_str("~"),
Token::Ampersat => f.write_str("@"),
Token::AtSign => f.write_str("@"),
Token::ShiftLeft => f.write_str("<<"),
Token::ShiftRight => f.write_str(">>"),
Token::SquareRoot => f.write_str("|/"),
Token::CubeRoot => f.write_str("||/"),
Token::PGSquareRoot => f.write_str("|/"),
Token::PGCubeRoot => f.write_str("||/"),
}
}
}
Expand Down Expand Up @@ -434,15 +433,11 @@ impl<'a> Tokenizer<'a> {
'|' => {
chars.next(); // consume the '|'
match chars.peek() {
Some('/') if dialect_of!(self is PostgreSqlDialect) => {
self.consume_and_return(chars, Token::SquareRoot)
}
Some('/') => self.consume_and_return(chars, Token::PGSquareRoot),
Some('|') => {
chars.next(); // consume the second '|'
match chars.peek() {
Some('/') if dialect_of!(self is PostgreSqlDialect) => {
self.consume_and_return(chars, Token::CubeRoot)
}
Some('/') => self.consume_and_return(chars, Token::PGCubeRoot),
_ => Ok(Some(Token::StringConcat)),
}
}
Expand All @@ -462,9 +457,7 @@ impl<'a> Tokenizer<'a> {
chars.next(); // consume
match chars.peek() {
Some('=') => self.consume_and_return(chars, Token::Neq),
Some('!') if dialect_of!(self is PostgreSqlDialect) => {
self.consume_and_return(chars, Token::DoubleExclamationMark)
}
Some('!') => self.consume_and_return(chars, Token::DoubleExclamationMark),
_ => Ok(Some(Token::ExclamationMark)),
}
}
Expand All @@ -473,9 +466,7 @@ impl<'a> Tokenizer<'a> {
match chars.peek() {
Some('=') => self.consume_and_return(chars, Token::LtEq),
Some('>') => self.consume_and_return(chars, Token::Neq),
Some('<') if dialect_of!(self is PostgreSqlDialect) => {
self.consume_and_return(chars, Token::ShiftLeft)
}
Some('<') => self.consume_and_return(chars, Token::ShiftLeft),
_ => Ok(Some(Token::Lt)),
}
}
Expand Down Expand Up @@ -510,15 +501,9 @@ impl<'a> Tokenizer<'a> {
comment,
})))
}
'~' if dialect_of!(self is PostgreSqlDialect) => {
self.consume_and_return(chars, Token::Tilde)
}
'#' if dialect_of!(self is PostgreSqlDialect) => {
self.consume_and_return(chars, Token::Sharp)
}
'@' if dialect_of!(self is PostgreSqlDialect) => {
self.consume_and_return(chars, Token::Ampersat)
}
'~' => self.consume_and_return(chars, Token::Tilde),
'#' => self.consume_and_return(chars, Token::Sharp),
'@' => self.consume_and_return(chars, Token::AtSign),
other => self.consume_and_return(chars, Token::Char(other)),
},
None => Ok(None),
Expand Down Expand Up @@ -641,7 +626,6 @@ mod tests {
use super::super::dialect::GenericDialect;
use super::super::dialect::MsSqlDialect;
use super::*;
use crate::dialect::PostgreSqlDialect;

#[test]
fn tokenize_select_1() {
Expand Down Expand Up @@ -1014,65 +998,6 @@ mod tests {
compare(expected, tokens);
}

#[test]
fn tokenize_postgresql_bitwise_operations() {
let sql = String::from("SELECT ~one << two # three >> four");
let dialect = PostgreSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap();

let expected = vec![
Token::make_keyword("SELECT"),
Token::Whitespace(Whitespace::Space),
Token::Tilde,
Token::make_word("one", None),
Token::Whitespace(Whitespace::Space),
Token::ShiftLeft,
Token::Whitespace(Whitespace::Space),
Token::make_word("two", None),
Token::Whitespace(Whitespace::Space),
Token::Sharp,
Token::Whitespace(Whitespace::Space),
Token::make_word("three", None),
Token::Whitespace(Whitespace::Space),
Token::ShiftRight,
Token::Whitespace(Whitespace::Space),
Token::make_word("four", None),
];

compare(expected, tokens);
}

#[test]
fn tokenize_postgresql_math_operations() {
let sql = String::from("SELECT !!5 5! @-6 |/4 ||/8");
let dialect = PostgreSqlDialect {};
let mut tokenizer = Tokenizer::new(&dialect, &sql);
let tokens = tokenizer.tokenize().unwrap();

let expected = vec![
Token::make_keyword("SELECT"),
Token::Whitespace(Whitespace::Space),
Token::DoubleExclamationMark,
Token::Number("5".to_string()),
Token::Whitespace(Whitespace::Space),
Token::Number("5".to_string()),
Token::ExclamationMark,
Token::Whitespace(Whitespace::Space),
Token::Ampersat,
Token::Minus,
Token::Number("6".to_string()),
Token::Whitespace(Whitespace::Space),
Token::SquareRoot,
Token::Number("4".to_string()),
Token::Whitespace(Whitespace::Space),
Token::CubeRoot,
Token::Number("8".to_string()),
];

compare(expected, tokens);
}

fn compare(expected: Vec<Token>, actual: Vec<Token>) {
//println!("------------------------------");
//println!("tokens = {:?}", actual);
Expand Down
Loading

0 comments on commit 1f2b8d3

Please sign in to comment.