From 68f36e2a762ea14eb0df72535e4adf27951f83ef Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Thu, 1 Feb 2024 16:01:57 -0500 Subject: [PATCH 1/6] UNLOAD, SET TBLPROPERTIES, SERDEPROPERTIES, EXECUTE --- src/ast/ddl.rs | 8 +++ src/ast/mod.rs | 42 +++++++++++++- src/keywords.rs | 2 + src/parser/mod.rs | 60 ++++++++++++++++++-- src/test_utils.rs | 1 + tests/sqlparser_common.rs | 107 ++++++++++++++++++++++++++++++++++++ tests/sqlparser_hive.rs | 2 + tests/sqlparser_postgres.rs | 16 ++++++ 8 files changed, 229 insertions(+), 9 deletions(-) diff --git a/src/ast/ddl.rs b/src/ast/ddl.rs index 7d87d88de2..134fa2e926 100644 --- a/src/ast/ddl.rs +++ b/src/ast/ddl.rs @@ -143,6 +143,7 @@ pub enum AlterTableOperation { /// /// Note: this is Snowflake specific SwapWith { table_name: ObjectName }, + SetTblProperties { table_properties: Vec } } #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -283,6 +284,13 @@ impl fmt::Display for AlterTableOperation { AlterTableOperation::SwapWith { table_name } => { write!(f, "SWAP WITH {table_name}") } + AlterTableOperation::SetTblProperties { table_properties } => { + write!( + f, + "SET TBLPROPERTIES({})", + display_comma_separated(table_properties) + ) + } } } } diff --git a/src/ast/mod.rs b/src/ast/mod.rs index a2c28c810e..a520c76059 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2158,11 +2158,11 @@ pub enum Statement { /// Note: this is a PostgreSQL-specific statement. Deallocate { name: Ident, prepare: bool }, /// ```sql - /// EXECUTE name [ ( parameter [, ...] ) ] + /// EXECUTE name [ ( parameter [, ...] ) ] [USING ] /// ``` /// /// Note: this is a PostgreSQL-specific statement. - Execute { name: Ident, parameters: Vec }, + Execute { name: Ident, parameters: Vec, using: Option }, /// ```sql /// PREPARE name [ ( data_type [, ...] ) ] AS statement /// ``` @@ -2301,6 +2301,14 @@ pub enum Statement { /// ``` /// Note: this is a MySQL-specific statement. See UnlockTables, + /// ```sql + /// UNLOAD(statement) TO [ WITH options ] + /// ``` + Unload { + query: Box, + to: Ident, + with: Vec + } } impl fmt::Display for Statement { @@ -2969,6 +2977,7 @@ impl fmt::Display for Statement { if let Some(HiveFormat { row_format, + serde_properties, storage, location, }) = hive_formats @@ -2993,6 +3002,20 @@ impl fmt::Display for Statement { } _ => (), } + if let Some(options) = options.as_ref() { + write!( + f, + " OPTIONS({})", + display_comma_separated(options.as_slice()) + )?; + } + if let Some(serde_properties) = serde_properties.as_ref() { + write!( + f, + " WITH SERDEPROPERTIES ({})", + display_comma_separated(serde_properties) + )?; + } if !*external { if let Some(loc) = location { write!(f, " LOCATION '{loc}'")?; @@ -3574,11 +3597,14 @@ impl fmt::Display for Statement { prepare = if *prepare { "PREPARE " } else { "" }, name = name, ), - Statement::Execute { name, parameters } => { + Statement::Execute { name, parameters, using } => { write!(f, "EXECUTE {name}")?; if !parameters.is_empty() { write!(f, "({})", display_comma_separated(parameters))?; } + if let Some(using) = using { + write!(f, " USING {using}")?; + }; Ok(()) } Statement::Prepare { @@ -3820,6 +3846,15 @@ impl fmt::Display for Statement { Statement::UnlockTables => { write!(f, "UNLOCK TABLES") } + Statement::Unload { query, to, with } => { + write!(f, "UNLOAD({query}) TO {to}")?; + + if with.len() > 0 { + write!(f, " WITH ({})", display_comma_separated(with))?; + } + + Ok(()) + } } } } @@ -4588,6 +4623,7 @@ pub enum HiveIOFormat { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct HiveFormat { pub row_format: Option, + pub serde_properties: Option>, pub storage: Option, pub location: Option, } diff --git a/src/keywords.rs b/src/keywords.rs index f14b92b76b..e75f08b64e 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -592,6 +592,7 @@ define_keywords!( SEQUENCEFILE, SEQUENCES, SERDE, + SERDEPROPERTIES, SERIALIZABLE, SESSION, SESSION_USER, @@ -685,6 +686,7 @@ define_keywords!( UNION, UNIQUE, UNKNOWN, + UNLOAD, UNLOCK, UNLOGGED, UNNEST, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 36ac2fd28c..41feb49417 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -21,6 +21,7 @@ use alloc::{ vec::Vec, }; use core::fmt; +use std::sync::atomic::spin_loop_hint; use log::debug; @@ -516,6 +517,7 @@ impl<'a> Parser<'a> { Keyword::MERGE => Ok(self.parse_merge()?), // `PRAGMA` is sqlite specific https://www.sqlite.org/pragma.html Keyword::PRAGMA => Ok(self.parse_pragma()?), + Keyword::UNLOAD => Ok(self.parse_unload()?), _ => self.expected("an SQL statement", next_token), }, Token::LParen => { @@ -4091,7 +4093,7 @@ impl<'a> Parser<'a> { pub fn parse_hive_formats(&mut self) -> Result { let mut hive_format = HiveFormat::default(); loop { - match self.parse_one_of_keywords(&[Keyword::ROW, Keyword::STORED, Keyword::LOCATION]) { + match self.parse_one_of_keywords(&[Keyword::ROW, Keyword::STORED, Keyword::LOCATION, Keyword::WITH]) { Some(Keyword::ROW) => { hive_format.row_format = Some(self.parse_row_format()?); } @@ -4113,6 +4115,12 @@ impl<'a> Parser<'a> { Some(Keyword::LOCATION) => { hive_format.location = Some(self.parse_literal_string()?); } + Some(Keyword::WITH) => { + let properties = self.parse_options(Keyword::SERDEPROPERTIES)?; + if properties.len() > 0 { + hive_format.serde_properties = Some(properties); + } + } None => break, _ => break, } @@ -4862,6 +4870,17 @@ impl<'a> Parser<'a> { } } + pub fn parse_options_with_keywords(&mut self, keywords: &[Keyword]) -> Result, ParserError> { + if self.parse_keywords(keywords) { + self.expect_token(&Token::LParen)?; + let options = self.parse_comma_separated(Parser::parse_sql_option)?; + self.expect_token(&Token::RParen)?; + Ok(options) + } else { + Ok(vec![]) + } + } + pub fn parse_index_type(&mut self) -> Result { if self.parse_keyword(Keyword::BTREE) { Ok(IndexType::BTree) @@ -5123,10 +5142,15 @@ impl<'a> Parser<'a> { let table_name = self.parse_object_name(false)?; AlterTableOperation::SwapWith { table_name } } else { - return self.expected( - "ADD, RENAME, PARTITION, SWAP or DROP after ALTER TABLE", - self.peek_token(), - ); + let options: Vec = self.parse_options_with_keywords(&[Keyword::SET, Keyword::TBLPROPERTIES])?; + if options.len() > 0 { + AlterTableOperation::SetTblProperties { table_properties: options } + } else { + return self.expected( + "ADD, RENAME, PARTITION, SWAP, DROP, or SET TBLPROPERTIES after ALTER TABLE", + self.peek_token(), + ); + } }; Ok(operation) } @@ -5144,6 +5168,7 @@ impl<'a> Parser<'a> { let if_exists = self.parse_keywords(&[Keyword::IF, Keyword::EXISTS]); let only = self.parse_keyword(Keyword::ONLY); // [ ONLY ] let table_name = self.parse_object_name(false)?; + let operations = self.parse_comma_separated(Parser::parse_alter_table_operation)?; Ok(Statement::AlterTable { name: table_name, @@ -8600,7 +8625,13 @@ impl<'a> Parser<'a> { self.expect_token(&Token::RParen)?; } - Ok(Statement::Execute { name, parameters }) + let using = if self.parse_keyword(Keyword::USING) { + Some(self.parse_expr()?) + } else { + None + }; + + Ok(Statement::Execute { name, parameters, using }) } pub fn parse_prepare(&mut self) -> Result { @@ -8732,6 +8763,23 @@ impl<'a> Parser<'a> { } } + pub fn parse_unload(&mut self) -> Result { + self.expect_token(&Token::LParen)?; + let query = self.parse_query()?; + self.expect_token(&Token::RParen)?; + + self.expect_keyword(Keyword::TO)?; + let to = self.parse_identifier(false)?; + + let with_options = self.parse_options(Keyword::WITH)?; + + Ok(Statement::Unload { + query: Box::new(query), + to: to, + with: with_options, + }) + } + // PRAGMA [schema-name '.'] pragma-name [('=' pragma-value) | '(' pragma-value ')'] pub fn parse_pragma(&mut self) -> Result { let name = self.parse_object_name(false)?; diff --git a/src/test_utils.rs b/src/test_utils.rs index 26cfec4636..1147cfcef2 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -251,6 +251,7 @@ pub fn alter_table_op_with_name(stmt: Statement, expected_name: &str) -> AlterTa _ => panic!("Expected ALTER TABLE statement"), } } + pub fn alter_table_op(stmt: Statement) -> AlterTableOperation { alter_table_op_with_name(stmt, "tab") } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index fcacb34596..6d2c8abb52 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -42,6 +42,8 @@ mod test_utils; #[cfg(test)] use pretty_assertions::assert_eq; +use sqlparser::ast::Value::RawStringLiteral; +use sqlparser::tokenizer::Token::{DollarQuotedString, SingleQuotedString}; #[test] fn parse_insert_values() { @@ -3497,6 +3499,23 @@ fn parse_alter_table() { } _ => unreachable!(), } + + let set_table_properties = "ALTER TABLE tab SET TBLPROPERTIES('classification' = 'parquet')"; + match alter_table_op(verified_stmt(set_table_properties)) { + AlterTableOperation::SetTblProperties { + table_properties, + } => { + assert_eq!(table_properties, [ + SqlOption { + name: Ident { value: "classification".to_string(), quote_style: Some('\'') }, + value: Expr::Value(Value::SingleQuotedString( + "parquet".to_string() + )), + } + ]); + } + _ => unreachable!(), + } } #[test] @@ -8351,6 +8370,94 @@ fn parse_binary_operators_without_whitespace() { ); } +#[test] +fn temp() { + let dialects: Vec> = vec![ + Box::new(GenericDialect {}), + Box::new(BigQueryDialect {}), + Box::new(HiveDialect {}), + Box::new(RedshiftSqlDialect {}), + Box::new(PostgreSqlDialect {}), + ]; + + //ctas, execute, insert_select, set, unload + + for dialect in dialects { + + let result = Parser::parse_sql(&*dialect, ""); + + match result { + Ok(parsed) => { + print!("YAYYY"); + }, + Err(parse_error) => { + print!("NOOO") + }, + } + }; +} + +#[test] +fn parse_unload() { + let unload = verified_stmt("UNLOAD(SELECT cola FROM tab) TO 's3://...' WITH (format = 'AVRO')"); + assert_eq!( + unload, + Statement::Unload { + query: Box::new(Query { + body: Box::new( + SetExpr::Select(Box::new(Select { + distinct: None, + top: None, + projection: vec![ + UnnamedExpr(Expr::Identifier(Ident::new("cola"))), + ], + into: None, + from: vec![TableWithJoins { + relation: TableFactor::Table { + name: ObjectName(vec![Ident::new("tab")]), + alias: None, + args: None, + with_hints: vec![], + version: None, + partitions: vec![], + }, + joins: vec![], + }], + lateral_views: vec![], + selection: None, + group_by: GroupByExpr::Expressions(vec![]), + cluster_by: vec![], + distribute_by: vec![], + sort_by: vec![], + having: None, + named_window: vec![], + qualify: None + })) + ), + with: None, + limit: None, + limit_by: vec![], + offset: None, + fetch: None, + locks: vec![], + for_clause: None, + order_by: vec![], + }), + to: Ident { + value: "s3://...".to_string(), + quote_style: Some('\'') + }, + with: vec![SqlOption { + name: Ident { + value: "format".to_string(), + quote_style: None + }, + value: Expr::Value(Value::SingleQuotedString("AVRO".to_string())) + }] + } + ); +} + #[test] fn test_savepoint() { match verified_stmt("SAVEPOINT test1") { diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 66eef09e17..658edee73c 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -27,9 +27,11 @@ use sqlparser::test_utils::*; fn parse_table_create() { let sql = r#"CREATE TABLE IF NOT EXISTS db.table (a BIGINT, b STRING, c TIMESTAMP) PARTITIONED BY (d STRING, e TIMESTAMP) STORED AS ORC LOCATION 's3://...' TBLPROPERTIES ("prop" = "2", "asdf" = '1234', 'asdf' = "1234", "asdf" = 2)"#; let iof = r#"CREATE TABLE IF NOT EXISTS db.table (a BIGINT, b STRING, c TIMESTAMP) PARTITIONED BY (d STRING, e TIMESTAMP) STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' LOCATION 's3://...'"#; + let serdeproperties = r#"CREATE EXTERNAL TABLE IF NOT EXISTS db.table (a STRING, b STRING, c STRING) PARTITIONED BY (d STRING, e STRING) ROW FORMAT SERDE 'org.apache.hadoop.hive.serde.config' WITH SERDEPROPERTIES ('prop_a' = 'a', 'prop_b' = 'b') STORED AS TEXTFILE LOCATION 's3://...' TBLPROPERTIES ('prop_c' = 'c')"#; hive().verified_stmt(sql); hive().verified_stmt(iof); + hive().verified_stmt(serdeproperties); } fn generic(options: Option) -> TestedDialects { diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index 45a3dbed26..f2a111745a 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -1420,6 +1420,7 @@ fn parse_execute() { Statement::Execute { name: "a".into(), parameters: vec![], + using: None } ); @@ -1432,6 +1433,21 @@ fn parse_execute() { Expr::Value(number("1")), Expr::Value(Value::SingleQuotedString("t".to_string())) ], + using: None + } + ); + + let stmt = pg_and_generic().verified_stmt("EXECUTE a USING CAST(1337 AS SMALLINT)"); + assert_eq!( + stmt, + Statement::Execute { + name: "a".into(), + parameters: vec![], + using: Some(Expr::Cast { + expr: Box::new(Expr::Value(Value::Number("1337".parse().unwrap(), false))), + data_type: DataType::SmallInt(None), + format: None + }) } ); } From b1f1d28bdcea6f2c87a113b964c3c31445ed4057 Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Thu, 1 Feb 2024 16:33:03 -0500 Subject: [PATCH 2/6] removed added imports --- src/parser/mod.rs | 1 - tests/sqlparser_common.rs | 29 ----------------------------- 2 files changed, 30 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 41feb49417..5e554c0861 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -21,7 +21,6 @@ use alloc::{ vec::Vec, }; use core::fmt; -use std::sync::atomic::spin_loop_hint; use log::debug; diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 6d2c8abb52..87bcede38f 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -42,8 +42,6 @@ mod test_utils; #[cfg(test)] use pretty_assertions::assert_eq; -use sqlparser::ast::Value::RawStringLiteral; -use sqlparser::tokenizer::Token::{DollarQuotedString, SingleQuotedString}; #[test] fn parse_insert_values() { @@ -8370,33 +8368,6 @@ fn parse_binary_operators_without_whitespace() { ); } -#[test] -fn temp() { - let dialects: Vec> = vec![ - Box::new(GenericDialect {}), - Box::new(BigQueryDialect {}), - Box::new(HiveDialect {}), - Box::new(RedshiftSqlDialect {}), - Box::new(PostgreSqlDialect {}), - ]; - - //ctas, execute, insert_select, set, unload - - for dialect in dialects { - - let result = Parser::parse_sql(&*dialect, ""); - - match result { - Ok(parsed) => { - print!("YAYYY"); - }, - Err(parse_error) => { - print!("NOOO") - }, - } - }; -} - #[test] fn parse_unload() { let unload = verified_stmt("UNLOAD(SELECT cola FROM tab) TO 's3://...' WITH (format = 'AVRO')"); From e918e5084591c2de48202e7490c351ab7706cc9d Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Thu, 1 Feb 2024 17:46:37 -0500 Subject: [PATCH 3/6] fixing bugs --- src/ast/mod.rs | 7 ------- src/parser/mod.rs | 5 ++++- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index a520c76059..c98215d86e 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -3002,13 +3002,6 @@ impl fmt::Display for Statement { } _ => (), } - if let Some(options) = options.as_ref() { - write!( - f, - " OPTIONS({})", - display_comma_separated(options.as_slice()) - )?; - } if let Some(serde_properties) = serde_properties.as_ref() { write!( f, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 5e554c0861..96b0861b59 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4115,9 +4115,12 @@ impl<'a> Parser<'a> { hive_format.location = Some(self.parse_literal_string()?); } Some(Keyword::WITH) => { - let properties = self.parse_options(Keyword::SERDEPROPERTIES)?; + self.prev_token(); + let properties = self.parse_options_with_keywords(&[Keyword::WITH, Keyword::SERDEPROPERTIES])?; if properties.len() > 0 { hive_format.serde_properties = Some(properties); + } else { + break; } } None => break, From c9c2634730745ee029cd3b2a29488c18e942f567 Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Thu, 1 Feb 2024 21:15:30 -0500 Subject: [PATCH 4/6] allow paren select insert into --- src/parser/mod.rs | 11 +++++++++-- tests/sqlparser_common.rs | 3 +++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 96b0861b59..f02abb39a6 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -7978,8 +7978,15 @@ impl<'a> Parser<'a> { let partitioned = self.parse_insert_partition()?; - // Hive allows you to specify columns after partitions as well if you want. - let after_columns = self.parse_parenthesized_column_list(Optional, false)?; + // make sure we are not about to consume a query + let mut after_columns = vec![]; + match self.peek_nth_token(1).token { + Token::Word(w) if w.keyword != Keyword::SELECT => { + // Hive allows you to specify columns after partitions as well if you want. + after_columns = self.parse_parenthesized_column_list(Optional, false)?; + }, + _ => {} + }; let source = Some(Box::new(self.parse_query()?)); diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 87bcede38f..48889b770d 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -106,6 +106,9 @@ fn parse_insert_values() { } verified_stmt("INSERT INTO customer WITH foo AS (SELECT 1) SELECT * FROM foo UNION VALUES (1)"); + + // allow parenthesis query after insert into + verified_stmt("INSERT INTO tbla (cola) (SELECT cola FROM tblb)"); } #[test] From 819fe4a0d77db5c2cbb7f6b91da5327639f2da3a Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Fri, 2 Feb 2024 15:05:57 -0500 Subject: [PATCH 5/6] hive locations, execute cases --- src/ast/mod.rs | 93 ++++++++++++++++++++++++++++++----- src/keywords.rs | 8 +++ src/lib.rs | 1 + src/parser/mod.rs | 97 +++++++++++++++++++++++++++++++++++-- src/test_utils.rs | 1 + tests/sqlparser_hive.rs | 18 +++++++ tests/sqlparser_postgres.rs | 24 ++++++--- 7 files changed, 217 insertions(+), 25 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index c98215d86e..d79601c9f9 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -1747,6 +1747,7 @@ pub enum Statement { if_exists: bool, only: bool, operations: Vec, + location: Option, }, /// ```sql /// ALTER INDEX @@ -2162,7 +2163,7 @@ pub enum Statement { /// ``` /// /// Note: this is a PostgreSQL-specific statement. - Execute { name: Ident, parameters: Vec, using: Option }, + Execute { name: Ident, parameters: Vec, using: Vec }, /// ```sql /// PREPARE name [ ( data_type [, ...] ) ] AS statement /// ``` @@ -2986,7 +2987,12 @@ impl fmt::Display for Statement { Some(HiveRowFormat::SERDE { class }) => { write!(f, " ROW FORMAT SERDE '{class}'")? } - Some(HiveRowFormat::DELIMITED) => write!(f, " ROW FORMAT DELIMITED")?, + Some(HiveRowFormat::DELIMITED { delimiters }) => { + write!(f, " ROW FORMAT DELIMITED")?; + if delimiters.len() > 0 { + write!(f, " {}", display_separated(delimiters, " "))?; + } + }, None => (), } match storage { @@ -3016,12 +3022,10 @@ impl fmt::Display for Statement { } } if *external { - write!( - f, - " STORED AS {} LOCATION '{}'", - file_format.as_ref().unwrap(), - location.as_ref().unwrap() - )?; + if file_format.is_some() { + write!( f, " STORED AS {}", file_format.as_ref().unwrap())?; + } + write!(f, " LOCATION '{}'", location.as_ref().unwrap())?; } if !table_properties.is_empty() { write!( @@ -3271,6 +3275,7 @@ impl fmt::Display for Statement { if_exists, only, operations, + location, } => { write!(f, "ALTER TABLE ")?; if *if_exists { @@ -3283,7 +3288,11 @@ impl fmt::Display for Statement { f, "{name} {operations}", operations = display_comma_separated(operations) - ) + )?; + if let Some(loc) = location { + write!(f, " {loc}")? + } + Ok(()) } Statement::AlterIndex { name, operation } => { write!(f, "ALTER INDEX {name} {operation}") @@ -3595,8 +3604,8 @@ impl fmt::Display for Statement { if !parameters.is_empty() { write!(f, "({})", display_comma_separated(parameters))?; } - if let Some(using) = using { - write!(f, " USING {using}")?; + if using.len() > 0 { + write!(f, " USING {}", display_comma_separated(using))?; }; Ok(()) } @@ -4594,7 +4603,50 @@ pub enum HiveDistributionStyle { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum HiveRowFormat { SERDE { class: String }, - DELIMITED, + DELIMITED { + delimiters: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct HiveRowDelimiter { + pub delimiter: HiveDelimiter, + pub char: Ident +} + +impl fmt::Display for HiveRowDelimiter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} ", self.delimiter)?; + write!(f, "{}", self.char) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum HiveDelimiter { + FieldsTerminatedBy, + FieldsEscapedBy, + CollectionItemsTerminatedBy, + MapKeysTerminatedBy, + LinesTerminatedBy, + NullDefinedAs +} + +impl fmt::Display for HiveDelimiter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use HiveDelimiter::*; + f.write_str(match self { + FieldsTerminatedBy => { "FIELDS TERMINATED BY" } + FieldsEscapedBy => { "ESCAPED BY" } + CollectionItemsTerminatedBy => { "COLLECTION ITEMS TERMINATED BY" } + MapKeysTerminatedBy => { "MAP KEYS TERMINATED BY" } + LinesTerminatedBy => { "LINES TERMINATED BY" } + NullDefinedAs => { "NULL DEFINED AS" } + }) + } } #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -5510,6 +5562,23 @@ impl fmt::Display for LockTableType { } } +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct HiveSetLocation { + pub has_set: bool, + pub location: Ident, +} + +impl fmt::Display for HiveSetLocation { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.has_set { + write!(f, "SET ")?; + } + write!(f, "LOCATION {}", self.location) + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/keywords.rs b/src/keywords.rs index e75f08b64e..d7a99e0b51 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -153,6 +153,7 @@ define_keywords!( COLLATE, COLLATION, COLLECT, + COLLECTION, COLUMN, COLUMNS, COMMENT, @@ -212,6 +213,7 @@ define_keywords!( DEFAULT, DEFERRABLE, DEFERRED, + DEFINED, DELAYED, DELETE, DELIMITED, @@ -258,6 +260,7 @@ define_keywords!( EQUALS, ERROR, ESCAPE, + ESCAPED, EVENT, EVERY, EXCEPT, @@ -366,6 +369,7 @@ define_keywords!( ISOLATION, ISOWEEK, ISOYEAR, + ITEMS, JAR, JOIN, JSON, @@ -374,6 +378,7 @@ define_keywords!( JSON_TABLE, JULIAN, KEY, + KEYS, KILL, LAG, LANGUAGE, @@ -388,6 +393,7 @@ define_keywords!( LIKE, LIKE_REGEX, LIMIT, + LINES, LISTAGG, LN, LOCAL, @@ -402,6 +408,7 @@ define_keywords!( LOW_PRIORITY, MACRO, MANAGEDLOCATION, + MAP, MATCH, MATCHED, MATERIALIZED, @@ -649,6 +656,7 @@ define_keywords!( TBLPROPERTIES, TEMP, TEMPORARY, + TERMINATED, TEXT, TEXTFILE, THEN, diff --git a/src/lib.rs b/src/lib.rs index ba0132ed83..fe5e30a31a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -76,6 +76,7 @@ extern crate alloc; #[macro_use] #[cfg(test)] extern crate pretty_assertions; +extern crate core; pub mod ast; #[macro_use] diff --git a/src/parser/mod.rs b/src/parser/mod.rs index f02abb39a6..51db71b649 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4138,7 +4138,76 @@ impl<'a> Parser<'a> { let class = self.parse_literal_string()?; Ok(HiveRowFormat::SERDE { class }) } - _ => Ok(HiveRowFormat::DELIMITED), + _ => { + let mut row_delimiters = vec![]; + + loop { + match self.parse_one_of_keywords(&[Keyword::FIELDS, Keyword::COLLECTION, Keyword::MAP, Keyword::LINES, Keyword::NULL]) { + Some(Keyword::FIELDS) => { + if self.parse_keywords(&[Keyword::TERMINATED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::FieldsTerminatedBy, + char: self.parse_identifier(false)? + }); + + if self.parse_keywords(&[Keyword::ESCAPED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::FieldsEscapedBy, + char: self.parse_identifier(false)? + }); + } + } else { + break; + } + } + Some(Keyword::COLLECTION) => { + if self.parse_keywords(&[Keyword::ITEMS, Keyword::TERMINATED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::CollectionItemsTerminatedBy, + char: self.parse_identifier(false)? + }); + } else { + break; + } + } + Some(Keyword::MAP) => { + if self.parse_keywords(&[Keyword::KEYS, Keyword::TERMINATED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::MapKeysTerminatedBy, + char: self.parse_identifier(false)? + }); + } else { + break; + } + } + Some(Keyword::LINES) => { + if self.parse_keywords(&[Keyword::TERMINATED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::LinesTerminatedBy, + char: self.parse_identifier(false)? + }); + } else { + break; + } + } + Some(Keyword::NULL) => { + if self.parse_keywords(&[Keyword::DEFINED, Keyword::AS]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::NullDefinedAs, + char: self.parse_identifier(false)? + }); + } else { + break; + } + } + _ => { break; } + } + } + + Ok(HiveRowFormat::DELIMITED { + delimiters: row_delimiters + }) + }, } } @@ -5172,11 +5241,26 @@ impl<'a> Parser<'a> { let table_name = self.parse_object_name(false)?; let operations = self.parse_comma_separated(Parser::parse_alter_table_operation)?; + + let mut location = None; + if self.parse_keyword(Keyword::LOCATION) { + location = Some(HiveSetLocation{ + has_set: false, + location: self.parse_identifier(false)? + }); + } else if self.parse_keywords(&[Keyword::SET, Keyword::LOCATION]) { + location = Some(HiveSetLocation{ + has_set: true, + location: self.parse_identifier(false)? + }); + } + Ok(Statement::AlterTable { name: table_name, if_exists, only, operations, + location }) } Keyword::INDEX => { @@ -8634,10 +8718,13 @@ impl<'a> Parser<'a> { self.expect_token(&Token::RParen)?; } - let using = if self.parse_keyword(Keyword::USING) { - Some(self.parse_expr()?) - } else { - None + let mut using = vec![]; + if self.parse_keyword(Keyword::USING) { + using.push(self.parse_expr()?); + + while self.consume_token(&Token::Comma) { + using.push(self.parse_expr()?); + } }; Ok(Statement::Execute { name, parameters, using }) diff --git a/src/test_utils.rs b/src/test_utils.rs index 1147cfcef2..852f197ef8 100644 --- a/src/test_utils.rs +++ b/src/test_utils.rs @@ -242,6 +242,7 @@ pub fn alter_table_op_with_name(stmt: Statement, expected_name: &str) -> AlterTa if_exists, only: is_only, operations, + location: _, } => { assert_eq!(name.to_string(), expected_name); assert!(!if_exists); diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 658edee73c..2d5086c275 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -124,6 +124,18 @@ fn test_alter_partition() { hive().verified_stmt(alter); } +#[test] +fn test_alter_with_location() { + let alter = "ALTER TABLE db.table PARTITION (a = 2) RENAME TO PARTITION (a = 1) LOCATION 's3://...'"; + hive().verified_stmt(alter); +} + +#[test] +fn test_alter_with_set_location() { + let alter = "ALTER TABLE db.table PARTITION (a = 2) RENAME TO PARTITION (a = 1) SET LOCATION 's3://...'"; + hive().verified_stmt(alter); +} + #[test] fn test_add_partition() { let add = "ALTER TABLE db.table ADD IF NOT EXISTS PARTITION (a = 'asdf', b = 2)"; @@ -193,6 +205,12 @@ fn create_temp_table() { hive().one_statement_parses_to(query2, query); } +#[test] +fn create_delimited_table() { + let query = "CREATE EXTERNAL TABLE tab (cola STRING, colb BIGINT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\"' MAP KEYS TERMINATED BY '\"' LOCATION 's3://...'"; + hive().verified_stmt(query); +} + #[test] fn create_local_directory() { let query = diff --git a/tests/sqlparser_postgres.rs b/tests/sqlparser_postgres.rs index f2a111745a..303631bea1 100644 --- a/tests/sqlparser_postgres.rs +++ b/tests/sqlparser_postgres.rs @@ -677,6 +677,7 @@ fn parse_alter_table_add_columns() { if_exists, only, operations, + location: _ } => { assert_eq!(name.to_string(), "tab"); assert!(if_exists); @@ -1420,7 +1421,7 @@ fn parse_execute() { Statement::Execute { name: "a".into(), parameters: vec![], - using: None + using: vec![] } ); @@ -1433,21 +1434,28 @@ fn parse_execute() { Expr::Value(number("1")), Expr::Value(Value::SingleQuotedString("t".to_string())) ], - using: None + using: vec![] } ); - let stmt = pg_and_generic().verified_stmt("EXECUTE a USING CAST(1337 AS SMALLINT)"); + let stmt = pg_and_generic().verified_stmt("EXECUTE a USING CAST(1337 AS SMALLINT), CAST(7331 AS SMALLINT)"); assert_eq!( stmt, Statement::Execute { name: "a".into(), parameters: vec![], - using: Some(Expr::Cast { - expr: Box::new(Expr::Value(Value::Number("1337".parse().unwrap(), false))), - data_type: DataType::SmallInt(None), - format: None - }) + using: vec![ + Expr::Cast { + expr: Box::new(Expr::Value(Value::Number("1337".parse().unwrap(), false))), + data_type: DataType::SmallInt(None), + format: None + }, + Expr::Cast { + expr: Box::new(Expr::Value(Value::Number("7331".parse().unwrap(), false))), + data_type: DataType::SmallInt(None), + format: None + }, + ] } ); } From 2bf947b14bd23d108f35953b8169389fde345f10 Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Fri, 2 Feb 2024 16:16:17 -0500 Subject: [PATCH 6/6] added describe changes --- src/ast/mod.rs | 58 +++++++++++++++++++++++++++++++-------- src/keywords.rs | 1 + src/parser/mod.rs | 15 ++++++++-- tests/sqlparser_common.rs | 18 ++++++++++-- tests/sqlparser_hive.rs | 10 +++++++ 5 files changed, 85 insertions(+), 17 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index d79601c9f9..145d0bc216 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2191,7 +2191,8 @@ pub enum Statement { /// Note: this is a MySQL-specific statement. See ExplainTable { /// If true, query used the MySQL `DESCRIBE` alias for explain - describe_alias: bool, + describe_alias: DescribeAlias, + hive_format: Option, /// Table name #[cfg_attr(feature = "visitor", visit(with = "visit_relation"))] table_name: ObjectName, @@ -2201,7 +2202,7 @@ pub enum Statement { /// ``` Explain { // If true, query used the MySQL `DESCRIBE` alias for explain - describe_alias: bool, + describe_alias: DescribeAlias, /// Carry out the command and show actual run times and other statistics. analyze: bool, // Display additional information regarding the plan. @@ -2359,12 +2360,13 @@ impl fmt::Display for Statement { } Statement::ExplainTable { describe_alias, + hive_format, table_name, } => { - if *describe_alias { - write!(f, "DESCRIBE ")?; - } else { - write!(f, "EXPLAIN ")?; + write!(f, "{describe_alias} ")?; + + if let Some(format) = hive_format { + write!(f, "{} ", format)?; } write!(f, "{table_name}") @@ -2376,11 +2378,7 @@ impl fmt::Display for Statement { statement, format, } => { - if *describe_alias { - write!(f, "DESCRIBE ")?; - } else { - write!(f, "EXPLAIN ")?; - } + write!(f, "{describe_alias} ")?; if *analyze { write!(f, "ANALYZE ")?; @@ -4649,6 +4647,44 @@ impl fmt::Display for HiveDelimiter { } } +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum HiveDescribeFormat { + Extended, + Formatted, +} + +impl fmt::Display for HiveDescribeFormat { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use HiveDescribeFormat::*; + f.write_str(match self { + Extended => { "EXTENDED" } + Formatted => { "FORMATTED" } + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum DescribeAlias { + Describe, + Explain, + Desc, +} + +impl fmt::Display for DescribeAlias { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use DescribeAlias::*; + f.write_str(match self { + Describe => { "DESCRIBE" } + Explain => { "EXPLAIN" } + Desc => { "DESC" } + }) + } +} + #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] diff --git a/src/keywords.rs b/src/keywords.rs index d7a99e0b51..9ef99bd3f6 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -302,6 +302,7 @@ define_keywords!( FORCE_QUOTE, FOREIGN, FORMAT, + FORMATTED, FORWARD, FRAME_ROW, FREE, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 51db71b649..348a7802ea 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -464,8 +464,9 @@ impl<'a> Parser<'a> { Token::Word(w) => match w.keyword { Keyword::KILL => Ok(self.parse_kill()?), Keyword::FLUSH => Ok(self.parse_flush()?), - Keyword::DESCRIBE => Ok(self.parse_explain(true)?), - Keyword::EXPLAIN => Ok(self.parse_explain(false)?), + Keyword::DESC => Ok(self.parse_explain(DescribeAlias::Desc)?), + Keyword::DESCRIBE => Ok(self.parse_explain(DescribeAlias::Describe)?), + Keyword::EXPLAIN => Ok(self.parse_explain(DescribeAlias::Explain)?), Keyword::ANALYZE => Ok(self.parse_analyze()?), Keyword::SELECT | Keyword::WITH | Keyword::VALUES => { self.prev_token(); @@ -6507,7 +6508,7 @@ impl<'a> Parser<'a> { Ok(Statement::Kill { modifier, id }) } - pub fn parse_explain(&mut self, describe_alias: bool) -> Result { + pub fn parse_explain(&mut self, describe_alias: DescribeAlias) -> Result { let analyze = self.parse_keyword(Keyword::ANALYZE); let verbose = self.parse_keyword(Keyword::VERBOSE); let mut format = None; @@ -6527,9 +6528,17 @@ impl<'a> Parser<'a> { format, }), _ => { + let mut hive_format = None; + match self.parse_one_of_keywords(&[Keyword::EXTENDED, Keyword::FORMATTED]) { + Some(Keyword::EXTENDED) => { hive_format = Some(HiveDescribeFormat::Extended) } + Some(Keyword::FORMATTED) => { hive_format = Some(HiveDescribeFormat::Formatted) } + _ => {} + } + let table_name = self.parse_object_name(false)?; Ok(Statement::ExplainTable { describe_alias, + hive_format, table_name, }) } diff --git a/tests/sqlparser_common.rs b/tests/sqlparser_common.rs index 48889b770d..a72065fc61 100644 --- a/tests/sqlparser_common.rs +++ b/tests/sqlparser_common.rs @@ -3893,19 +3893,31 @@ fn run_explain_analyze( #[test] fn parse_explain_table() { - let validate_explain = |query: &str, expected_describe_alias: bool| match verified_stmt(query) { + let validate_explain = |query: &str, expected_describe_alias: DescribeAlias| match verified_stmt(query) { Statement::ExplainTable { describe_alias, + hive_format, table_name, } => { assert_eq!(describe_alias, expected_describe_alias); + assert_eq!(hive_format, None); assert_eq!("test_identifier", table_name.to_string()); } _ => panic!("Unexpected Statement, must be ExplainTable"), }; - validate_explain("EXPLAIN test_identifier", false); - validate_explain("DESCRIBE test_identifier", true); + validate_explain("EXPLAIN test_identifier", DescribeAlias::Explain); + validate_explain("DESCRIBE test_identifier", DescribeAlias::Describe); +} + +#[test] +fn explain_describe() { + verified_stmt("DESCRIBE test.table"); +} + +#[test] +fn explain_desc() { + verified_stmt("DESC test.table"); } #[test] diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 2d5086c275..21101fe335 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -48,6 +48,16 @@ fn parse_describe() { generic(None).verified_stmt(describe); } +#[test] +fn explain_describe_formatted() { + hive().verified_stmt("DESCRIBE FORMATTED test.table"); +} + +#[test] +fn explain_describe_extended() { + hive().verified_stmt("DESCRIBE EXTENDED test.table"); +} + #[test] fn parse_insert_overwrite() { let insert_partitions = r#"INSERT OVERWRITE TABLE db.new_table PARTITION (a = '1', b) SELECT a, b, c FROM db.table"#;