From 65e2f4ac3840f234dd63f2f1f132e6cc54e4189c Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Thu, 29 Feb 2024 10:32:18 -0500 Subject: [PATCH 1/4] adding delimited --- src/ast/mod.rs | 52 ++++++++++++++++++++++++++++-- src/keywords.rs | 8 +++++ src/parser/mod.rs | 71 ++++++++++++++++++++++++++++++++++++++++- tests/sqlparser_hive.rs | 6 ++++ 4 files changed, 134 insertions(+), 3 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index a2c28c810e..acadeb35cb 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2977,7 +2977,12 @@ impl fmt::Display for Statement { Some(HiveRowFormat::SERDE { class }) => { write!(f, " ROW FORMAT SERDE '{class}'")? } - Some(HiveRowFormat::DELIMITED) => write!(f, " ROW FORMAT DELIMITED")?, + Some(HiveRowFormat::DELIMITED { delimiters }) => { + write!(f, " ROW FORMAT DELIMITED")?; + if delimiters.len() > 0 { + write!(f, " {}", display_separated(delimiters, " "))?; + } + }, None => (), } match storage { @@ -4566,7 +4571,50 @@ pub enum HiveDistributionStyle { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum HiveRowFormat { SERDE { class: String }, - DELIMITED, + DELIMITED { + delimiters: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub struct HiveRowDelimiter { + pub delimiter: HiveDelimiter, + pub char: Ident +} + +impl fmt::Display for HiveRowDelimiter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{} ", self.delimiter)?; + write!(f, "{}", self.char) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] +pub enum HiveDelimiter { + FieldsTerminatedBy, + FieldsEscapedBy, + CollectionItemsTerminatedBy, + MapKeysTerminatedBy, + LinesTerminatedBy, + NullDefinedAs +} + +impl fmt::Display for HiveDelimiter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use HiveDelimiter::*; + f.write_str(match self { + FieldsTerminatedBy => { "FIELDS TERMINATED BY" } + FieldsEscapedBy => { "ESCAPED BY" } + CollectionItemsTerminatedBy => { "COLLECTION ITEMS TERMINATED BY" } + MapKeysTerminatedBy => { "MAP KEYS TERMINATED BY" } + LinesTerminatedBy => { "LINES TERMINATED BY" } + NullDefinedAs => { "NULL DEFINED AS" } + }) + } } #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] diff --git a/src/keywords.rs b/src/keywords.rs index f14b92b76b..6c47bf4cce 100644 --- a/src/keywords.rs +++ b/src/keywords.rs @@ -153,6 +153,7 @@ define_keywords!( COLLATE, COLLATION, COLLECT, + COLLECTION, COLUMN, COLUMNS, COMMENT, @@ -212,6 +213,7 @@ define_keywords!( DEFAULT, DEFERRABLE, DEFERRED, + DEFINED, DELAYED, DELETE, DELIMITED, @@ -258,6 +260,7 @@ define_keywords!( EQUALS, ERROR, ESCAPE, + ESCAPED, EVENT, EVERY, EXCEPT, @@ -366,6 +369,7 @@ define_keywords!( ISOLATION, ISOWEEK, ISOYEAR, + ITEMS, JAR, JOIN, JSON, @@ -374,6 +378,7 @@ define_keywords!( JSON_TABLE, JULIAN, KEY, + KEYS, KILL, LAG, LANGUAGE, @@ -388,6 +393,7 @@ define_keywords!( LIKE, LIKE_REGEX, LIMIT, + LINES, LISTAGG, LN, LOCAL, @@ -402,6 +408,7 @@ define_keywords!( LOW_PRIORITY, MACRO, MANAGEDLOCATION, + MAP, MATCH, MATCHED, MATERIALIZED, @@ -648,6 +655,7 @@ define_keywords!( TBLPROPERTIES, TEMP, TEMPORARY, + TERMINATED, TEXT, TEXTFILE, THEN, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 36ac2fd28c..4d5775d5dd 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4128,7 +4128,76 @@ impl<'a> Parser<'a> { let class = self.parse_literal_string()?; Ok(HiveRowFormat::SERDE { class }) } - _ => Ok(HiveRowFormat::DELIMITED), + _ => { + let mut row_delimiters = vec![]; + + loop { + match self.parse_one_of_keywords(&[Keyword::FIELDS, Keyword::COLLECTION, Keyword::MAP, Keyword::LINES, Keyword::NULL]) { + Some(Keyword::FIELDS) => { + if self.parse_keywords(&[Keyword::TERMINATED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::FieldsTerminatedBy, + char: self.parse_identifier(false)? + }); + + if self.parse_keywords(&[Keyword::ESCAPED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::FieldsEscapedBy, + char: self.parse_identifier(false)? + }); + } + } else { + break; + } + } + Some(Keyword::COLLECTION) => { + if self.parse_keywords(&[Keyword::ITEMS, Keyword::TERMINATED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::CollectionItemsTerminatedBy, + char: self.parse_identifier(false)? + }); + } else { + break; + } + } + Some(Keyword::MAP) => { + if self.parse_keywords(&[Keyword::KEYS, Keyword::TERMINATED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::MapKeysTerminatedBy, + char: self.parse_identifier(false)? + }); + } else { + break; + } + } + Some(Keyword::LINES) => { + if self.parse_keywords(&[Keyword::TERMINATED, Keyword::BY]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::LinesTerminatedBy, + char: self.parse_identifier(false)? + }); + } else { + break; + } + } + Some(Keyword::NULL) => { + if self.parse_keywords(&[Keyword::DEFINED, Keyword::AS]) { + row_delimiters.push(HiveRowDelimiter { + delimiter: HiveDelimiter::NullDefinedAs, + char: self.parse_identifier(false)? + }); + } else { + break; + } + } + _ => { break; } + } + } + + Ok(HiveRowFormat::DELIMITED { + delimiters: row_delimiters + }) + }, } } diff --git a/tests/sqlparser_hive.rs b/tests/sqlparser_hive.rs index 66eef09e17..0cd77d8143 100644 --- a/tests/sqlparser_hive.rs +++ b/tests/sqlparser_hive.rs @@ -191,6 +191,12 @@ fn create_temp_table() { hive().one_statement_parses_to(query2, query); } +#[test] +fn create_delimited_table() { + let query = "CREATE TABLE tab (cola STRING, colb BIGINT) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' ESCAPED BY '\"' MAP KEYS TERMINATED BY '\"'"; + hive().verified_stmt(query); +} + #[test] fn create_local_directory() { let query = From 6268f171b0488c76846c2c344baa349a61e48895 Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Thu, 29 Feb 2024 16:16:58 -0500 Subject: [PATCH 2/4] fixing codestyle and lint --- src/ast/mod.rs | 24 +++++++++++------------- src/parser/mod.rs | 28 ++++++++++++++++++---------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index acadeb35cb..03a64ed589 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -2979,10 +2979,10 @@ impl fmt::Display for Statement { } Some(HiveRowFormat::DELIMITED { delimiters }) => { write!(f, " ROW FORMAT DELIMITED")?; - if delimiters.len() > 0 { + if !delimiters.is_empty() { write!(f, " {}", display_separated(delimiters, " "))?; } - }, + } None => (), } match storage { @@ -4571,9 +4571,7 @@ pub enum HiveDistributionStyle { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub enum HiveRowFormat { SERDE { class: String }, - DELIMITED { - delimiters: Vec, - }, + DELIMITED { delimiters: Vec }, } #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] @@ -4581,7 +4579,7 @@ pub enum HiveRowFormat { #[cfg_attr(feature = "visitor", derive(Visit, VisitMut))] pub struct HiveRowDelimiter { pub delimiter: HiveDelimiter, - pub char: Ident + pub char: Ident, } impl fmt::Display for HiveRowDelimiter { @@ -4600,19 +4598,19 @@ pub enum HiveDelimiter { CollectionItemsTerminatedBy, MapKeysTerminatedBy, LinesTerminatedBy, - NullDefinedAs + NullDefinedAs, } impl fmt::Display for HiveDelimiter { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { use HiveDelimiter::*; f.write_str(match self { - FieldsTerminatedBy => { "FIELDS TERMINATED BY" } - FieldsEscapedBy => { "ESCAPED BY" } - CollectionItemsTerminatedBy => { "COLLECTION ITEMS TERMINATED BY" } - MapKeysTerminatedBy => { "MAP KEYS TERMINATED BY" } - LinesTerminatedBy => { "LINES TERMINATED BY" } - NullDefinedAs => { "NULL DEFINED AS" } + FieldsTerminatedBy => "FIELDS TERMINATED BY", + FieldsEscapedBy => "ESCAPED BY", + CollectionItemsTerminatedBy => "COLLECTION ITEMS TERMINATED BY", + MapKeysTerminatedBy => "MAP KEYS TERMINATED BY", + LinesTerminatedBy => "LINES TERMINATED BY", + NullDefinedAs => "NULL DEFINED AS", }) } } diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 4d5775d5dd..87b0105451 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4132,18 +4132,24 @@ impl<'a> Parser<'a> { let mut row_delimiters = vec![]; loop { - match self.parse_one_of_keywords(&[Keyword::FIELDS, Keyword::COLLECTION, Keyword::MAP, Keyword::LINES, Keyword::NULL]) { + match self.parse_one_of_keywords(&[ + Keyword::FIELDS, + Keyword::COLLECTION, + Keyword::MAP, + Keyword::LINES, + Keyword::NULL + ]) { Some(Keyword::FIELDS) => { if self.parse_keywords(&[Keyword::TERMINATED, Keyword::BY]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::FieldsTerminatedBy, - char: self.parse_identifier(false)? + char: self.parse_identifier(false)?, }); if self.parse_keywords(&[Keyword::ESCAPED, Keyword::BY]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::FieldsEscapedBy, - char: self.parse_identifier(false)? + char: self.parse_identifier(false)?, }); } } else { @@ -4154,7 +4160,7 @@ impl<'a> Parser<'a> { if self.parse_keywords(&[Keyword::ITEMS, Keyword::TERMINATED, Keyword::BY]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::CollectionItemsTerminatedBy, - char: self.parse_identifier(false)? + char: self.parse_identifier(false)?, }); } else { break; @@ -4164,7 +4170,7 @@ impl<'a> Parser<'a> { if self.parse_keywords(&[Keyword::KEYS, Keyword::TERMINATED, Keyword::BY]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::MapKeysTerminatedBy, - char: self.parse_identifier(false)? + char: self.parse_identifier(false)?, }); } else { break; @@ -4174,7 +4180,7 @@ impl<'a> Parser<'a> { if self.parse_keywords(&[Keyword::TERMINATED, Keyword::BY]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::LinesTerminatedBy, - char: self.parse_identifier(false)? + char: self.parse_identifier(false)?, }); } else { break; @@ -4184,20 +4190,22 @@ impl<'a> Parser<'a> { if self.parse_keywords(&[Keyword::DEFINED, Keyword::AS]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::NullDefinedAs, - char: self.parse_identifier(false)? + char: self.parse_identifier(false)?, }); } else { break; } } - _ => { break; } + _ => { + break; + } } } Ok(HiveRowFormat::DELIMITED { - delimiters: row_delimiters + delimiters: row_delimiters, }) - }, + } } } From b9abc628a0886c1feaeb8ecd9357be22fac6b19b Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Thu, 29 Feb 2024 16:18:54 -0500 Subject: [PATCH 3/4] another pass at codestyle --- src/parser/mod.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 87b0105451..79d9b8de45 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4137,7 +4137,7 @@ impl<'a> Parser<'a> { Keyword::COLLECTION, Keyword::MAP, Keyword::LINES, - Keyword::NULL + Keyword::NULL, ]) { Some(Keyword::FIELDS) => { if self.parse_keywords(&[Keyword::TERMINATED, Keyword::BY]) { @@ -4157,7 +4157,11 @@ impl<'a> Parser<'a> { } } Some(Keyword::COLLECTION) => { - if self.parse_keywords(&[Keyword::ITEMS, Keyword::TERMINATED, Keyword::BY]) { + if self.parse_keywords(&[ + Keyword::ITEMS, + Keyword::TERMINATED, + Keyword::BY, + ]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::CollectionItemsTerminatedBy, char: self.parse_identifier(false)?, @@ -4167,7 +4171,11 @@ impl<'a> Parser<'a> { } } Some(Keyword::MAP) => { - if self.parse_keywords(&[Keyword::KEYS, Keyword::TERMINATED, Keyword::BY]) { + if self.parse_keywords(&[ + Keyword::KEYS, + Keyword::TERMINATED, + Keyword::BY + ]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::MapKeysTerminatedBy, char: self.parse_identifier(false)?, From 8b6926eb69ccb93ced9db72a2984408133dbf2be Mon Sep 17 00:00:00 2001 From: jonathanlehto Date: Thu, 29 Feb 2024 16:20:28 -0500 Subject: [PATCH 4/4] adding comma --- src/parser/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 79d9b8de45..b63346ff84 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -4174,7 +4174,7 @@ impl<'a> Parser<'a> { if self.parse_keywords(&[ Keyword::KEYS, Keyword::TERMINATED, - Keyword::BY + Keyword::BY, ]) { row_delimiters.push(HiveRowDelimiter { delimiter: HiveDelimiter::MapKeysTerminatedBy,