Non-Latin characters support (apache#840)

* Non latin characters

---------

Co-authored-by: Maciej Skrzypkowski <[email protected]>

* Test for mysql

---------

Co-authored-by: Maciej Skrzypkowski <[email protected]>

Author: mskrzypkows

src/dialect/generic.rs

@@ -17,12 +17,11 @@ pub struct GenericDialect;
 
 impl Dialect for GenericDialect {
     fn is_identifier_start(&self, ch: char) -> bool {
-        ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch == '_' || ch == '#' || ch == '@'
+        ch.is_alphabetic() || ch == '_' || ch == '#' || ch == '@'
     }
 
     fn is_identifier_part(&self, ch: char) -> bool {
-        ch.is_ascii_lowercase()
-            || ch.is_ascii_uppercase()
+        ch.is_alphabetic()
             || ch.is_ascii_digit()
             || ch == '@'
             || ch == '$'

src/dialect/mssql.rs

@@ -22,13 +22,11 @@ impl Dialect for MsSqlDialect {
 
     fn is_identifier_start(&self, ch: char) -> bool {
         // See https://docs.microsoft.com/en-us/sql/relational-databases/databases/database-identifiers?view=sql-server-2017#rules-for-regular-identifiers
-        // We don't support non-latin "letters" currently.
-        ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch == '_' || ch == '#' || ch == '@'
+        ch.is_alphabetic() || ch == '_' || ch == '#' || ch == '@'
     }
 
     fn is_identifier_part(&self, ch: char) -> bool {
-        ch.is_ascii_lowercase()
-            || ch.is_ascii_uppercase()
+        ch.is_alphabetic()
             || ch.is_ascii_digit()
             || ch == '@'
             || ch == '$'

src/dialect/mysql.rs

@@ -20,8 +20,7 @@ impl Dialect for MySqlDialect {
         // See https://dev.mysql.com/doc/refman/8.0/en/identifiers.html.
         // We don't yet support identifiers beginning with numbers, as that
         // makes it hard to distinguish numeric literals.
-        ch.is_ascii_lowercase()
-            || ch.is_ascii_uppercase()
+        ch.is_alphabetic()
             || ch == '_'
             || ch == '$'
             || ch == '@'

src/dialect/postgresql.rs

@@ -23,16 +23,12 @@ impl Dialect for PostgreSqlDialect {
     fn is_identifier_start(&self, ch: char) -> bool {
         // See https://www.postgresql.org/docs/11/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
         // We don't yet support identifiers beginning with "letters with
-        // diacritical marks and non-Latin letters"
-        ch.is_ascii_lowercase() || ch.is_ascii_uppercase() || ch == '_'
+        // diacritical marks"
+        ch.is_alphabetic() || ch == '_'
     }
 
     fn is_identifier_part(&self, ch: char) -> bool {
-        ch.is_ascii_lowercase()
-            || ch.is_ascii_uppercase()
-            || ch.is_ascii_digit()
-            || ch == '$'
-            || ch == '_'
+        ch.is_alphabetic() || ch.is_ascii_digit() || ch == '$' || ch == '_'
     }
 
     fn parse_statement(&self, parser: &mut Parser) -> Option<Result<Statement, ParserError>> {

src/tokenizer.rs

@@ -1519,20 +1519,16 @@ mod tests {
 
     #[test]
     fn tokenize_invalid_string() {
-        let sql = String::from("\nمصطفىh");
+        let sql = String::from("\n💝مصطفىh");
 
         let dialect = GenericDialect {};
         let mut tokenizer = Tokenizer::new(&dialect, &sql);
         let tokens = tokenizer.tokenize().unwrap();
         // println!("tokens: {:#?}", tokens);
         let expected = vec![
             Token::Whitespace(Whitespace::Newline),
-            Token::Char('م'),
-            Token::Char('ص'),
-            Token::Char('ط'),
-            Token::Char('ف'),
-            Token::Char('ى'),
-            Token::make_word("h", None),
+            Token::Char('💝'),
+            Token::make_word("مصطفىh", None),
         ];
         compare(expected, tokens);
     }
@@ -1582,7 +1578,7 @@ mod tests {
 
     #[test]
     fn tokenize_invalid_string_cols() {
-        let sql = String::from("\n\nSELECT * FROM table\tمصطفىh");
+        let sql = String::from("\n\nSELECT * FROM table\t💝مصطفىh");
 
         let dialect = GenericDialect {};
         let mut tokenizer = Tokenizer::new(&dialect, &sql);
@@ -1599,12 +1595,8 @@ mod tests {
             Token::Whitespace(Whitespace::Space),
             Token::make_keyword("table"),
             Token::Whitespace(Whitespace::Tab),
-            Token::Char('م'),
-            Token::Char('ص'),
-            Token::Char('ط'),
-            Token::Char('ف'),
-            Token::Char('ى'),
-            Token::make_word("h", None),
+            Token::Char('💝'),
+            Token::make_word("مصطفىh", None),
         ];
         compare(expected, tokens);
     }

tests/sqlparser_common.rs

@@ -6734,3 +6734,23 @@ fn make_where_clause(num: usize) -> String {
     }
     output
 }
+
+#[test]
+fn parse_non_latin_identifiers() {
+    let supported_dialects = TestedDialects {
+        dialects: vec![
+            Box::new(GenericDialect {}),
+            Box::new(PostgreSqlDialect {}),
+            Box::new(MsSqlDialect {}),
+            Box::new(RedshiftSqlDialect {}),
+            Box::new(MySqlDialect {}),
+        ],
+    };
+
+    supported_dialects.verified_stmt("SELECT a.説明 FROM test.public.inter01 AS a");
+    supported_dialects.verified_stmt("SELECT a.説明 FROM inter01 AS a, inter01_transactions AS b WHERE a.説明 = b.取引 GROUP BY a.説明");
+    supported_dialects.verified_stmt("SELECT 説明, hühnervögel, garçon, Москва, 東京 FROM inter01");
+    assert!(supported_dialects
+        .parse_sql_statements("SELECT 💝 FROM table1")
+        .is_err());
+}