Add support for trailing commas (apache#810)

* Add support for trailing commas

* Support trailing commas for brace/bracket

* Andrew's comments

Author: ankrgyl

src/parser.rs

@@ -195,12 +195,20 @@ impl std::error::Error for ParserError {}
 // By default, allow expressions up to this deep before erroring
 const DEFAULT_REMAINING_DEPTH: usize = 50;
 
+#[derive(Default)]
+pub struct ParserOptions {
+    pub trailing_commas: bool,
+}
+
 pub struct Parser<'a> {
     tokens: Vec<TokenWithLocation>,
     /// The index of the first unprocessed token in `self.tokens`
     index: usize,
     /// The current dialect to use
     dialect: &'a dyn Dialect,
+    /// Additional options that allow you to mix & match behavior otherwise
+    /// constrained to certain dialects (e.g. trailing commas)
+    options: ParserOptions,
     /// ensure the stack does not overflow by limiting recusion depth
     recursion_counter: RecursionCounter,
 }
@@ -227,6 +235,7 @@ impl<'a> Parser<'a> {
             index: 0,
             dialect,
             recursion_counter: RecursionCounter::new(DEFAULT_REMAINING_DEPTH),
+            options: ParserOptions::default(),
         }
     }
 
@@ -255,6 +264,31 @@ impl<'a> Parser<'a> {
         self
     }
 
+    /// Specify additional parser options
+    ///
+    ///
+    /// [`Parser`] supports additional options ([`ParserOptions`]) that allow you to
+    /// mix & match behavior otherwise constrained to certain dialects (e.g. trailing
+    /// commas).
+    ///
+    /// Example:
+    /// ```
+    /// # use sqlparser::{parser::{Parser, ParserError, ParserOptions}, dialect::GenericDialect};
+    /// # fn main() -> Result<(), ParserError> {
+    /// let dialect = GenericDialect{};
+    /// let result = Parser::new(&dialect)
+    ///   .with_options(ParserOptions { trailing_commas: true })
+    ///   .try_with_sql("SELECT a, b, COUNT(*), FROM foo GROUP BY a, b,")?
+    ///   .parse_statements();
+    ///   assert!(matches!(result, Ok(_)));
+    /// # Ok(())
+    /// # }
+    /// ```
+    pub fn with_options(mut self, options: ParserOptions) -> Self {
+        self.options = options;
+        self
+    }
+
     /// Reset this parser to parse the specified token stream
     pub fn with_tokens_with_locations(mut self, tokens: Vec<TokenWithLocation>) -> Self {
         self.tokens = tokens;
@@ -2196,15 +2230,32 @@ impl<'a> Parser<'a> {
 
     /// Parse a comma-separated list of 1+ SelectItem
     pub fn parse_projection(&mut self) -> Result<Vec<SelectItem>, ParserError> {
+        // BigQuery allows trailing commas, but only in project lists
+        // e.g. `SELECT 1, 2, FROM t`
+        // https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#trailing_commas
+        //
+        // This pattern could be captured better with RAII type semantics, but it's quite a bit of
+        // code to add for just one case, so we'll just do it manually here.
+        let old_value = self.options.trailing_commas;
+        self.options.trailing_commas |= dialect_of!(self is BigQueryDialect);
+
+        let ret = self.parse_comma_separated(|p| p.parse_select_item());
+        self.options.trailing_commas = old_value;
+
+        ret
+    }
+
+    /// Parse a comma-separated list of 1+ items accepted by `F`
+    pub fn parse_comma_separated<T, F>(&mut self, mut f: F) -> Result<Vec<T>, ParserError>
+    where
+        F: FnMut(&mut Parser<'a>) -> Result<T, ParserError>,
+    {
         let mut values = vec![];
         loop {
-            values.push(self.parse_select_item()?);
+            values.push(f(self)?);
             if !self.consume_token(&Token::Comma) {
                 break;
-            } else if dialect_of!(self is BigQueryDialect) {
-                // BigQuery allows trailing commas.
-                // e.g. `SELECT 1, 2, FROM t`
-                // https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#trailing_commas
+            } else if self.options.trailing_commas {
                 match self.peek_token().token {
                     Token::Word(kw)
                         if keywords::RESERVED_FOR_COLUMN_ALIAS
@@ -2213,29 +2264,18 @@ impl<'a> Parser<'a> {
                     {
                         break;
                     }
-                    Token::RParen | Token::EOF => break,
+                    Token::RParen
+                    | Token::SemiColon
+                    | Token::EOF
+                    | Token::RBracket
+                    | Token::RBrace => break,
                     _ => continue,
                 }
             }
         }
         Ok(values)
     }
 
-    /// Parse a comma-separated list of 1+ items accepted by `F`
-    pub fn parse_comma_separated<T, F>(&mut self, mut f: F) -> Result<Vec<T>, ParserError>
-    where
-        F: FnMut(&mut Parser<'a>) -> Result<T, ParserError>,
-    {
-        let mut values = vec![];
-        loop {
-            values.push(f(self)?);
-            if !self.consume_token(&Token::Comma) {
-                break;
-            }
-        }
-        Ok(values)
-    }
-
     /// Run a parser method `f`, reverting back to the current position
     /// if unsuccessful.
     #[must_use]

tests/sqlparser_bigquery.rs

@@ -181,6 +181,8 @@ fn parse_join_constraint_unnest_alias() {
 fn parse_trailing_comma() {
     for (sql, canonical) in [
         ("SELECT a,", "SELECT a"),
+        ("SELECT 1,", "SELECT 1"),
+        ("SELECT 1,2,", "SELECT 1, 2"),
         ("SELECT a, b,", "SELECT a, b"),
         ("SELECT a, b AS c,", "SELECT a, b AS c"),
         ("SELECT a, b AS c, FROM t", "SELECT a, b AS c FROM t"),