fixed parititoned by multiple columns in hql

xnuinside · Sep 17, 2021 · b2ab6b8 · b2ab6b8
1 parent 7c9c173
commit b2ab6b8
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 14 deletions.
diff --git a/CHANGELOG.txt b/CHANGELOG.txt
@@ -1,3 +1,13 @@
+**v0.20.0**
+Fixes:
+
+1. Fixed issue with PARTITIONED BY multiple columns in HQL - https://github.com/xnuinside/simple-ddl-parser/issues/66
+
+
+Features:
+
+1. Added support for HQL statement TBLPROPERTIES - https://github.com/xnuinside/simple-ddl-parser/issues/65
+
 **v0.19.5**
 Fixes:
 

diff --git a/simple_ddl_parser/ddl_parser.py b/simple_ddl_parser/ddl_parser.py
@@ -56,7 +56,8 @@ def t_ID(self, t):
         r"([0-9]\.[0-9])\w|([a-zA-Z_,0-9:><\/\=\-\+\~\%$\*'\()!{}\[\]\"\`]+)"
         t.type = tok.symbol_tokens.get(t.value, "ID")
         skip_id_tokens = ["(", ")", ","]
-        if t.type == "LP" and not self.lexer.after_columns:
+        print(self.lexer.is_table, t.value not in skip_id_tokens, self.lexer.lp_open)
+        if t.type == "LP":
             self.lexer.lp_open += 1
             self.lexer.columns_def = True
             return t
@@ -108,6 +109,7 @@ def set_last_token(self, t):
             self.lexer.is_table = False
         elif t.type == "TABLE" or t.type == "INDEX":
             self.lexer.is_table = True
+        print(t.value, t.type)
         return t
 
     def t_newline(self, t):

diff --git a/simple_ddl_parser/dialects/hql.py b/simple_ddl_parser/dialects/hql.py
@@ -1,5 +1,7 @@
 from simple_ddl_parser.utils import check_spec
 
+from simple_ddl_parser.utils import remove_par
+
 
 class HQL:
     def p_expression_location(self, p):
@@ -78,7 +80,21 @@ def p_expression_stored_as(self, p):
         p[0]["stored_as"] = p_list[-1]
 
     def p_expression_partitioned_by_hql(self, p):
-        """expr : expr PARTITIONED BY LP pid_with_type RP"""
+        """expr : expr PARTITIONED BY pid_with_type """
         p[0] = p[1]
         p_list = list(p)
-        p[0]["partitioned_by"] = p_list[-2]
+        print(list(p_list))
+        p[0]["partitioned_by"] = p_list[-1]
+
+    def p_pid_with_type(self, p):
+        """pid_with_type :  LP column
+        | pid_with_type COMMA column
+        | pid_with_type RP
+        """
+        p_list = remove_par(list(p))
+        if not isinstance(p_list[1], list):
+            p[0] = [p_list[1]]
+        else:
+            p[0] = p_list[1]
+            if len(p_list) > 2:
+                p[0].append(p_list[-1])
diff --git a/simple_ddl_parser/dialects/sql.py b/simple_ddl_parser/dialects/sql.py
@@ -955,17 +955,6 @@ def p_alter_check(self, p):
             p[0]["check"]["constraint_name"] = p[2]["constraint"]["name"]
         p[0]["check"]["statement"] = p_list[-1]["check"]
 
-    def p_pid_with_type(self, p):
-        """pid_with_type :  column
-        | pid_with_type COMMA column
-        """
-        p_list = list(p)
-        if not isinstance(p_list[1], list):
-            p[0] = [p_list[1]]
-        else:
-            p[0] = p_list[1]
-            p[0].append(p_list[-1])
-
     def p_pid(self, p):
         """pid :  ID
         | STRING

diff --git a/tests/test_hql_output_mode.py b/tests/test_hql_output_mode.py
@@ -1642,3 +1642,49 @@ def test_special_characters_in_comment():
     """
     result = DDLParser(ddl).run(group_by_type=True, output_mode="hql")
     assert expected == result
+
+
+def test_partitioned_by_multiple_columns():
+    ddl = """
+    CREATE EXTERNAL TABLE test (
+    test STRING NULL COMMENT 'xxxx',
+    )
+    PARTITIONED BY (snapshot STRING, cluster STRING)
+    """
+    parse_result = DDLParser(ddl).run(output_mode="hql", group_by_type=True)
+    expected = {'domains': [],
+ 'schemas': [],
+ 'sequences': [],
+ 'tables': [{'alter': {},
+             'checks': [],
+             'collection_items_terminated_by': None,
+             'columns': [{'check': None,
+                          'comment': "'xxxx'",
+                          'default': None,
+                          'name': 'test',
+                          'nullable': True,
+                          'references': None,
+                          'size': None,
+                          'type': 'STRING',
+                          'unique': False}],
+             'comment': None,
+             'external': True,
+             'fields_terminated_by': None,
+             'index': [],
+             'lines_terminated_by': None,
+             'location': None,
+             'map_keys_terminated_by': None,
+             'partitioned_by': [{'name': 'snapshot',
+                                 'size': None,
+                                 'type': 'STRING'},
+                                {'name': 'cluster',
+                                 'size': None,
+                                 'type': 'STRING'}],
+             'primary_key': [],
+             'row_format': None,
+             'schema': None,
+             'stored_as': None,
+             'table_name': 'test',
+             'tablespace': None}],
+ 'types': []}
+    assert parse_result == expected