From ce44b2be16524338084c7537fd5226afb8bf73d3 Mon Sep 17 00:00:00 2001 From: Anupam Date: Thu, 22 Jun 2023 10:41:10 -0400 Subject: [PATCH 01/23] initial --- src/sql/parse.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/sql/parse.py b/src/sql/parse.py index 05bf862e8..ae6be1b57 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -53,6 +53,18 @@ def parse(cell, config): if len(pieces) == 1: return result cell = pieces[1] + char_pointer = cell.find("SELECT") + cell = cell[:char_pointer].strip() + " " + cell[char_pointer:] + else: + if pieces[0].endswith("SELECT"): + char_pointer = pieces[0].find("SELECT") + pieces[0] = pieces[0][:char_pointer].replace(" ", "") + pieces[1] = "SELECT " + pieces[1] + else: + char_pointer = pieces[1].find("SELECT") + pieces[0] = pieces[0] + pieces[1][:char_pointer] + pieces[1] = pieces[1][char_pointer:] + # handle no space situation around = if pieces[0].endswith("=<<"): result["result_var"] = pieces[0][:-3] From 10e2d5181431e55608b7f379257d7609b273da6c Mon Sep 17 00:00:00 2001 From: Anupam Date: Fri, 23 Jun 2023 20:39:23 -0400 Subject: [PATCH 02/23] changelog --- CHANGELOG.md | 1 + src/sql/parse.py | 9 +++++++++ src/tests/test_parse.py | 4 ++++ 3 files changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 70bae7a1f..70f6a5d4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ * [Fix] Fixed CI issue by updating `invalid_connection_string_duckdb` in `test_magic.py` (#631) * [Fix] Refactored `ResultSet` to lazy loading (#470) +* [Fix] Improving << parsing logic (#610) ## 0.7.9 (2023-06-19) diff --git a/src/sql/parse.py b/src/sql/parse.py index ae6be1b57..ab8f2ceaa 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -45,6 +45,11 @@ def parse(cell, config): "return_result_var": False, } + select_pointer = cell.lower().find("select") + if select_pointer != -1: + if cell[select_pointer - 1] != " ": + cell = cell[:select_pointer] + " " + cell[select_pointer:] + pieces = cell.split(None, 1) if not pieces: return result @@ -70,6 +75,10 @@ def parse(cell, config): result["result_var"] = pieces[0][:-3] result["return_result_var"] = True cell = pieces[1] + elif pieces[0].endswith("<<"): + result["result_var"] = pieces[0][:-2] + result["return_result_var"] = False + cell = pieces[1] pieces = cell.split(None, 2) # handle flexible spacing around << diff --git a/src/tests/test_parse.py b/src/tests/test_parse.py index a72fb9920..cf97f4beb 100644 --- a/src/tests/test_parse.py +++ b/src/tests/test_parse.py @@ -88,6 +88,10 @@ def test_parse_shovel_operator(): "dest =<< SELECT * FROM work", "dest = << SELECT * FROM work", "dest=<< SELECT * FROM work", + "dest=< Date: Sat, 24 Jun 2023 00:35:10 -0400 Subject: [PATCH 06/23] pytest-newline --- src/sql/parse.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index f9dd70e6c..e9cf65f9b 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -47,8 +47,10 @@ def parse(cell, config): select_pointer = cell.lower().find("select") if select_pointer != -1: - if cell[select_pointer - 1] != " ": + if cell[select_pointer - 1] != " " and cell[select_pointer - 1] != "\n": cell = cell[:select_pointer] + " " + cell[select_pointer:] + else: + pass pieces = cell.split(None, 1) if not pieces: From 3f3055984f4f0a8a600071f6fd93826aac09ac83 Mon Sep 17 00:00:00 2001 From: Anupam Date: Sat, 24 Jun 2023 00:40:50 -0400 Subject: [PATCH 07/23] lint --- src/sql/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index e9cf65f9b..ac395013f 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -49,7 +49,7 @@ def parse(cell, config): if select_pointer != -1: if cell[select_pointer - 1] != " " and cell[select_pointer - 1] != "\n": cell = cell[:select_pointer] + " " + cell[select_pointer:] - else: + else: pass pieces = cell.split(None, 1) From 19d1c13970ba3f7e8027758344bcb65f38f31be5 Mon Sep 17 00:00:00 2001 From: Anupam Date: Sat, 24 Jun 2023 08:00:36 -0400 Subject: [PATCH 08/23] pytest --- src/sql/parse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index ac395013f..2db2fbc44 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -46,7 +46,7 @@ def parse(cell, config): } select_pointer = cell.lower().find("select") - if select_pointer != -1: + if select_pointer != -1 and select_pointer != 0: if cell[select_pointer - 1] != " " and cell[select_pointer - 1] != "\n": cell = cell[:select_pointer] + " " + cell[select_pointer:] else: From 1dbf7a5821b2606654b23959190cd0ae97e60cce Mon Sep 17 00:00:00 2001 From: Anupam Date: Tue, 27 Jun 2023 11:11:39 -0400 Subject: [PATCH 09/23] Empty-Commit From 43e7eae8107b933333dcb0bda146bda0c5942efc Mon Sep 17 00:00:00 2001 From: Anupam Date: Tue, 27 Jun 2023 11:27:53 -0400 Subject: [PATCH 10/23] added more test cases --- src/tests/test_parse.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/tests/test_parse.py b/src/tests/test_parse.py index 74f30f646..f6b0df9c0 100644 --- a/src/tests/test_parse.py +++ b/src/tests/test_parse.py @@ -88,20 +88,29 @@ def test_parse_shovel_operator(): "dest =<< SELECT * FROM work", "dest = << SELECT * FROM work", "dest=<< SELECT * FROM work", + "dest< Date: Thu, 29 Jun 2023 12:19:49 -0400 Subject: [PATCH 14/23] refactor --- src/sql/parse.py | 50 ++++++++++++------------------------------------ 1 file changed, 12 insertions(+), 38 deletions(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index 2db2fbc44..00e0c5d13 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -37,7 +37,6 @@ def parse(cell, config): We're grandfathering the connection string and `<<` operator in. """ - result = { "connection": "", "sql": "", @@ -45,13 +44,6 @@ def parse(cell, config): "return_result_var": False, } - select_pointer = cell.lower().find("select") - if select_pointer != -1 and select_pointer != 0: - if cell[select_pointer - 1] != " " and cell[select_pointer - 1] != "\n": - cell = cell[:select_pointer] + " " + cell[select_pointer:] - else: - pass - pieces = cell.split(None, 1) if not pieces: return result @@ -60,42 +52,24 @@ def parse(cell, config): if len(pieces) == 1: return result cell = pieces[1] + else: + cell = cell - # handle no space situation around = - if pieces[0].endswith("=<<"): - result["result_var"] = pieces[0][:-3] - result["return_result_var"] = True - cell = pieces[1] - elif pieces[0].endswith("<<"): - result["result_var"] = pieces[0][:-2] - result["return_result_var"] = False - cell = pieces[1] + pointer = cell.find("<<") + if pointer != -1: + left = cell[:pointer].replace(" ", "").replace("\n", "") + right = cell[pointer + 2 :].strip() - pieces = cell.split(None, 2) - # handle flexible spacing around << - if len(pieces) > 1 and pieces[1] == "<<": - if pieces[0].endswith("="): - result["result_var"] = pieces[0][:-1] + if "=" in left: + result["result_var"] = left[:-1] result["return_result_var"] = True else: - result["result_var"] = pieces[0] + result["result_var"] = left - if len(pieces) == 2: - return result - cell = pieces[2] - # handle flexible spacing around =<< - elif len(pieces) > 1 and ( - (pieces[1] == "=<<") or (pieces[1] == "=" and pieces[2].startswith("<<")) - ): - result["result_var"] = pieces[0] - result["return_result_var"] = True - if pieces[1] == "=<<": - cell = pieces[2] - else: - pieces = cell.split(None, 3) - cell = pieces[3] + result["sql"] = right + else: + result["sql"] = cell.strip() - result["sql"] = cell return result From 0def7c4627e65889673a87ab45ab0deb1b013789 Mon Sep 17 00:00:00 2001 From: Anupam Date: Thu, 29 Jun 2023 13:04:17 -0400 Subject: [PATCH 15/23] test --- src/sql/parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index 00e0c5d13..20f078f67 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -58,7 +58,7 @@ def parse(cell, config): pointer = cell.find("<<") if pointer != -1: left = cell[:pointer].replace(" ", "").replace("\n", "") - right = cell[pointer + 2 :].strip() + right = cell[pointer + 2 :].strip(' ') if "=" in left: result["result_var"] = left[:-1] @@ -68,7 +68,7 @@ def parse(cell, config): result["sql"] = right else: - result["sql"] = cell.strip() + result["sql"] = cell.strip(' ') return result From 3941ff572bc994cce685a129152e0b03e413c3f9 Mon Sep 17 00:00:00 2001 From: Anupam Date: Thu, 29 Jun 2023 13:06:44 -0400 Subject: [PATCH 16/23] lint --- src/sql/parse.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index 20f078f67..405778320 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -58,7 +58,7 @@ def parse(cell, config): pointer = cell.find("<<") if pointer != -1: left = cell[:pointer].replace(" ", "").replace("\n", "") - right = cell[pointer + 2 :].strip(' ') + right = cell[pointer + 2 :].strip(" ") if "=" in left: result["result_var"] = left[:-1] @@ -68,7 +68,7 @@ def parse(cell, config): result["sql"] = right else: - result["sql"] = cell.strip(' ') + result["sql"] = cell.strip(" ") return result From 5222cf31572300dd5a304abdfc7c31c0baeed159 Mon Sep 17 00:00:00 2001 From: Anupam Date: Fri, 30 Jun 2023 11:55:22 -0400 Subject: [PATCH 17/23] pytest --- src/sql/parse.py | 2 +- src/tests/test_parse.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index 405778320..a87613ee4 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -69,7 +69,7 @@ def parse(cell, config): result["sql"] = right else: result["sql"] = cell.strip(" ") - + print(result) return result diff --git a/src/tests/test_parse.py b/src/tests/test_parse.py index e4c060d77..f501de559 100644 --- a/src/tests/test_parse.py +++ b/src/tests/test_parse.py @@ -144,7 +144,7 @@ def test_parse_early_newlines(): def test_parse_connect_shovel_over_newlines(): assert parse("\nsqlite://\ndest\n<<\nSELECT *\nFROM work", empty_config) == { "connection": "sqlite://", - "sql": "SELECT *\nFROM work", + "sql": "\nSELECT *\nFROM work", "result_var": "dest", "return_result_var": False, } From 18e48251bc9ad5a3df17b20194979f1fb98ac1df Mon Sep 17 00:00:00 2001 From: Anupam Date: Mon, 3 Jul 2023 08:20:34 -0400 Subject: [PATCH 18/23] added test and removed print --- src/sql/parse.py | 1 - src/tests/test_parse.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index a87613ee4..e305011fa 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -69,7 +69,6 @@ def parse(cell, config): result["sql"] = right else: result["sql"] = cell.strip(" ") - print(result) return result diff --git a/src/tests/test_parse.py b/src/tests/test_parse.py index f501de559..d1d594d9a 100644 --- a/src/tests/test_parse.py +++ b/src/tests/test_parse.py @@ -111,6 +111,8 @@ def test_parse_return_shovel_operator_with_equal(input_string, ip): "dest< Date: Tue, 11 Jul 2023 22:23:30 -0400 Subject: [PATCH 21/23] removed else --- src/sql/parse.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index e305011fa..71b2fbd43 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -52,12 +52,11 @@ def parse(cell, config): if len(pieces) == 1: return result cell = pieces[1] - else: - cell = cell + cell = cell.replace("\n", " ") pointer = cell.find("<<") if pointer != -1: - left = cell[:pointer].replace(" ", "").replace("\n", "") + left = cell[:pointer].replace(" ", "") right = cell[pointer + 2 :].strip(" ") if "=" in left: From bc585a364aa95849b64df01362ce1b3f0f9c942d Mon Sep 17 00:00:00 2001 From: Anupam Date: Tue, 11 Jul 2023 22:58:16 -0400 Subject: [PATCH 22/23] removed test --- src/sql/parse.py | 3 +-- src/tests/test_parse.py | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/sql/parse.py b/src/sql/parse.py index 71b2fbd43..780293b58 100644 --- a/src/sql/parse.py +++ b/src/sql/parse.py @@ -53,10 +53,9 @@ def parse(cell, config): return result cell = pieces[1] - cell = cell.replace("\n", " ") pointer = cell.find("<<") if pointer != -1: - left = cell[:pointer].replace(" ", "") + left = cell[:pointer].replace(" ", "").replace("\n", "") right = cell[pointer + 2 :].strip(" ") if "=" in left: diff --git a/src/tests/test_parse.py b/src/tests/test_parse.py index 4035ab1c1..d1d594d9a 100644 --- a/src/tests/test_parse.py +++ b/src/tests/test_parse.py @@ -113,8 +113,6 @@ def test_parse_return_shovel_operator_with_equal(input_string, ip): "dest << SELECT * FROM work", "dest <