From 740c0db7d2de96de3611895831313ec98e0780ed Mon Sep 17 00:00:00 2001 From: skryzh Date: Thu, 17 Oct 2024 18:46:07 +0300 Subject: [PATCH 1/4] [fix] fixing SQL-Hint deleting when clearing script from comments --- .gitignore | 4 +++- sqlparse/filters/others.py | 35 +++++++++++++++++++++++++++++------ tests/test_format.py | 30 ++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index 77479f17..23b78534 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,6 @@ MANIFEST .cache/ *.egg-info/ htmlcov/ -.pytest_cache \ No newline at end of file +.pytest_cache +**/.vscode +**/.env diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py index 3388a782..568a8d0e 100644 --- a/sqlparse/filters/others.py +++ b/sqlparse/filters/others.py @@ -15,9 +15,9 @@ class StripCommentsFilter: @staticmethod def _process(tlist): - def get_next_comment(): + def get_next_comment(idx=-1): # TODO(andi) Comment types should be unified, see related issue38 - return tlist.token_next_by(i=sql.Comment, t=T.Comment) + return tlist.token_next_by(i=sql.Comment, t=T.Comment, idx=idx) def _get_insert_token(token): """Returns either a whitespace or the line breaks from token.""" @@ -31,15 +31,35 @@ def _get_insert_token(token): else: return sql.Token(T.Whitespace, ' ') + sql_hints = (T.Comment.Multiline.Hint, T.Comment.Single.Hint) tidx, token = get_next_comment() while token: + # skipping token remove if token is a SQL hint + is_sql_hint = False + if token.ttype in sql_hints: + is_sql_hint = True + elif isinstance(token, sql.Comment): + comment_tokens = token.tokens + if len(comment_tokens) > 0: + if comment_tokens[0].ttype in sql_hints: + is_sql_hint = True + + if is_sql_hint: + # using current index as start index to search next token for + # preventing infinite loop in cases when token type is a + # "SQL-Hint"and has to be skipped + tidx, token = get_next_comment(idx=tidx) + continue + pidx, prev_ = tlist.token_prev(tidx, skip_ws=False) nidx, next_ = tlist.token_next(tidx, skip_ws=False) # Replace by whitespace if prev and next exist and if they're not # whitespaces. This doesn't apply if prev or next is a parenthesis. - if (prev_ is None or next_ is None - or prev_.is_whitespace or prev_.match(T.Punctuation, '(') - or next_.is_whitespace or next_.match(T.Punctuation, ')')): + if ( + prev_ is None or next_ is None + or prev_.is_whitespace or prev_.match(T.Punctuation, '(') + or next_.is_whitespace or next_.match(T.Punctuation, ')') + ): # Insert a whitespace to ensure the following SQL produces # a valid SQL (see #425). if prev_ is not None and not prev_.match(T.Punctuation, '('): @@ -48,7 +68,10 @@ def _get_insert_token(token): else: tlist.tokens[tidx] = _get_insert_token(token) - tidx, token = get_next_comment() + # using current index as start index to search next token for + # preventing infinite loop in cases when token type is a + # "SQL-Hint"and has to be skipped + tidx, token = get_next_comment(idx=tidx) def process(self, stmt): [self.process(sgroup) for sgroup in stmt.get_sublists()] diff --git a/tests/test_format.py b/tests/test_format.py index df94630c..26cb5edd 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -61,6 +61,25 @@ def test_strip_comments_single(self): 'from foo--comment\nf' res = sqlparse.format(sql, strip_comments=True) assert res == 'select a\nfrom foo\nf' + # SQL-Hints have to be preserved + sql = 'select --+full(u)' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + sql = '#+ hint\nselect * from foo' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + + sql = 'select --+full(u)\n--comment simple' + res = sqlparse.format(sql, strip_comments=True) + assert res == 'select --+full(u)\n' + + sql = '#+ hint\nselect * from foo\n# comment simple' + res = sqlparse.format(sql, strip_comments=True) + assert res == '#+ hint\nselect * from foo\n' + + # sql = '' + # res = sqlparse.format(sql, strip_comments=True) + # assert res == '' def test_strip_comments_invalid_option(self): sql = 'select-- foo\nfrom -- bar\nwhere' @@ -83,6 +102,17 @@ def test_strip_comments_multi(self): sql = 'select (/* sql /* starts here */ select 2)' res = sqlparse.format(sql, strip_comments=True, strip_whitespace=True) assert res == 'select (select 2)' + # SQL-Hints have to be preserved + sql = 'SELECT /*+cluster(T)*/* FROM T_EEE T where A >:1' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + sql = 'insert /*+ DIRECT */ into sch.table_name as select * from foo' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + + sql = '' + res = sqlparse.format(sql, strip_comments=True) + assert res == '' def test_strip_comments_preserves_linebreak(self): sql = 'select * -- a comment\r\nfrom foo' From bf68cb12d022df91bf42490ffaf15aeae821925b Mon Sep 17 00:00:00 2001 From: skryzh Date: Fri, 18 Oct 2024 14:50:11 +0300 Subject: [PATCH 2/4] [fix] issue262 adding comment and tests --- sqlparse/filters/others.py | 2 +- tests/test_format.py | 10 ---------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py index 568a8d0e..0aa28fb1 100644 --- a/sqlparse/filters/others.py +++ b/sqlparse/filters/others.py @@ -34,7 +34,7 @@ def _get_insert_token(token): sql_hints = (T.Comment.Multiline.Hint, T.Comment.Single.Hint) tidx, token = get_next_comment() while token: - # skipping token remove if token is a SQL hint + # skipping token remove if token is a SQL-Hint. issue262 is_sql_hint = False if token.ttype in sql_hints: is_sql_hint = True diff --git a/tests/test_format.py b/tests/test_format.py index 26cb5edd..2ef3358a 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -68,19 +68,13 @@ def test_strip_comments_single(self): sql = '#+ hint\nselect * from foo' res = sqlparse.format(sql, strip_comments=True) assert res == sql - sql = 'select --+full(u)\n--comment simple' res = sqlparse.format(sql, strip_comments=True) assert res == 'select --+full(u)\n' - sql = '#+ hint\nselect * from foo\n# comment simple' res = sqlparse.format(sql, strip_comments=True) assert res == '#+ hint\nselect * from foo\n' - # sql = '' - # res = sqlparse.format(sql, strip_comments=True) - # assert res == '' - def test_strip_comments_invalid_option(self): sql = 'select-- foo\nfrom -- bar\nwhere' with pytest.raises(SQLParseError): @@ -110,10 +104,6 @@ def test_strip_comments_multi(self): res = sqlparse.format(sql, strip_comments=True) assert res == sql - sql = '' - res = sqlparse.format(sql, strip_comments=True) - assert res == '' - def test_strip_comments_preserves_linebreak(self): sql = 'select * -- a comment\r\nfrom foo' res = sqlparse.format(sql, strip_comments=True) From a09e295c451446063612af8fb1de6385e74b77c9 Mon Sep 17 00:00:00 2001 From: skryzh Date: Mon, 28 Oct 2024 11:44:46 +0300 Subject: [PATCH 3/4] [feature] deleting envs from .gitignore, creating test for SQL-Hints --- .gitignore | 2 -- sqlparse/filters/others.py | 4 ++-- tests/test_format.py | 40 +++++++++++++++++++------------------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 23b78534..b8b00490 100644 --- a/.gitignore +++ b/.gitignore @@ -8,5 +8,3 @@ MANIFEST *.egg-info/ htmlcov/ .pytest_cache -**/.vscode -**/.env diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py index 0aa28fb1..6c1680ec 100644 --- a/sqlparse/filters/others.py +++ b/sqlparse/filters/others.py @@ -47,7 +47,7 @@ def _get_insert_token(token): if is_sql_hint: # using current index as start index to search next token for # preventing infinite loop in cases when token type is a - # "SQL-Hint"and has to be skipped + # "SQL-Hint" and has to be skipped tidx, token = get_next_comment(idx=tidx) continue @@ -70,7 +70,7 @@ def _get_insert_token(token): # using current index as start index to search next token for # preventing infinite loop in cases when token type is a - # "SQL-Hint"and has to be skipped + # "SQL-Hint" and has to be skipped tidx, token = get_next_comment(idx=tidx) def process(self, stmt): diff --git a/tests/test_format.py b/tests/test_format.py index 2ef3358a..4dae299d 100644 --- a/tests/test_format.py +++ b/tests/test_format.py @@ -61,19 +61,6 @@ def test_strip_comments_single(self): 'from foo--comment\nf' res = sqlparse.format(sql, strip_comments=True) assert res == 'select a\nfrom foo\nf' - # SQL-Hints have to be preserved - sql = 'select --+full(u)' - res = sqlparse.format(sql, strip_comments=True) - assert res == sql - sql = '#+ hint\nselect * from foo' - res = sqlparse.format(sql, strip_comments=True) - assert res == sql - sql = 'select --+full(u)\n--comment simple' - res = sqlparse.format(sql, strip_comments=True) - assert res == 'select --+full(u)\n' - sql = '#+ hint\nselect * from foo\n# comment simple' - res = sqlparse.format(sql, strip_comments=True) - assert res == '#+ hint\nselect * from foo\n' def test_strip_comments_invalid_option(self): sql = 'select-- foo\nfrom -- bar\nwhere' @@ -96,13 +83,6 @@ def test_strip_comments_multi(self): sql = 'select (/* sql /* starts here */ select 2)' res = sqlparse.format(sql, strip_comments=True, strip_whitespace=True) assert res == 'select (select 2)' - # SQL-Hints have to be preserved - sql = 'SELECT /*+cluster(T)*/* FROM T_EEE T where A >:1' - res = sqlparse.format(sql, strip_comments=True) - assert res == sql - sql = 'insert /*+ DIRECT */ into sch.table_name as select * from foo' - res = sqlparse.format(sql, strip_comments=True) - assert res == sql def test_strip_comments_preserves_linebreak(self): sql = 'select * -- a comment\r\nfrom foo' @@ -126,6 +106,26 @@ def test_strip_comments_preserves_whitespace(self): res = sqlparse.format(sql, strip_comments=True) assert res == 'SELECT 1 AS foo' + def test_strip_comments_preserves_hint(self): + sql = 'select --+full(u)' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + sql = '#+ hint\nselect * from foo' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + sql = 'select --+full(u)\n--comment simple' + res = sqlparse.format(sql, strip_comments=True) + assert res == 'select --+full(u)\n' + sql = '#+ hint\nselect * from foo\n# comment simple' + res = sqlparse.format(sql, strip_comments=True) + assert res == '#+ hint\nselect * from foo\n' + sql = 'SELECT /*+cluster(T)*/* FROM T_EEE T where A >:1' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + sql = 'insert /*+ DIRECT */ into sch.table_name as select * from foo' + res = sqlparse.format(sql, strip_comments=True) + assert res == sql + def test_strip_ws(self): f = lambda sql: sqlparse.format(sql, strip_whitespace=True) s = 'select\n* from foo\n\twhere ( 1 = 2 )\n' From b5e0e2a62af0a7b8b8cb5d8a135f1033893b5fd7 Mon Sep 17 00:00:00 2001 From: skryzh Date: Mon, 28 Oct 2024 11:53:19 +0300 Subject: [PATCH 4/4] [feature] returning .gitignore to previous version --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index b8b00490..77479f17 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,4 @@ MANIFEST .cache/ *.egg-info/ htmlcov/ -.pytest_cache +.pytest_cache \ No newline at end of file