From 8eca681701da6aadfccc1cb84b022f198d6f95cf Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Fri, 1 Jul 2022 12:36:24 +0200 Subject: [PATCH 01/12] add a new knob on whether to align newline comments with inline comments or not --- yapf/yapflib/reformatter.py | 48 +++++++++++++++++++++++++------------ yapf/yapflib/style.py | 6 +++++ 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index 14e0bde70..778c60b5e 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -277,7 +277,8 @@ def _AlignTrailingComments(final_lines): for tok in line.tokens: if (tok.is_comment and isinstance(tok.spaces_required_before, list) and tok.value.startswith('#')): - # All trailing comments and comments that appear on a line by themselves + # All trailing comments + # NOTE not including comments that appear on a line by themselves # in this block should be indented at the same level. The block is # terminated by an empty line or EOF. Enumerate through each line in # the block and calculate the max line length. Once complete, use the @@ -319,10 +320,20 @@ def _AlignTrailingComments(final_lines): whitespace_prefix = whitespace_prefix[newline_index + 1:] - if line_tok.is_comment: - pc_line_lengths.append(len(line_content)) + '''The part is added by Xiao on the top of original yapf code + because we don't want comments on newlines align with comments inline + ''' + if style.Get('ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS'): + if line_tok.is_comment: + pc_line_lengths.append(len(line_content)) + else: + line_content += '{}{}'.format(whitespace_prefix, line_tok.value) else: - line_content += '{}{}'.format(whitespace_prefix, line_tok.value) + if line_tok.is_comment and not line_tok.formatted_whitespace_prefix.startswith('\n'): + pc_line_lengths.append(len(line_content)) + else: + line_content += '{}{}'.format(whitespace_prefix, line_tok.value) + if pc_line_lengths: max_line_length = max(max_line_length, max(pc_line_lengths)) @@ -351,7 +362,7 @@ def _AlignTrailingComments(final_lines): pc_line_length_index = 0 for line_tok in this_line.tokens: - if line_tok.is_comment: + if line_tok.is_comment and '\n' not in line_tok.formatted_whitespace_prefix: assert pc_line_length_index < len(pc_line_lengths) assert pc_line_lengths[pc_line_length_index] < aligned_col @@ -360,19 +371,25 @@ def _AlignTrailingComments(final_lines): whitespace = ' ' * ( aligned_col - pc_line_lengths[pc_line_length_index] - 1) pc_line_length_index += 1 - - line_content = [] - - for comment_line_index, comment_line in enumerate( - line_tok.value.split('\n')): - line_content.append('{}{}'.format(whitespace, + + ''' this is added by Xiao because we don't want comments on newlines + to align with comments inline + ''' + if not style.Get('ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS'): + line_content = '{}{}'.format(whitespace, line_tok.value.strip()) + else: + line_content = [] + + for comment_line_index, comment_line in enumerate( + line_tok.value.split('\n')): + line_content.append('{}{}'.format(whitespace, comment_line.strip())) - if comment_line_index == 0: - whitespace = ' ' * (aligned_col - 1) - - line_content = '\n'.join(line_content) + if comment_line_index == 0: + whitespace = ' ' * (aligned_col - 1) + line_content = '\n'.join(line_content) + # Account for initial whitespace already slated for the # beginning of the line. existing_whitespace_prefix = \ @@ -394,6 +411,7 @@ def _AlignTrailingComments(final_lines): final_lines_index += 1 + def _FormatFinalLines(final_lines, verify): """Compose the final output from the finalized lines.""" formatted_code = [] diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py index 233a64e6b..9cf56c79a 100644 --- a/yapf/yapflib/style.py +++ b/yapf/yapflib/style.py @@ -54,6 +54,9 @@ def SetGlobalStyle(style): _STYLE_HELP = dict( ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=textwrap.dedent("""\ Align closing bracket with visual indentation."""), + ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS=textwrap.dedent("""\ + Align comments on newlines with the inline comments in the + same block. This is the default setting for yapf."""), ALLOW_MULTILINE_LAMBDAS=textwrap.dedent("""\ Allow lambdas to be formatted on more than one line."""), ALLOW_MULTILINE_DICTIONARY_KEYS=textwrap.dedent("""\ @@ -419,6 +422,7 @@ def CreatePEP8Style(): """Create the PEP8 formatting style.""" return dict( ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True, + ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS=False, ALLOW_MULTILINE_LAMBDAS=False, ALLOW_MULTILINE_DICTIONARY_KEYS=False, ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=True, @@ -509,6 +513,7 @@ def CreateYapfStyle(): style['SPLIT_BEFORE_BITWISE_OPERATOR'] = True style['SPLIT_BEFORE_DOT'] = True style['SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN'] = True + style['ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS'] = True return style @@ -607,6 +612,7 @@ def _IntOrIntListConverter(s): # Note: this dict has to map all the supported style options. _STYLE_OPTION_VALUE_CONVERTER = dict( ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=_BoolConverter, + ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS=_BoolConverter, ALLOW_MULTILINE_LAMBDAS=_BoolConverter, ALLOW_MULTILINE_DICTIONARY_KEYS=_BoolConverter, ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=_BoolConverter, From 30cda85061d097e39d156ad0ef13dfd9729176ee Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Tue, 19 Jul 2022 16:18:38 +0200 Subject: [PATCH 02/12] fix bugs and add contain_object --- yapf/yapflib/reformatter.py | 49 ++++++++++++++++++++++++------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index 778c60b5e..2256701cc 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -22,6 +22,7 @@ from __future__ import unicode_literals import collections +from distutils.errors import LinkError import heapq import re @@ -277,7 +278,7 @@ def _AlignTrailingComments(final_lines): for tok in line.tokens: if (tok.is_comment and isinstance(tok.spaces_required_before, list) and tok.value.startswith('#')): - # All trailing comments + # All trailing comments # NOTE not including comments that appear on a line by themselves # in this block should be indented at the same level. The block is # terminated by an empty line or EOF. Enumerate through each line in @@ -310,7 +311,15 @@ def _AlignTrailingComments(final_lines): line_content = '' pc_line_lengths = [] + #NOTE added by Xiao + contain_object = False for line_tok in this_line.tokens: + + #NOTE added by Xiao + if (line_tok.value in [')', ']','}'] + and line_tok.formatted_whitespace_prefix.startswith('\n')): + contain_object = True + whitespace_prefix = line_tok.formatted_whitespace_prefix newline_index = whitespace_prefix.rfind('\n') @@ -323,23 +332,23 @@ def _AlignTrailingComments(final_lines): '''The part is added by Xiao on the top of original yapf code because we don't want comments on newlines align with comments inline ''' - if style.Get('ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS'): - if line_tok.is_comment: - pc_line_lengths.append(len(line_content)) - else: - line_content += '{}{}'.format(whitespace_prefix, line_tok.value) + # if comment starts with '\n', it will save length 0 + if line_tok.is_comment: + pc_line_lengths.append(len(line_content)) else: - if line_tok.is_comment and not line_tok.formatted_whitespace_prefix.startswith('\n'): - pc_line_lengths.append(len(line_content)) - else: - line_content += '{}{}'.format(whitespace_prefix, line_tok.value) - + line_content += '{}{}'.format(whitespace_prefix, line_tok.value) if pc_line_lengths: max_line_length = max(max_line_length, max(pc_line_lengths)) all_pc_line_lengths.append(pc_line_lengths) + #NOTE added by Xiao + # if it's a logical line with object(dict/list/tuple) + # that have its items in separate lines + if contain_object: + break + # Calculate the aligned column value max_line_length += 2 @@ -362,7 +371,7 @@ def _AlignTrailingComments(final_lines): pc_line_length_index = 0 for line_tok in this_line.tokens: - if line_tok.is_comment and '\n' not in line_tok.formatted_whitespace_prefix: + if line_tok.is_comment: assert pc_line_length_index < len(pc_line_lengths) assert pc_line_lengths[pc_line_length_index] < aligned_col @@ -370,14 +379,18 @@ def _AlignTrailingComments(final_lines): # we need to apply a whitespace prefix to each line. whitespace = ' ' * ( aligned_col - pc_line_lengths[pc_line_length_index] - 1) - pc_line_length_index += 1 - + + ''' this is added by Xiao because we don't want comments on newlines to align with comments inline ''' if not style.Get('ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS'): + # if this comment starts with '\n', pass and go to next comment + if pc_line_lengths[pc_line_length_index] == 0: + pc_line_length_index += 1 + continue line_content = '{}{}'.format(whitespace, line_tok.value.strip()) - else: + else: line_content = [] for comment_line_index, comment_line in enumerate( @@ -389,7 +402,10 @@ def _AlignTrailingComments(final_lines): whitespace = ' ' * (aligned_col - 1) line_content = '\n'.join(line_content) - + + # after process, go to next pre comment tokens length + pc_line_length_index += 1 + # Account for initial whitespace already slated for the # beginning of the line. existing_whitespace_prefix = \ @@ -411,7 +427,6 @@ def _AlignTrailingComments(final_lines): final_lines_index += 1 - def _FormatFinalLines(final_lines, verify): """Compose the final output from the finalized lines.""" formatted_code = [] From ff058ffcca4a4456dad4215f7726d43e601407e3 Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Tue, 26 Jul 2022 11:04:30 +0200 Subject: [PATCH 03/12] fix the dedentation problem of newline comments inside lline --- yapf/pytree/comment_splicer.py | 1 - yapf/yapflib/format_decision_state.py | 6 ++++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/yapf/pytree/comment_splicer.py b/yapf/pytree/comment_splicer.py index ae5ffe66f..9e8f02c48 100644 --- a/yapf/pytree/comment_splicer.py +++ b/yapf/pytree/comment_splicer.py @@ -42,7 +42,6 @@ def SpliceComments(tree): # This is a list because Python 2.x doesn't have 'nonlocal' :) prev_leaf = [None] _AnnotateIndents(tree) - def _VisitNodeRec(node): """Recursively visit each node to splice comments into the AST.""" # This loop may insert into node.children, so we'll iterate over a copy. diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index c299d1c85..5b785c157 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -644,6 +644,12 @@ def _AddTokenOnNewline(self, dry_run, must_split): if not dry_run: indent_level = self.line.depth + #-----------------------NOTE----------------------------------- + # we want newline comments inside the lline to + # keep their original indentations + if current.is_comment: + indent_level = int(current.column/style.Get('INDENT_WIDTH')) + #---------------------Added by Xiao---------------------------- spaces = self.column if spaces: spaces -= indent_level * style.Get('INDENT_WIDTH') From 7d52ced8fefb62483a154d728324abfc6c8510af Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Fri, 29 Jul 2022 16:09:59 +0200 Subject: [PATCH 04/12] update --- yapf/yapflib/format_decision_state.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 5b785c157..4b5b7b395 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -644,7 +644,7 @@ def _AddTokenOnNewline(self, dry_run, must_split): if not dry_run: indent_level = self.line.depth - #-----------------------NOTE----------------------------------- + #-----------------------NOTE below----------------------------------- # we want newline comments inside the lline to # keep their original indentations if current.is_comment: From da36c9da23d4e520725ed6b58dc9033fc7a1e69d Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Tue, 16 Aug 2022 11:08:54 +0200 Subject: [PATCH 05/12] fixed existing spaces problem and pseudo paran problem in dictionary values and added tests --- yapf/yapflib/format_decision_state.py | 6 --- yapf/yapflib/reformatter.py | 22 ++++----- yapf/yapflib/style.py | 5 +-- yapftests/yapf_test.py | 64 ++++++++++++++++++++++++++- 4 files changed, 76 insertions(+), 21 deletions(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index 4b5b7b395..c299d1c85 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -644,12 +644,6 @@ def _AddTokenOnNewline(self, dry_run, must_split): if not dry_run: indent_level = self.line.depth - #-----------------------NOTE below----------------------------------- - # we want newline comments inside the lline to - # keep their original indentations - if current.is_comment: - indent_level = int(current.column/style.Get('INDENT_WIDTH')) - #---------------------Added by Xiao---------------------------- spaces = self.column if spaces: spaces -= indent_level * style.Get('INDENT_WIDTH') diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index 2256701cc..93198a9d0 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -311,11 +311,12 @@ def _AlignTrailingComments(final_lines): line_content = '' pc_line_lengths = [] - #NOTE added by Xiao + #NOTE contain_object = False for line_tok in this_line.tokens: - #NOTE added by Xiao + #NOTE if a line with inline comment is itself + # with newlines object, we want to start new alignment if (line_tok.value in [')', ']','}'] and line_tok.formatted_whitespace_prefix.startswith('\n')): contain_object = True @@ -329,13 +330,10 @@ def _AlignTrailingComments(final_lines): whitespace_prefix = whitespace_prefix[newline_index + 1:] - '''The part is added by Xiao on the top of original yapf code - because we don't want comments on newlines align with comments inline - ''' # if comment starts with '\n', it will save length 0 if line_tok.is_comment: pc_line_lengths.append(len(line_content)) - else: + elif not line_tok.is_pseudo: line_content += '{}{}'.format(whitespace_prefix, line_tok.value) if pc_line_lengths: @@ -343,8 +341,7 @@ def _AlignTrailingComments(final_lines): all_pc_line_lengths.append(pc_line_lengths) - #NOTE added by Xiao - # if it's a logical line with object(dict/list/tuple) + #NOTE if it's a logical line with object(dict/list/tuple) # that have its items in separate lines if contain_object: break @@ -381,7 +378,7 @@ def _AlignTrailingComments(final_lines): aligned_col - pc_line_lengths[pc_line_length_index] - 1) - ''' this is added by Xiao because we don't want comments on newlines + ''' this is added when we don't want comments on newlines to align with comments inline ''' if not style.Get('ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS'): @@ -410,8 +407,11 @@ def _AlignTrailingComments(final_lines): # beginning of the line. existing_whitespace_prefix = \ line_tok.formatted_whitespace_prefix.lstrip('\n') - - if line_content.startswith(existing_whitespace_prefix): + # in case that the existing spaces larger than spaces that needed to pad + if (len(whitespace) == 1 or len(whitespace) > 1 and + len(existing_whitespace_prefix)>len(whitespace)): + line_tok.whitespace_prefix = '' + elif line_content.startswith(existing_whitespace_prefix): line_content = line_content[len(existing_whitespace_prefix):] line_tok.value = line_content diff --git a/yapf/yapflib/style.py b/yapf/yapflib/style.py index 9cf56c79a..e861ed101 100644 --- a/yapf/yapflib/style.py +++ b/yapf/yapflib/style.py @@ -55,7 +55,7 @@ def SetGlobalStyle(style): ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=textwrap.dedent("""\ Align closing bracket with visual indentation."""), ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS=textwrap.dedent("""\ - Align comments on newlines with the inline comments in the + Align comments on newlines with the inline comments in the same block. This is the default setting for yapf."""), ALLOW_MULTILINE_LAMBDAS=textwrap.dedent("""\ Allow lambdas to be formatted on more than one line."""), @@ -422,7 +422,7 @@ def CreatePEP8Style(): """Create the PEP8 formatting style.""" return dict( ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT=True, - ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS=False, + ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS=True, ALLOW_MULTILINE_LAMBDAS=False, ALLOW_MULTILINE_DICTIONARY_KEYS=False, ALLOW_SPLIT_BEFORE_DEFAULT_OR_NAMED_ASSIGNS=True, @@ -513,7 +513,6 @@ def CreateYapfStyle(): style['SPLIT_BEFORE_BITWISE_OPERATOR'] = True style['SPLIT_BEFORE_DOT'] = True style['SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN'] = True - style['ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS'] = True return style diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py index 2330f4e18..285d23c7d 100644 --- a/yapftests/yapf_test.py +++ b/yapftests/yapf_test.py @@ -26,7 +26,7 @@ from lib2to3.pgen2 import tokenize -from yapf.yapflib import errors +from yapf.yapflib import errors, reformatter from yapf.yapflib import py3compat from yapf.yapflib import style from yapf.yapflib import yapf_api @@ -1875,6 +1875,68 @@ def testDisabledLine(self): """) self._Check(unformatted_code, expected_formatted_code) + #NOTE test if don't align newline comments with inline comments + def testNewlineCommentsInsideInlineComment(self): + try: + style.SetGlobalStyle( + style.CreateStyleFromConfig('{align_newline_comments_with_inline_comments:false,' + 'spaces_before_comment:15, 25,35}')) + unformatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + func(1) # comment 1 + func(2) # comment 2 + # comment 3 + func(3) # comment 4 inline + # comment 4 newline + # comment 4 newline + + # comment 5 Not aligned + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + func(1) # comment 1 + func(2) # comment 2 + # comment 3 + func(3) # comment 4 inline + # comment 4 newline + # comment 4 newline + + # comment 5 Not aligned + """) + llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) + self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) + finally: + style.SetGlobalStyle(self._OwnStyle()) + + # test when there is an object with newline entries in between + def testObjectWithNewlineEntriesInBetween(self): + + unformatted_code = textwrap.dedent("""\ + func( 1 ) # Line 1 + func( 2 ) # Line 2 + d = {key1: value1, key2: value2, key3: value3} # Line 3 +func( 3 ) # Line 4 +func( 4 ) # line 5 + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: + if True: + if True: + func(1) # comment 1 + func(2) # comment 2 + # comment 3 + func(3) # comment 4 inline + # comment 4 newline + # comment 4 newline + + # comment 5 Not aligned + """) + + class _SpacesAroundDictListTupleTestImpl(unittest.TestCase): From 881e8ca64cc65223c22c67ccc80c6faf448deebf Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Tue, 16 Aug 2022 14:28:03 +0200 Subject: [PATCH 06/12] test update for comment alignment --- yapftests/yapf_test.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py index 285d23c7d..bbde64604 100644 --- a/yapftests/yapf_test.py +++ b/yapftests/yapf_test.py @@ -1919,23 +1919,21 @@ def testObjectWithNewlineEntriesInBetween(self): func( 1 ) # Line 1 func( 2 ) # Line 2 d = {key1: value1, key2: value2, key3: value3} # Line 3 -func( 3 ) # Line 4 -func( 4 ) # line 5 + func( 3 ) # Line 4 + func( 4 ) # line 5 """) # noqa expected_formatted_code = textwrap.dedent("""\ - if True: - if True: - if True: - func(1) # comment 1 - func(2) # comment 2 - # comment 3 - func(3) # comment 4 inline - # comment 4 newline - # comment 4 newline - - # comment 5 Not aligned + func( 1 ) # Line 1 + func( 2 ) # Line 2 + d = { + key1: value1, + key2: value2, + key3: value3, + } # Line 3 + func( 3 ) # Line 4 + func( 4 ) # line 5 """) - + self._Check(unformatted_code, expected_formatted_code) class _SpacesAroundDictListTupleTestImpl(unittest.TestCase): From d2c3d835e3ea36c8d6cf5443850f1cae5a1914c2 Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Tue, 16 Aug 2022 15:22:34 +0200 Subject: [PATCH 07/12] fix the test in yapf_test.py --- yapftests/yapf_test.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py index bbde64604..dd919db7f 100644 --- a/yapftests/yapf_test.py +++ b/yapftests/yapf_test.py @@ -1918,20 +1918,20 @@ def testObjectWithNewlineEntriesInBetween(self): unformatted_code = textwrap.dedent("""\ func( 1 ) # Line 1 func( 2 ) # Line 2 - d = {key1: value1, key2: value2, key3: value3} # Line 3 + d = {key1: value1, key2: value2, key3: value3,} # Line 3 func( 3 ) # Line 4 func( 4 ) # line 5 """) # noqa expected_formatted_code = textwrap.dedent("""\ - func( 1 ) # Line 1 - func( 2 ) # Line 2 + func(1) # Line 1 + func(2) # Line 2 d = { key1: value1, key2: value2, key3: value3, } # Line 3 - func( 3 ) # Line 4 - func( 4 ) # line 5 + func(3) # Line 4 + func(4) # line 5 """) self._Check(unformatted_code, expected_formatted_code) From 55b21519ff772ac6ef6a55b59e0b206ca0845add Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Thu, 18 Aug 2022 18:02:51 +0200 Subject: [PATCH 08/12] fix the existing spaces before again --- yapf/yapflib/reformatter.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index 93198a9d0..eec03d2f4 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -389,14 +389,14 @@ def _AlignTrailingComments(final_lines): line_content = '{}{}'.format(whitespace, line_tok.value.strip()) else: line_content = [] - + padded_space = whitespace for comment_line_index, comment_line in enumerate( line_tok.value.split('\n')): - line_content.append('{}{}'.format(whitespace, + line_content.append('{}{}'.format(padded_space, comment_line.strip())) if comment_line_index == 0: - whitespace = ' ' * (aligned_col - 1) + padded_space = ' ' * (aligned_col - 1) line_content = '\n'.join(line_content) @@ -407,9 +407,9 @@ def _AlignTrailingComments(final_lines): # beginning of the line. existing_whitespace_prefix = \ line_tok.formatted_whitespace_prefix.lstrip('\n') - # in case that the existing spaces larger than spaces that needed to pad - if (len(whitespace) == 1 or len(whitespace) > 1 and - len(existing_whitespace_prefix)>len(whitespace)): + # in case that the existing spaces larger than + # spaces that needed to pad, set the whitespace_prefix to empty + if len(existing_whitespace_prefix)>len(whitespace): line_tok.whitespace_prefix = '' elif line_content.startswith(existing_whitespace_prefix): line_content = line_content[len(existing_whitespace_prefix):] From a592e8d34667b74bd413df409b585d4bc8ad8b97 Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Thu, 18 Aug 2022 18:16:21 +0200 Subject: [PATCH 09/12] update the new knob and a change in inline comments alignment --- yapf/yapflib/reformatter.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index eec03d2f4..1f0868a9b 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -333,7 +333,7 @@ def _AlignTrailingComments(final_lines): # if comment starts with '\n', it will save length 0 if line_tok.is_comment: pc_line_lengths.append(len(line_content)) - elif not line_tok.is_pseudo: + else: line_content += '{}{}'.format(whitespace_prefix, line_tok.value) if pc_line_lengths: @@ -389,14 +389,13 @@ def _AlignTrailingComments(final_lines): line_content = '{}{}'.format(whitespace, line_tok.value.strip()) else: line_content = [] - padded_space = whitespace for comment_line_index, comment_line in enumerate( line_tok.value.split('\n')): - line_content.append('{}{}'.format(padded_space, + line_content.append('{}{}'.format(whitespace, comment_line.strip())) if comment_line_index == 0: - padded_space = ' ' * (aligned_col - 1) + whitespace = ' ' * (aligned_col - 1) line_content = '\n'.join(line_content) @@ -407,11 +406,8 @@ def _AlignTrailingComments(final_lines): # beginning of the line. existing_whitespace_prefix = \ line_tok.formatted_whitespace_prefix.lstrip('\n') - # in case that the existing spaces larger than - # spaces that needed to pad, set the whitespace_prefix to empty - if len(existing_whitespace_prefix)>len(whitespace): - line_tok.whitespace_prefix = '' - elif line_content.startswith(existing_whitespace_prefix): + + if line_content.startswith(existing_whitespace_prefix): line_content = line_content[len(existing_whitespace_prefix):] line_tok.value = line_content From da9e537124dd472afc24116b8d69c48dfe8f6810 Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Fri, 19 Aug 2022 18:17:17 +0200 Subject: [PATCH 10/12] fix the dedentation problem of standalone newline comments inside a logical line --- yapf/yapflib/format_decision_state.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index c299d1c85..a72176bec 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -934,16 +934,19 @@ def _GetNewlineColumn(self): previous = current.previous_token top_of_stack = self.stack[-1] + cont_aligned_indent = self._IndentWithContinuationAlignStyle( + top_of_stack.indent) + if isinstance(current.spaces_required_before, list): - # Don't set the value here, as we need to look at the lines near - # this one to determine the actual horizontal alignment value. - return 0 + # only when the commet is not inside an object(list, dictionary, + # function call)logical line that is in many output lines + if self.paren_level == 0: + # Don't set the value here, as we need to look at the lines near + # this one to determine the actual horizontal alignment value. + return 0 elif current.spaces_required_before > 2 or self.line.disable: return current.spaces_required_before - cont_aligned_indent = self._IndentWithContinuationAlignStyle( - top_of_stack.indent) - if current.OpensScope(): return cont_aligned_indent if self.paren_level else self.first_indent From a6075c3a6000720aa7917a0e130a16639487231d Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Tue, 30 Aug 2022 10:26:57 +0200 Subject: [PATCH 11/12] update changelog and readme --- CHANGELOG | 10 +++++ README.rst | 13 ++++++ yapf/pytree/comment_splicer.py | 1 + yapf/yapflib/format_decision_state.py | 5 ++- yapf/yapflib/reformatter.py | 14 +++--- yapftests/yapf_test.py | 63 +++++++++++++-------------- 6 files changed, 64 insertions(+), 42 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 5ac3e6329..86013442e 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,6 +2,16 @@ # All notable changes to this project will be documented in this file. # This project adheres to [Semantic Versioning](http://semver.org/). +## [0.41.0] 2022-08-30 +### Added +- Add a new knob align_newline_comments_with_inline_comments. This can + be used when you don't want to align newline comments with inline comments. +### Changes +- Make the comments to start new alignment after multiline objects like + dictionaries, lists or function calls or function definitions. +- changes made to make the newline comments inside the multiline objects + not to indent as its parent level. + ## [0.40.0] UNRELEASED ### Added - Add a new Python parser to generate logical lines. diff --git a/README.rst b/README.rst index 5734a5d76..b0bdd3911 100644 --- a/README.rst +++ b/README.rst @@ -390,6 +390,19 @@ Options:: Knobs ===== +``ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS`` + Align newline comments with the inline comments. The default setting is True. + If it is set to be False, new lines comments will not align with inline comments, + as following: + + .. code-block:: python + def f(): + result = { + "a": 1, # comment inline + # comment newline + "abc": 2 + } + ``ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT`` Align closing bracket with visual indentation. diff --git a/yapf/pytree/comment_splicer.py b/yapf/pytree/comment_splicer.py index 9e8f02c48..ae5ffe66f 100644 --- a/yapf/pytree/comment_splicer.py +++ b/yapf/pytree/comment_splicer.py @@ -42,6 +42,7 @@ def SpliceComments(tree): # This is a list because Python 2.x doesn't have 'nonlocal' :) prev_leaf = [None] _AnnotateIndents(tree) + def _VisitNodeRec(node): """Recursively visit each node to splice comments into the AST.""" # This loop may insert into node.children, so we'll iterate over a copy. diff --git a/yapf/yapflib/format_decision_state.py b/yapf/yapflib/format_decision_state.py index a72176bec..b2c961384 100644 --- a/yapf/yapflib/format_decision_state.py +++ b/yapf/yapflib/format_decision_state.py @@ -938,8 +938,9 @@ def _GetNewlineColumn(self): top_of_stack.indent) if isinstance(current.spaces_required_before, list): - # only when the commet is not inside an object(list, dictionary, - # function call)logical line that is in many output lines + # only when the commet is not inside an object logical line that has + # its entries output on separate output lines(e.g. list, dictionary, + # function call), aka when the comment' parent level is 0 if self.paren_level == 0: # Don't set the value here, as we need to look at the lines near # this one to determine the actual horizontal alignment value. diff --git a/yapf/yapflib/reformatter.py b/yapf/yapflib/reformatter.py index 1f0868a9b..7adcd2a41 100644 --- a/yapf/yapflib/reformatter.py +++ b/yapf/yapflib/reformatter.py @@ -278,13 +278,14 @@ def _AlignTrailingComments(final_lines): for tok in line.tokens: if (tok.is_comment and isinstance(tok.spaces_required_before, list) and tok.value.startswith('#')): - # All trailing comments - # NOTE not including comments that appear on a line by themselves + # All trailing comments and comments that appear on a line by themselves # in this block should be indented at the same level. The block is # terminated by an empty line or EOF. Enumerate through each line in # the block and calculate the max line length. Once complete, use the # first col value greater than that value and create the necessary for # each line accordingly. + # NOTE comments that appear on a line by themselves will be excluded if + # align_newline_comments_with_inline_comments is false. all_pc_line_lengths = [] # All pre-comment line lengths max_line_length = 0 @@ -311,11 +312,10 @@ def _AlignTrailingComments(final_lines): line_content = '' pc_line_lengths = [] - #NOTE contain_object = False for line_tok in this_line.tokens: - #NOTE if a line with inline comment is itself + #if a line with inline comment is itself # with newlines object, we want to start new alignment if (line_tok.value in [')', ']','}'] and line_tok.formatted_whitespace_prefix.startswith('\n')): @@ -377,10 +377,8 @@ def _AlignTrailingComments(final_lines): whitespace = ' ' * ( aligned_col - pc_line_lengths[pc_line_length_index] - 1) - - ''' this is added when we don't want comments on newlines - to align with comments inline - ''' + #this is added when we don't want comments on newlines + #to align with comments inline if not style.Get('ALIGN_NEWLINE_COMMENTS_WITH_INLINE_COMMENTS'): # if this comment starts with '\n', pass and go to next comment if pc_line_lengths[pc_line_length_index] == 0: diff --git a/yapftests/yapf_test.py b/yapftests/yapf_test.py index dd919db7f..60aa5e99a 100644 --- a/yapftests/yapf_test.py +++ b/yapftests/yapf_test.py @@ -26,7 +26,7 @@ from lib2to3.pgen2 import tokenize -from yapf.yapflib import errors, reformatter +from yapf.yapflib import errors from yapf.yapflib import py3compat from yapf.yapflib import style from yapf.yapflib import yapf_api @@ -1875,42 +1875,41 @@ def testDisabledLine(self): """) self._Check(unformatted_code, expected_formatted_code) - #NOTE test if don't align newline comments with inline comments + # test if don't align newline comments with inline comments def testNewlineCommentsInsideInlineComment(self): - try: - style.SetGlobalStyle( - style.CreateStyleFromConfig('{align_newline_comments_with_inline_comments:false,' - 'spaces_before_comment:15, 25,35}')) - unformatted_code = textwrap.dedent("""\ + unformatted_code = textwrap.dedent("""\ + if True: if True: if True: - if True: - func(1) # comment 1 - func(2) # comment 2 - # comment 3 - func(3) # comment 4 inline - # comment 4 newline - # comment 4 newline - - # comment 5 Not aligned - """) # noqa - expected_formatted_code = textwrap.dedent("""\ + func(1) # comment 1 + func(2) # comment 2 + # comment 3 + func(3) # comment 4 inline + # comment 4 newline + # comment 4 newline + + # comment 5 Not aligned + """) # noqa + expected_formatted_code = textwrap.dedent("""\ + if True: if True: if True: - if True: - func(1) # comment 1 - func(2) # comment 2 - # comment 3 - func(3) # comment 4 inline - # comment 4 newline - # comment 4 newline - - # comment 5 Not aligned - """) - llines = yapf_test_helper.ParseAndUnwrap(unformatted_code) - self.assertCodeEqual(expected_formatted_code, reformatter.Reformat(llines)) - finally: - style.SetGlobalStyle(self._OwnStyle()) + func(1) # comment 1 + func(2) # comment 2 + # comment 3 + func(3) # comment 4 inline + # comment 4 newline + # comment 4 newline + + # comment 5 Not aligned + """) + + formatted_code, _ = yapf_api.FormatCode( + unformatted_code, style_config=style.SetGlobalStyle(style.CreateStyleFromConfig( + '{align_newline_comments_with_inline_comments:false,' + 'spaces_before_comment:15, 25,35}'))) + self.assertCodeEqual(expected_formatted_code, formatted_code) + # test when there is an object with newline entries in between def testObjectWithNewlineEntriesInBetween(self): From a1722a77e35aca90cd319fa4890abfe96e882d00 Mon Sep 17 00:00:00 2001 From: Xiao Wang Date: Tue, 30 Aug 2022 14:08:29 +0200 Subject: [PATCH 12/12] fix readme --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index b0bdd3911..1392c65d6 100644 --- a/README.rst +++ b/README.rst @@ -396,6 +396,7 @@ Knobs as following: .. code-block:: python + def f(): result = { "a": 1, # comment inline