content: # Copyright 2015 Google Inc. All Rights Reserved.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  """Implements a format decision state object that manages whitespace decisions.
  
  Each token is processed one at a time, at which point its whitespace formatting
  decisions are made. A graph of potential whitespace formatting is created,
  where each node in the graph is a format decision state object. The heuristic
  tries formatting the token with and without a newline before it to determine
  which one has the least penalty. Therefore, the format decision state object for
  each decision needs to be its own unique copy.
  
  Once the heuristic determines the best formatting, it makes a non-dry run pass
  through the code to commit the whitespace formatting.
  
    FormatDecisionState: main class exported by this module.
  """
  
  from yapf.pytree import split_penalty
  from yapf.pytree.pytree_utils import NodeName
  from yapf.yapflib import logical_line
  from yapf.yapflib import object_state
  from yapf.yapflib import style
  from yapf.yapflib import subtypes
  
  
  class FormatDecisionState(object):
    """The current state when indenting a logical line.
  
    The FormatDecisionState object is meant to be copied instead of referenced.
  
    Attributes:
      first_indent: The indent of the first token.
      column: The number of used columns in the current line.
      line: The logical line we're currently processing.
      next_token: The next token to be formatted.
      paren_level: The level of nesting inside (), [], and {}.
      lowest_level_on_line: The lowest paren_level on the current line.
      stack: A stack (of _ParenState) keeping track of properties applying to
        parenthesis levels.
      comp_stack: A stack (of ComprehensionState) keeping track of properties
        applying to comprehensions.
      param_list_stack: A stack (of ParameterListState) keeping track of
        properties applying to function parameter lists.
      ignore_stack_for_comparison: Ignore the stack of _ParenState for state
        comparison.
      column_limit: The column limit specified by the style.
    """
  
    def __init__(self, line, first_indent):
      """Initializer.
  
      Initializes to the state after placing the first token from 'line' at
      'first_indent'.
  
      Arguments:
        line: (LogicalLine) The logical line we're currently processing.
        first_indent: (int) The indent of the first token.
      """
      self.next_token = line.first
      self.column = first_indent
      self.line = line
      self.paren_level = 0
      self.lowest_level_on_line = 0
      self.ignore_stack_for_comparison = False
      self.stack = [_ParenState(first_indent, first_indent)]
      self.comp_stack = []
      self.param_list_stack = []
      self.first_indent = first_indent
      self.column_limit = style.Get('COLUMN_LIMIT')
  
    def Clone(self):
      """Clones a FormatDecisionState object."""
      new = FormatDecisionState(self.line, self.first_indent)
      new.next_token = self.next_token
      new.column = self.column
      new.line = self.line
      new.paren_level = self.paren_level
      new.line.depth = self.line.depth
      new.lowest_level_on_line = self.lowest_level_on_line
      new.ignore_stack_for_comparison = self.ignore_stack_for_comparison
      new.first_indent = self.first_indent
      new.stack = [state.Clone() for state in self.stack]
      new.comp_stack = [state.Clone() for state in self.comp_stack]
      new.param_list_stack = [state.Clone() for state in self.param_list_stack]
      return new
  
    def __eq__(self, other):
      # Note: 'first_indent' is implicit in the stack. Also, we ignore 'previous',
      # because it shouldn't have a bearing on this comparison. (I.e., it will
      # report equal if 'next_token' does.)
      return (self.next_token == other.next_token and
              self.column == other.column and
              self.paren_level == other.paren_level and
              self.line.depth == other.line.depth and
              self.lowest_level_on_line == other.lowest_level_on_line and
              (self.ignore_stack_for_comparison or
               other.ignore_stack_for_comparison or self.stack == other.stack and
               self.comp_stack == other.comp_stack and
               self.param_list_stack == other.param_list_stack))
  
    def __ne__(self, other):
      return not self == other
  
    def __hash__(self):
      return hash((self.next_token, self.column, self.paren_level,
                   self.line.depth, self.lowest_level_on_line))
  
    def __repr__(self):
      return ('column::%d, next_token::%s, paren_level::%d, stack::[\n\t%s' %
              (self.column, repr(self.next_token), self.paren_level,
               '\n\t'.join(repr(s) for s in self.stack) + ']'))
  
    def CanSplit(self, must_split):
      """Determine if we can split before the next token.
  
      Arguments:
        must_split: (bool) A newline was required before this token.
  
      Returns:
        True if the line can be split before the next token.
      """
      current = self.next_token
      previous = current.previous_token
  
      if current.is_pseudo:
        return False
  
      if (not must_split and subtypes.DICTIONARY_KEY_PART in current.subtypes and
          subtypes.DICTIONARY_KEY not in current.subtypes and
          not style.Get('ALLOW_MULTILINE_DICTIONARY_KEYS')):
        # In some situations, a dictionary may be multiline, but pylint doesn't
        # like it. So don't allow it unless forced to.
        return False
  
      if (not must_split and subtypes.DICTIONARY_VALUE in current.subtypes and
          not style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE')):
        return False
  
      if previous and previous.value == '(' and current.value == ')':
        # Don't split an empty function call list if we aren't splitting before
        # dict values.
        token = previous.previous_token
        while token:
          prev = token.previous_token
          if not prev or prev.name not in {'NAME', 'DOT'}:
            break
          token = token.previous_token
        if token and subtypes.DICTIONARY_VALUE in token.subtypes:
          if not style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE'):
            return False
  
      if previous and previous.value == '.' and current.value == '.':
        return False
  
      return current.can_break_before
  
    def MustSplit(self):
      """Returns True if the line must split before the next token."""
      current = self.next_token
      previous = current.previous_token
  
      if current.is_pseudo:
        return False
  
      if current.must_break_before:
        return True
  
      if not previous:
        return False
  
      if style.Get('SPLIT_ALL_COMMA_SEPARATED_VALUES') and previous.value == ',':
        if (subtypes.COMP_FOR in current.subtypes or
            subtypes.LAMBDEF in current.subtypes):
          return False
  
        return True
  
      if (style.Get('FORCE_MULTILINE_DICT') and
          subtypes.DICTIONARY_KEY in current.subtypes and not current.is_comment):
        return True
  
      if (style.Get('SPLIT_ALL_TOP_LEVEL_COMMA_SEPARATED_VALUES') and
          previous.value == ','):
  
        if (subtypes.COMP_FOR in current.subtypes or
            subtypes.LAMBDEF in current.subtypes):
          return False
  
        # Avoid breaking in a container that fits in the current line if possible
        opening = _GetOpeningBracket(current)
  
        # Can't find opening bracket, behave the same way as
        # SPLIT_ALL_COMMA_SEPARATED_VALUES.
        if not opening:
          return True
  
        if current.is_comment:
          # Don't require splitting before a comment, since it may be related to
          # the current line.
          return False
  
        # Allow the fallthrough code to handle the closing bracket.
        if current != opening.matching_bracket:
          # If the container doesn't fit in the current line, must split
          return not self._ContainerFitsOnStartLine(opening)
  
      if (self.stack[-1].split_before_closing_bracket and
          (current.value in '}]' and style.Get('SPLIT_BEFORE_CLOSING_BRACKET') or
           current.value in '}])' and style.Get('INDENT_CLOSING_BRACKETS'))):
        # Split before the closing bracket if we can.
        if (subtypes.SUBSCRIPT_BRACKET not in current.subtypes or
            (previous.value == ',' and
             not style.Get('DISABLE_ENDING_COMMA_HEURISTIC'))):
          return current.node_split_penalty != split_penalty.UNBREAKABLE
  
      if (current.value == ')' and previous.value == ',' and
          not _IsSingleElementTuple(current.matching_bracket)):
        return True
  
      # Prevent splitting before the first argument in compound statements
      # with the exception of function declarations.
      if (style.Get('SPLIT_BEFORE_FIRST_ARGUMENT') and
          _IsCompoundStatement(self.line.first) and
          not _IsFunctionDef(self.line.first)):
        return False
  
      ###########################################################################
      # List Splitting
      if (style.Get('DEDENT_CLOSING_BRACKETS') or
          style.Get('INDENT_CLOSING_BRACKETS') or
          style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')):
        bracket = current if current.ClosesScope() else previous
        if subtypes.SUBSCRIPT_BRACKET not in bracket.subtypes:
          if bracket.OpensScope():
            if style.Get('COALESCE_BRACKETS'):
              if current.OpensScope():
                # Prefer to keep all opening brackets together.
                return False
  
            if (not _IsLastScopeInLine(bracket) or
                logical_line.IsSurroundedByBrackets(bracket)):
              last_token = bracket.matching_bracket
            else:
              last_token = _LastTokenInLine(bracket.matching_bracket)
  
            if not self._FitsOnLine(bracket, last_token):
              # Split before the first element if the whole list can't fit on a
              # single line.
              self.stack[-1].split_before_closing_bracket = True
              return True
  
          elif (style.Get('DEDENT_CLOSING_BRACKETS') or
                style.Get('INDENT_CLOSING_BRACKETS')) and current.ClosesScope():
            # Split before and dedent the closing bracket.
            return self.stack[-1].split_before_closing_bracket
  
      if (style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') and
          current.is_name):
        # An expression that's surrounded by parens gets split after the opening
        # parenthesis.
        def SurroundedByParens(token):
          """Check if it's an expression surrounded by parentheses."""
          while token:
            if token.value == ',':
              return False
            if token.value == ')':
              return not token.next_token
            if token.OpensScope():
              token = token.matching_bracket.next_token
            else:
              token = token.next_token
          return False
  
        if (previous.value == '(' and not previous.is_pseudo and
            not logical_line.IsSurroundedByBrackets(previous)):
          pptoken = previous.previous_token
          if (pptoken and not pptoken.is_name and not pptoken.is_keyword and
              SurroundedByParens(current)):
            return True
  
      if (current.is_name or current.is_string) and previous.value == ',':
        # If the list has function calls in it and the full list itself cannot
        # fit on the line, then we want to split. Otherwise, we'll get something
        # like this:
        #
        #     X = [
        #         Bar(xxx='some string',
        #             yyy='another long string',
        #             zzz='a third long string'), Bar(
        #                 xxx='some string',
        #                 yyy='another long string',
        #                 zzz='a third long string')
        #     ]
        #
        # or when a string formatting syntax.
        func_call_or_string_format = False
        tok = current.next_token
        if current.is_name:
          while tok and (tok.is_name or tok.value == '.'):
            tok = tok.next_token
          func_call_or_string_format = tok and tok.value == '('
        elif current.is_string:
          while tok and tok.is_string:
            tok = tok.next_token
          func_call_or_string_format = tok and tok.value == '%'
        if func_call_or_string_format:
          open_bracket = logical_line.IsSurroundedByBrackets(current)
          if open_bracket:
            if open_bracket.value in '[{':
              if not self._FitsOnLine(open_bracket,
                                      open_bracket.matching_bracket):
                return True
            elif tok.value == '(':
              if not self._FitsOnLine(current, tok.matching_bracket):
                return True
  
      if (current.OpensScope() and previous.value == ',' and
          subtypes.DICTIONARY_KEY not in current.next_token.subtypes):
        # If we have a list of tuples, then we can get a similar look as above. If
        # the full list cannot fit on the line, then we want a split.
        open_bracket = logical_line.IsSurroundedByBrackets(current)
        if (open_bracket and open_bracket.value in '[{' and
            subtypes.SUBSCRIPT_BRACKET not in open_bracket.subtypes):
          if not self._FitsOnLine(current, current.matching_bracket):
            return True
  
      ###########################################################################
      # Dict/Set Splitting
      if (style.Get('EACH_DICT_ENTRY_ON_SEPARATE_LINE') and
          subtypes.DICTIONARY_KEY in current.subtypes and not current.is_comment):
        # Place each dictionary entry onto its own line.
        if previous.value == '{' and previous.previous_token:
          opening = _GetOpeningBracket(previous.previous_token)
          if (opening and opening.value == '(' and opening.previous_token and
              opening.previous_token.is_name):
            # This is a dictionary that's an argument to a function.
            if (self._FitsOnLine(previous, previous.matching_bracket) and
                previous.matching_bracket.next_token and
                (not opening.matching_bracket.next_token or
                 opening.matching_bracket.next_token.value != '.') and
                _ScopeHasNoCommas(previous)):
              # Don't split before the key if:
              #   - The dictionary fits on a line, and
              #   - The function call isn't part of a builder-style call and
              #   - The dictionary has one entry and no trailing comma
              return False
        return True
  
      if (style.Get('SPLIT_BEFORE_DICT_SET_GENERATOR') and
          subtypes.DICT_SET_GENERATOR in current.subtypes):
        # Split before a dict/set generator.
        return True
  
      if (subtypes.DICTIONARY_VALUE in current.subtypes or
          (previous.is_pseudo and previous.value == '(' and
           not current.is_comment)):
        # Split before the dictionary value if we can't fit every dictionary
        # entry on its own line.
        if not current.OpensScope():
          opening = _GetOpeningBracket(current)
          if not self._EachDictEntryFitsOnOneLine(opening):
            return style.Get('ALLOW_SPLIT_BEFORE_DICT_VALUE')
  
      if previous.value == '{':
        # Split if the dict/set cannot fit on one line and ends in a comma.
        closing = previous.matching_bracket
        if (not self._FitsOnLine(previous, closing) and
            closing.previous_token.value == ','):
          self.stack[-1].split_before_closing_bracket = True
          return True
  
      ###########################################################################
      # Argument List Splitting
  
      if style.Get('SPLIT_ARGUMENTS_WHEN_COMMA_TERMINATED'):
        # Split before arguments in a function call or definition if the
        # arguments are terminated by a comma.
        opening = _GetOpeningBracket(current)
        if opening and opening.previous_token and opening.previous_token.is_name:
          if previous.value in '(,':
            if opening.matching_bracket.previous_token.value == ',':
              return True
  
      if (style.Get('SPLIT_BEFORE_NAMED_ASSIGNS') and not current.is_comment and
          subtypes.DEFAULT_OR_NAMED_ASSIGN_ARG_LIST in current.subtypes):
        if (previous.value not in {'=', ':', '*', '**'} and
            current.value not in ':=,)' and not _IsFunctionDefinition(previous)):
          # If we're going to split the lines because of named arguments, then we
          # want to split after the opening bracket as well. But not when this is
          # part of a function definition.
          if previous.value == '(':
            # Make sure we don't split after the opening bracket if the
            # continuation indent is greater than the opening bracket:
            #
            #  a(
            #      b=1,
            #      c=2)
            if (self._FitsOnLine(previous, previous.matching_bracket) and
                logical_line.IsSurroundedByBrackets(previous)):
              # An argument to a function is a function call with named
              # assigns.
              return False
  
            # Don't split if not required
            if (not style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') and
                not style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')):
              return False
  
            column = self.column - self.stack[-1].last_space
            return column > style.Get('CONTINUATION_INDENT_WIDTH')
  
          opening = _GetOpeningBracket(current)
          if opening:
            return not self._ContainerFitsOnStartLine(opening)
  
      if (current.value not in '{)' and previous.value == '(' and
          self._ArgumentListHasDictionaryEntry(current)):
        return True
  
      if ((current.is_name or current.value in {'*', '**'}) and
          previous.value == ','):
        # If we have a function call within an argument list and it won't fit on
        # the remaining line, but it will fit on a line by itself, then go ahead
        # and split before the call.
        opening = _GetOpeningBracket(current)
        if (opening and opening.value == '(' and opening.previous_token and
            (opening.previous_token.is_name or
             opening.previous_token.value in {'*', '**'})):
          is_func_call = False
          opening = current
          while opening:
            if opening.value == '(':
              is_func_call = True
              break
            if (not (opening.is_name or opening.value in {'*', '**'}) and
                opening.value != '.'):
              break
            opening = opening.next_token
  
          if is_func_call:
            if (not self._FitsOnLine(current, opening.matching_bracket) or
                (opening.matching_bracket.next_token and
                 opening.matching_bracket.next_token.value != ',' and
                 not opening.matching_bracket.next_token.ClosesScope())):
              return True
  
      pprevious = previous.previous_token
  
      # A function call with a dictionary as its first argument may result in
      # unreadable formatting if the dictionary spans multiple lines. The
      # dictionary itself is formatted just fine, but the remaining arguments are
      # indented too far:
      #
      #     function_call({
      #         KEY_1: 'value one',
      #         KEY_2: 'value two',
      #     },
      #                   default=False)
      if (current.value == '{' and previous.value == '(' and pprevious and
          pprevious.is_name):
        dict_end = current.matching_bracket
        next_token = dict_end.next_token
        if next_token.value == ',' and not self._FitsOnLine(current, dict_end):
          return True
  
      if (current.is_name and pprevious and pprevious.is_name and
          previous.value == '('):
  
        if (not self._FitsOnLine(previous, previous.matching_bracket) and
            _IsFunctionCallWithArguments(current)):
          # There is a function call, with more than 1 argument, where the first
          # argument is itself a function call with arguments that does not fit
          # into the line.  In this specific case, if we split after the first
          # argument's opening '(', then the formatting will look bad for the
          # rest of the arguments. E.g.:
          #
          #     outer_function_call(inner_function_call(
          #         inner_arg1, inner_arg2),
          #                         outer_arg1, outer_arg2)
          #
          # Instead, enforce a split before that argument to keep things looking
          # good.
          if (style.Get('SPLIT_BEFORE_EXPRESSION_AFTER_OPENING_PAREN') or
              style.Get('SPLIT_BEFORE_FIRST_ARGUMENT')):
            return True
  
          opening = _GetOpeningBracket(current)
          if (opening and opening.value == '(' and opening.previous_token and
              (opening.previous_token.is_name or
               opening.previous_token.value in {'*', '**'})):
            is_func_call = False
            opening = current
            while opening:
              if opening.value == '(':
                is_func_call = True
                break
              if (not (opening.is_name or opening.value in {'*', '**'}) and
                  opening.value != '.'):
                break
              opening = opening.next_token
  
            if is_func_call:
              if (not self._FitsOnLine(current, opening.matching_bracket) or
                  (opening.matching_bracket.next_token and
                   opening.matching_bracket.next_token.value != ',' and
                   not opening.matching_bracket.next_token.ClosesScope())):
                return True
  
      if (previous.OpensScope() and not current.OpensScope() and
          not current.is_comment and
          subtypes.SUBSCRIPT_BRACKET not in previous.subtypes):
        if pprevious and not pprevious.is_keyword and not pprevious.is_name:
          # We want to split if there's a comment in the container.
          token = current
          while token != previous.matching_bracket:
            if token.is_comment:
              return True
            token = token.next_token
        if previous.value == '(':
          pptoken = previous.previous_token
          if not pptoken or not pptoken.is_name:
            # Split after the opening of a tuple if it doesn't fit on the current
            # line and it's not a function call.
            if self._FitsOnLine(previous, previous.matching_bracket):
              return False
          elif not self._FitsOnLine(previous, previous.matching_bracket):
            if len(previous.container_elements) == 1:
              return False
  
            elements = previous.container_elements + [previous.matching_bracket]
            i = 1
            while i < len(elements):
              if (not elements[i - 1].OpensScope() and
                  not self._FitsOnLine(elements[i - 1], elements[i])):
                return True
              i += 1
  
            if (self.column_limit - self.column) / float(self.column_limit) < 0.3:
              # Try not to squish all of the arguments off to the right.
              return True
        else:
          # Split after the opening of a container if it doesn't fit on the
          # current line.
          if not self._FitsOnLine(previous, previous.matching_bracket):
            return True
  
      ###########################################################################
      # Original Formatting Splitting
      # These checks rely upon the original formatting. This is in order to
      # attempt to keep hand-written code in the same condition as it was before.
      # However, this may cause the formatter to fail to be idempotent.
      if (style.Get('SPLIT_BEFORE_BITWISE_OPERATOR') and current.value in '&|' and
          previous.lineno < current.lineno):
        # Retain the split before a bitwise operator.
        return True
  
      if (current.is_comment and
          previous.lineno < current.lineno - current.value.count('\n')):
        # If a comment comes in the middle of a logical line (like an if
        # conditional with comments interspersed), then we want to split if the
        # original comments were on a separate line.
        return True
  
      return False
  
    def AddTokenToState(self, newline, dry_run, must_split=False):
      """Add a token to the format decision state.
  
      Allow the heuristic to try out adding the token with and without a newline.
      Later on, the algorithm will determine which one has the lowest penalty.
  
      Arguments:
        newline: (bool) Add the token on a new line if True.
        dry_run: (bool) Don't commit whitespace changes to the FormatToken if
          True.
        must_split: (bool) A newline was required before this token.
  
      Returns:
        The penalty of splitting after the current token.
      """
      self._PushParameterListState(newline)
  
      penalty = 0
      if newline:
        penalty = self._AddTokenOnNewline(dry_run, must_split)
      else:
        self._AddTokenOnCurrentLine(dry_run)
  
      penalty += self._CalculateComprehensionState(newline)
      penalty += self._CalculateParameterListState(newline)
  
      return self.MoveStateToNextToken() + penalty
  
    def _AddTokenOnCurrentLine(self, dry_run):
      """Puts the token on the current line.
  
      Appends the next token to the state and updates information necessary for
      indentation.
  
      Arguments:
        dry_run: (bool) Commit whitespace changes to the FormatToken if True.
      """
      current = self.next_token
      previous = current.previous_token
  
      spaces = current.spaces_required_before
      if isinstance(spaces, list):
        # Don't set the value here, as we need to look at the lines near
        # this one to determine the actual horizontal alignment value.
        spaces = 0
  
      if not dry_run:
        current.AddWhitespacePrefix(newlines_before=0, spaces=spaces)
  
      if previous.OpensScope():
        if not current.is_comment:
          # Align closing scopes that are on a newline with the opening scope:
          #
          #     foo = [a,
          #            b,
          #           ]
          self.stack[-1].closing_scope_indent = self.column - 1
          if style.Get('ALIGN_CLOSING_BRACKET_WITH_VISUAL_INDENT'):
            self.stack[-1].closing_scope_indent += 1
          self.stack[-1].indent = self.column + spaces
        else:
          self.stack[-1].closing_scope_indent = (
              self.stack[-1].indent - style.Get('CONTINUATION_INDENT_WIDTH'))
  
      self.column += spaces
  
    def _AddTokenOnNewline(self, dry_run, must_split):
      """Adds a line break and necessary indentation.
  
      Appends the next token to the state and updates information necessary for
      indentation.
  
      Arguments:
        dry_run: (bool) Don't commit whitespace changes to the FormatToken if
          True.
        must_split: (bool) A newline was required before this token.
  
      Returns:
        The split penalty for splitting after the current state.
      """
      current = self.next_token
      previous = current.previous_token
  
      self.column = self._GetNewlineColumn()
  
      if not dry_run:
        indent_level = self.line.depth
        spaces = self.column
        if spaces:
          spaces -= indent_level * style.Get('INDENT_WIDTH')
        current.AddWhitespacePrefix(
            newlines_before=1, spaces=spaces, indent_level=indent_level)
  
      if not current.is_comment:
        self.stack[-1].last_space = self.column
      self.lowest_level_on_line = self.paren_level
  
      if (previous.OpensScope() or
          (previous.is_comment and previous.previous_token is not None and
           previous.previous_token.OpensScope())):
        dedent = (style.Get('CONTINUATION_INDENT_WIDTH'),
                  0)[style.Get('INDENT_CLOSING_BRACKETS')]
        self.stack[-1].closing_scope_indent = (
            max(0, self.stack[-1].indent - dedent))
        self.stack[-1].split_before_closing_bracket = True
  
      # Calculate the split penalty.
      penalty = current.split_penalty
  
      if must_split:
        # Don't penalize for a must split.
        return penalty
  
      if previous.is_pseudo and previous.value == '(':
        # Small penalty for splitting after a pseudo paren.
        penalty += 50
  
      # Add a penalty for each increasing newline we add, but don't penalize for
      # splitting before an if-expression or list comprehension.
      if current.value not in {'if', 'for'}:
        last = self.stack[-1]
        last.num_line_splits += 1
        penalty += (
            style.Get('SPLIT_PENALTY_FOR_ADDED_LINE_SPLIT') *
            last.num_line_splits)
  
      if current.OpensScope() and previous.OpensScope():
        # Prefer to keep opening brackets coalesced (unless it's at the beginning
        # of a function call).
        pprev = previous.previous_token
        if not pprev or not pprev.is_name:
          penalty += 10
  
      return penalty + 10
  
    def MoveStateToNextToken(self):
      """Calculate format decision state information and move onto the next token.
  
      Before moving onto the next token, we first calculate the format decision
      state given the current token and its formatting decisions. Then the format
      decision state is set up so that the next token can be added.
  
      Returns:
        The penalty for the number of characters over the column limit.
      """
      current = self.next_token
      if not current.OpensScope() and not current.ClosesScope():
        self.lowest_level_on_line = min(self.lowest_level_on_line,
                                        self.paren_level)
  
      # If we encounter an opening bracket, we add a level to our stack to prepare
      # for the subsequent tokens.
      if current.OpensScope():
        last = self.stack[-1]
        new_indent = style.Get('CONTINUATION_INDENT_WIDTH') + last.last_space
  
        self.stack.append(_ParenState(new_indent, self.stack[-1].last_space))
        self.paren_level += 1
  
      # If we encounter a closing bracket, we can remove a level from our
      # parenthesis stack.
      if len(self.stack) > 1 and current.ClosesScope():
        if subtypes.DICTIONARY_KEY_PART in current.subtypes:
          self.stack[-2].last_space = self.stack[-2].indent
        else:
          self.stack[-2].last_space = self.stack[-1].last_space
        self.stack.pop()
        self.paren_level -= 1
  
      is_multiline_string = current.is_string and '\n' in current.value
      if is_multiline_string:
        # This is a multiline string. Only look at the first line.
        self.column += len(current.value.split('\n')[0])
      elif not current.is_pseudo:
        self.column += len(current.value)
  
      self.next_token = self.next_token.next_token
  
      # Calculate the penalty for overflowing the column limit.
      penalty = 0
      if (not current.is_pylint_comment and not current.is_pytype_comment and
          not current.is_copybara_comment and self.column > self.column_limit):
        excess_characters = self.column - self.column_limit
        penalty += style.Get('SPLIT_PENALTY_EXCESS_CHARACTER') * excess_characters
  
      if is_multiline_string:
        # If this is a multiline string, the column is actually the
        # end of the last line in the string.
        self.column = len(current.value.split('\n')[-1])
  
      return penalty
  
    def _CalculateComprehensionState(self, newline):
      """Makes required changes to comprehension state.
  
      Args:
        newline: Whether the current token is to be added on a newline.
  
      Returns:
        The penalty for the token-newline combination given the current
        comprehension state.
      """
      current = self.next_token
      previous = current.previous_token
      top_of_stack = self.comp_stack[-1] if self.comp_stack else None
      penalty = 0
  
      if top_of_stack is not None:
        # Check if the token terminates the current comprehension.
        if current == top_of_stack.closing_bracket:
          last = self.comp_stack.pop()
          # Lightly penalize comprehensions that are split across multiple lines.
          if last.has_interior_split:
            penalty += style.Get('SPLIT_PENALTY_COMPREHENSION')
  
          return penalty
  
        if newline:
          top_of_stack.has_interior_split = True
  
      if (subtypes.COMP_EXPR in current.subtypes and
          subtypes.COMP_EXPR not in previous.subtypes):
        self.comp_stack.append(object_state.ComprehensionState(current))
        return penalty
  
      if current.value == 'for' and subtypes.COMP_FOR in current.subtypes:
        if top_of_stack.for_token is not None:
          # Treat nested comprehensions like normal comp_if expressions.
          # Example:
          #     my_comp = [
          #         a.qux + b.qux
          #         for a in foo
          #   -->   for b in bar   <--
          #         if a.zut + b.zut
          #     ]
          if (style.Get('SPLIT_COMPLEX_COMPREHENSION') and
              top_of_stack.has_split_at_for != newline and
              (top_of_stack.has_split_at_for or
               not top_of_stack.HasTrivialExpr())):
            penalty += split_penalty.UNBREAKABLE
        else:
          top_of_stack.for_token = current
          top_of_stack.has_split_at_for = newline
  
  