Merge pull request #41 from neilccbrown/bug-fixes

Fix a couple of bugs in type inference

Author: Tobias-Kohn

tpParser/shared/src/main/scala/tigerpython/parser/ast/AstNode.scala

@@ -459,7 +459,9 @@ object AstNode {
     override def isValidAssignTarget: Boolean = true
     override def toString: String = "(%s)".format(names.mkString(", "))
   }
-  case class BooleanValue(pos: Int, value: Boolean) extends Expression(AstNodeKind.CONSTANT) {
+  case class BooleanValue(pos: Int, value: Boolean) extends Expression(AstNodeKind.CONSTANT) with Span {
+    // `True`/`False` are fixed-width keywords, so the end position needs no extra state to track.
+    def endPos: Int = pos + (if (value) 4 else 5)
     def notToString: String = if (value) "False" else "True"
     override def toString: String = if (value) "True" else "False"
   }
@@ -471,26 +473,34 @@ object AstNode {
     def apply(token: Token): Value = {
       val result = new Value(token.pos, ValueType.fromTokenType(token.tokenType))
       result.value = token.value
+      result.endPos = token.endPos
       result
     }
     def apply(pos: Int, intValue: Int): Value = {
       val result = new Value(pos, ValueType.INTEGER)
       result.value = intValue.toString
+      result.endPos = pos + result.value.length
       result
     }
   }
-  case class Value(pos: Int, valueType: ValueType.Value) extends Expression(AstNodeKind.CONSTANT) {
+  case class Value(pos: Int, valueType: ValueType.Value) extends Expression(AstNodeKind.CONSTANT) with Span {
     var value: String = _
+    // Defaults to a zero-width span at `pos` for values not constructed from a real source token
+    // (e.g. synthetic placeholders inserted during error recovery); real construction sites set
+    // this from the originating token's `endPos`.
+    var endPos: Int = pos
     def createNegative(): Value =
       if (value != null && value != "" &&
         (valueType == ValueType.INTEGER || valueType == ValueType.FLOAT)) {
         if (value(0) == '-') {
           val result = Value(pos + 1, valueType)
           result.value = value.drop(1)
+          result.endPos = result.pos + result.value.length
           result
         } else {
           val result = Value(pos - 1, valueType)
           result.value = "-" + value
+          result.endPos = result.pos + result.value.length
           result
         }
       } else

tpParser/shared/src/main/scala/tigerpython/parser/parsing/ExpressionParser.scala

@@ -648,16 +648,24 @@ class ExpressionParser(val parser: Parser, val parserState: ParserState) {
           checkMissingOperator(tokens)
           val result = AstNode.Value(token.pos, ValueType.FLOAT)
           result.value = token.value
+          result.endPos = token.endPos
           result
         case TokenType.INT | TokenType.LONG =>
           checkMissingOperator(tokens)
           val result = AstNode.Value(token.pos, ValueType.INTEGER)
           result.value = token.value
+          result.endPos = token.endPos
           result
         case TokenType.COMPLEX =>
-          AstNode.Value(token.pos, ValueType.COMPLEX)
+          val result = AstNode.Value(token.pos, ValueType.COMPLEX)
+          result.value = token.value
+          result.endPos = token.endPos
+          result
         case TokenType.NONE =>
-          AstNode.Value(token.pos, ValueType.NONE)
+          val result = AstNode.Value(token.pos, ValueType.NONE)
+          result.value = token.value
+          result.endPos = token.endPos
+          result
         case TokenType.TRUE =>
           AstNode.BooleanValue(token.pos, value = true)
         case TokenType.FALSE =>
@@ -679,7 +687,9 @@ class ExpressionParser(val parser: Parser, val parserState: ParserState) {
         case TokenType.BYTEARRAY =>
           while (tokens.hasType(TokenType.BYTEARRAY))
             tokens.next()
-          AstNode.Value(token.pos, ValueType.BYTE_ARRAY)
+          val result = AstNode.Value(token.pos, ValueType.BYTE_ARRAY)
+          result.endPos = tokens.prevEndPos
+          result
         case TokenType.LEFT_PARENS =>
           // Check for Lisp-Syntax
           if (tokens.getIndex <= 1 && tokens.peekType(1) == TokenType.NAME)

tpParser/shared/src/main/scala/tigerpython/utilities/completer/Completer.scala

@@ -116,7 +116,12 @@ class Completer(val moduleName: String,
         }
         nameWalker.getNodeForPosition(caretPos) match {
           case Some(prefixName) =>
-            val tokenRange = tokenLine.getTokenRange(prefixName.endPos, caretPos)
+            // `getTokenRange` includes tokens whose `pos` equals `caretPos` (its upper bound is
+            // inclusive), but a token starting exactly at the caret lies after it, not within the
+            // range up to it -- e.g. the `)` closing an enclosing call immediately after the caret
+            // in `wrap((expr).<caret>)`. Left in, that stray token breaks the length/last-token
+            // checks below, so it's filtered out here.
+            val tokenRange = tokenLine.getTokenRange(prefixName.endPos, caretPos).filter(_.pos < caretPos)
             if (0 < tokenRange.length && tokenRange.length <= 2 && tokenRange(0).tokenType == TokenType.DOT) {
               val n = if (filterType == FilterType.IMPORT_FROM)
                   scope.findName(moduleBase, prefixName)

tpParser/shared/src/main/scala/tigerpython/utilities/scopes/Scope.scala

@@ -148,6 +148,12 @@ abstract class Scope {
         Some(types.BuiltinTypes.LIST)
       case _: AstNode.StringValue =>
         Some(types.BuiltinTypes.STRING)
+      case expr: AstNode.Expression =>
+        // Catch-all for literal/expression kinds with no bespoke case above (int/float/complex/none
+        // literals, booleans, unary/binary ops, comparisons, ...): TypeAstWalker.getType already
+        // knows how to resolve these, so delegate rather than falling through to None, which would
+        // make e.g. `(5).bit_length` unresolvable and fall back to a full builtin-name dump.
+        Some(typeAstWalker.getType(expr))
       case _ =>
         None
     }

tpParser/shared/src/test/programs/completer/completer_int_literal_unwrapped.py

@@ -0,0 +1,3 @@
+# 4
+# bit_length
+(5).

tpParser/shared/src/test/programs/completer/completer_paren_wrapped_call_int_literal.py

@@ -0,0 +1,3 @@
+# 9
+# bit_length
+wrap((5).)

tpParser/shared/src/test/programs/completer/completer_paren_wrapped_call_method_result.py

@@ -0,0 +1,3 @@
+# 19
+# capitalize;center;count;decode;encode;endswith;expandtabs;find;format;index;isalnum;isalpha;isdigit;islower;isspace;istitle;isupper;join;ljust;lower;lstrip;partition;replace;rfind;rindex;rjust;rpartition;rsplit;rstrip;split;splitlines;startswith;strip;swapcase;title;translate;upper;zfill
+wrap(("a".upper()).)

tpParser/shared/src/test/programs/completer/completer_paren_wrapped_call_string_literal.py

@@ -0,0 +1,3 @@
+# 11
+# capitalize;center;count;decode;encode;endswith;expandtabs;find;format;index;isalnum;isalpha;isdigit;islower;isspace;istitle;isupper;join;ljust;lower;lstrip;partition;replace;rfind;rindex;rjust;rpartition;rsplit;rstrip;split;splitlines;startswith;strip;swapcase;title;translate;upper;zfill
+wrap(("a").)

tpParser/shared/src/test/programs/completer/completer_paren_wrapped_call_var.py

@@ -0,0 +1,4 @@
+# 47
+# capitalize;center;count;decode;encode;endswith;expandtabs;find;format;index;isalnum;isalpha;isdigit;islower;isspace;istitle;isupper;join;ljust;lower;lstrip;partition;replace;rfind;rindex;rjust;rpartition;rsplit;rstrip;split;splitlines;startswith;strip;swapcase;title;translate;upper;zfill
+myString = "Hello from Strype"
+wrap((myString).)