Track column position for each token

Author: theodorejb

composer.lock

@@ -16,6 +16,7 @@
 use DevTheorem\HandlebarsParser\Ast\InverseChain;
 use DevTheorem\HandlebarsParser\Ast\Literal;
 use DevTheorem\HandlebarsParser\Ast\MustacheStatement;
+use DevTheorem\HandlebarsParser\Ast\Node;
 use DevTheorem\HandlebarsParser\Ast\OpenBlock;
 use DevTheorem\HandlebarsParser\Ast\OpenHelper;
 use DevTheorem\HandlebarsParser\Ast\OpenPartialBlock;
@@ -167,13 +168,14 @@ protected function postprocessTokens(array $tokens): array
 
         if ($numTokens === 0) {
             // empty input - just add sentinel token
-            return [new Token(Lexer::T_EOF, "\0", 0)];
+            return [new Token(Lexer::T_EOF, "\0", 0, 0)];
         }
 
         $lastToken = $tokens[$numTokens - 1];
 
         // Add sentinel token
-        $tokens[] = new Token(Lexer::T_EOF, "\0", $lastToken->line);
+        $column = $lastToken->column + strlen($lastToken->text);
+        $tokens[] = new Token(Lexer::T_EOF, "\0", $lastToken->line, $column);
         return $tokens;
     }
 
@@ -384,6 +386,12 @@ protected function getErrorMessage(int $symbol, int $state, int $line): string
         return "Parse error on line $line" . $expectedString . ', got ' . $this->symbolToName[$symbol];
     }
 
+    private function getNodeError(string $message, Node $node): string
+    {
+        $start = $node->loc->start;
+        return $message . ' - ' . $start->line . ':' . $start->column;
+    }
+
     /**
      * Get limited number of expected tokens in given state.
      *
@@ -505,7 +513,10 @@ protected function locInfo(int $tokenStartPos, int $tokenEndPos): SourceLocation
             $source .= $token->text;
         }
 
-        return new SourceLocation($source, new Position($startToken->line, -1), new Position($endToken->line, -1));
+        $start = new Position($startToken->line, $startToken->column);
+        $end = new Position($endToken->line, $endToken->column + strlen($endToken->text) - 1);
+
+        return new SourceLocation($source, $start, $end);
     }
 
     protected function id(string $token): string
@@ -539,7 +550,7 @@ protected function prepareProgram(array $statements, ?SourceLocation $loc = null
                 $lastLoc = $statements[count($statements) - 1]->loc;
                 $loc = new SourceLocation($firstLoc->source, $firstLoc->start, $lastLoc->end);
             } else {
-                $loc = new SourceLocation('', new Position(0, -1), new Position(0, -1));
+                $loc = new SourceLocation('', new Position(0, 0), new Position(0, 0));
             }
         }
 
@@ -579,7 +590,8 @@ private function validateClose(OpenHelper $open, CloseBlock|string $close): void
         }
 
         if ($open->path->original !== $close) {
-            throw new \Exception("{$open->path->original} doesn't match {$close}");
+            $msg = $this->getNodeError("{$open->path->original} doesn't match {$close}", $open->path);
+            throw new \Exception($msg);
         }
     }
 
@@ -610,7 +622,8 @@ protected function preparePath(bool $data, SubExpression|null $sexpr, array $par
 
             if (!$isLiteral && ($part === '..' || $part === '.' || $part === 'this')) {
                 if (count($tail) > 0) {
-                    throw new \Exception('Invalid path: ' . $original);
+                    $msg = $this->getNodeError("Invalid path: $original", new Node('', $loc));
+                    throw new \Exception($msg);
                 } elseif ($part === '..') {
                     $depth++;
                 }

src/ParserAbstract.php

@@ -57,7 +57,6 @@ public function getNextToken(): ?Token
             return null;
         }
 
-        $line = substr_count($this->text, "\n", 0, $this->cursor + 1) + 1;
         $subject = substr($this->text, $this->cursor);
 
         foreach ($this->rules as $rule) {
@@ -66,6 +65,7 @@ public function getNextToken(): ?Token
             }
 
             if (preg_match("/\\A{$rule->pattern}/", $subject, $matches)) {
+                [$line, $column] = $this->getPosition();
                 $this->yytext = $matches[0];
                 $this->cursor += strlen($this->yytext);
                 $tokenName = ($rule->handler)();
@@ -75,13 +75,37 @@ public function getNextToken(): ?Token
                     return $this->getNextToken();
                 }
 
-                return new Token($tokenName, $this->yytext, $line);
+                return new Token($tokenName, $this->yytext, $line, $column);
             }
         }
 
         throw new \Exception('Unexpected token: "' . $subject[0] . '"');
     }
 
+    /**
+     * @return array{int, int}
+     */
+    private function getPosition(): array
+    {
+        $line = 1;
+        $column = -1;
+
+        for ($i = 0; $i < $this->cursor + 1; $i++) {
+            if ($this->text[$i] === "\n") {
+                $line++;
+                $column = -1;
+            } else {
+                $column++;
+            }
+        }
+
+        if ($column === -1) {
+            $column = 0;
+        }
+
+        return [$line, $column];
+    }
+
     protected function pushState(string $state): void
     {
         $this->states[] = $state;

src/Phlexer/Phlexer.php

@@ -8,5 +8,6 @@ public function __construct(
         public string $name,
         public string $text,
         public int $line,
+        public int $column,
     ) {}
 }

src/Phlexer/Token.php

@@ -59,14 +59,14 @@ public function testLineNumbers(): void
             {{lines}}
             _tpl;
         $expected = [
-            new Token(Lexer::T_CONTENT, "This\nis a ", 1),
-            new Token(Lexer::T_OPEN, '{{', 2),
-            new Token(Lexer::T_ID, 'template', 2),
-            new Token(Lexer::T_CLOSE, '}}', 2),
-            new Token(Lexer::T_CONTENT, "\nwith multiple\n", 3),
-            new Token(Lexer::T_OPEN, '{{', 4),
-            new Token(Lexer::T_ID, 'lines', 4),
-            new Token(Lexer::T_CLOSE, '}}', 4),
+            new Token(Lexer::T_CONTENT, "This\nis a ", 1, 0),
+            new Token(Lexer::T_OPEN, '{{', 2, 5),
+            new Token(Lexer::T_ID, 'template', 2, 7),
+            new Token(Lexer::T_CLOSE, '}}', 2, 15),
+            new Token(Lexer::T_CONTENT, "\nwith multiple\n", 3, 0),
+            new Token(Lexer::T_OPEN, '{{', 4, 0),
+            new Token(Lexer::T_ID, 'lines', 4, 2),
+            new Token(Lexer::T_CLOSE, '}}', 4, 7),
         ];
         $this->assertEquals($expected, (new Lexer())->tokenize($template));
     }