ConstExprParser: support numeric literal separator

Author: jiripudil

doc/grammars/type.abnf

@@ -85,18 +85,18 @@ ConstantExpr
 	/ ConstantFetch *ByteHorizontalWs
 
 ConstantFloat
-	= ["-"] 1*ByteDecDigit "." *ByteDecDigit [ConstantFloatExp]
-	/ ["-"] 1*ByteDecDigit ConstantFloatExp
-	/ ["-"] "." 1*ByteDecDigit [ConstantFloatExp]
+	= ["-"] 1*ByteDecDigit *("_" 1*ByteDecDigit) "." [1*ByteDecDigit *("_" 1*ByteDecDigit)] [ConstantFloatExp]
+	/ ["-"] 1*ByteDecDigit *("_" 1*ByteDecDigit) ConstantFloatExp
+	/ ["-"] "." 1*ByteDecDigit *("_" 1*ByteDecDigit) [ConstantFloatExp]
 
 ConstantFloatExp
-	= "e" ["-"] 1*ByteDecDigit
+	= "e" ["-"] 1*ByteDecDigit *("_" 1*ByteDecDigit)
 
 ConstantInt
-	= ["-"] "0b" 1*ByteBinDigit
-	/ ["-"] "0o" 1*ByteOctDigit
-	/ ["-"] "0x" 1*ByteHexDigit
-	/ ["-"] 1*ByteDecDigit
+	= ["-"] "0b" 1*ByteBinDigit *("_" 1*ByteBinDigit)
+	/ ["-"] "0o" 1*ByteOctDigit *("_" 1*ByteOctDigit)
+	/ ["-"] "0x" 1*ByteHexDigit *("_" 1*ByteHexDigit)
+	/ ["-"] 1*ByteDecDigit *("_" 1*ByteDecDigit)
 
 ConstantTrue
 	= "true"

src/Lexer/Lexer.php

@@ -160,8 +160,8 @@ private function generateRegexp(): string
 			self::TOKEN_PHPDOC_TAG => '@(?:[a-z][a-z0-9-\\\\]+:)?[a-z][a-z0-9-\\\\]*+',
 			self::TOKEN_PHPDOC_EOL => '\\r?+\\n[\\x09\\x20]*+(?:\\*(?!/)\\x20?+)?',
 
-			self::TOKEN_FLOAT => '(?:-?[0-9]++\\.[0-9]*+(?:e-?[0-9]++)?)|(?:-?[0-9]*+\\.[0-9]++(?:e-?[0-9]++)?)|(?:-?[0-9]++e-?[0-9]++)',
-			self::TOKEN_INTEGER => '-?(?:(?:0b[0-1]++)|(?:0o[0-7]++)|(?:0x[0-9a-f]++)|(?:[0-9]++))',
+			self::TOKEN_FLOAT => '(?:-?[0-9]++(_[0-9]++)*\\.[0-9]*(_[0-9]++)*+(?:e-?[0-9]++(_[0-9]++)*)?)|(?:-?[0-9]*+(_[0-9]++)*\\.[0-9]++(_[0-9]++)*(?:e-?[0-9]++(_[0-9]++)*)?)|(?:-?[0-9]++(_[0-9]++)*e-?[0-9]++(_[0-9]++)*)',
+			self::TOKEN_INTEGER => '-?(?:(?:0b[0-1]++(_[0-1]++)*)|(?:0o[0-7]++(_[0-7]++)*)|(?:0x[0-9a-f]++(_[0-9a-f]++)*)|(?:[0-9]++(_[0-9]++)*))',
 			self::TOKEN_SINGLE_QUOTED_STRING => '\'(?:\\\\[^\\r\\n]|[^\'\\r\\n\\\\])*+\'',
 			self::TOKEN_DOUBLE_QUOTED_STRING => '"(?:\\\\[^\\r\\n]|[^"\\r\\n\\\\])*+"',
 

src/Parser/ConstExprParser.php

@@ -4,6 +4,7 @@
 
 use PHPStan\PhpDocParser\Ast;
 use PHPStan\PhpDocParser\Lexer\Lexer;
+use function str_replace;
 use function strtolower;
 use function substr;
 
@@ -47,7 +48,7 @@ public function parse(TokenIterator $tokens, bool $trimStrings = false): Ast\Con
 
 			return $this->enrichWithAttributes(
 				$tokens,
-				new Ast\ConstExpr\ConstExprFloatNode($value),
+				new Ast\ConstExpr\ConstExprFloatNode(str_replace('_', '', $value)),
 				$startLine,
 				$startIndex
 			);
@@ -59,7 +60,7 @@ public function parse(TokenIterator $tokens, bool $trimStrings = false): Ast\Con
 
 			return $this->enrichWithAttributes(
 				$tokens,
-				new Ast\ConstExpr\ConstExprIntegerNode($value),
+				new Ast\ConstExpr\ConstExprIntegerNode(str_replace('_', '', $value)),
 				$startLine,
 				$startIndex
 			);

src/Parser/TypeParser.php

@@ -6,6 +6,7 @@
 use PHPStan\PhpDocParser\Ast;
 use PHPStan\PhpDocParser\Lexer\Lexer;
 use function in_array;
+use function str_replace;
 use function strpos;
 use function trim;
 
@@ -710,7 +711,7 @@ private function parseArrayShapeKey(TokenIterator $tokens)
 		$startLine = $tokens->currentTokenLine();
 
 		if ($tokens->isCurrentTokenType(Lexer::TOKEN_INTEGER)) {
-			$key = new Ast\ConstExpr\ConstExprIntegerNode($tokens->currentTokenValue());
+			$key = new Ast\ConstExpr\ConstExprIntegerNode(str_replace('_', '', $tokens->currentTokenValue()));
 			$tokens->next();
 
 		} elseif ($tokens->isCurrentTokenType(Lexer::TOKEN_SINGLE_QUOTED_STRING)) {

tests/PHPStan/Parser/ConstExprParserTest.php

@@ -176,6 +176,21 @@ public function provideIntegerNodeParseData(): Iterator
 			'-0X7Fb4',
 			new ConstExprIntegerNode('-0X7Fb4'),
 		];
+
+		yield [
+			'123_456',
+			new ConstExprIntegerNode('123456'),
+		];
+
+		yield [
+			'0b01_01_01',
+			new ConstExprIntegerNode('0b010101'),
+		];
+
+		yield [
+			'-0X7_Fb_4',
+			new ConstExprIntegerNode('-0X7Fb4'),
+		];
 	}
 
 
@@ -240,6 +255,11 @@ public function provideFloatNodeParseData(): Iterator
 			'-12.3e-4',
 			new ConstExprFloatNode('-12.3e-4'),
 		];
+
+		yield [
+			'-1_2.3_4e5_6',
+			new ConstExprFloatNode('-12.34e56'),
+		];
 	}
 
 

tests/PHPStan/Parser/TypeParserTest.php

@@ -1035,10 +1035,27 @@ public function provideParseData(): array
 				'123',
 				new ConstTypeNode(new ConstExprIntegerNode('123')),
 			],
+			[
+				'123_456',
+				new ConstTypeNode(new ConstExprIntegerNode('123456')),
+			],
+			[
+				'_123',
+				new IdentifierTypeNode('_123'),
+			],
+			[
+				'123_',
+				new ConstTypeNode(new ConstExprIntegerNode('123')),
+				Lexer::TOKEN_IDENTIFIER,
+			],
 			[
 				'123.2',
 				new ConstTypeNode(new ConstExprFloatNode('123.2')),
 			],
+			[
+				'123_456.789_012',
+				new ConstTypeNode(new ConstExprFloatNode('123456.789012')),
+			],
 			[
 				'"bar"',
 				new ConstTypeNode(new QuoteAwareConstExprStringNode('bar', QuoteAwareConstExprStringNode::DOUBLE_QUOTED)),