From a8be5b3305fa355147b81c72aee8d19a95e4a68e Mon Sep 17 00:00:00 2001 From: Alisdair Meredith Date: Sat, 9 Nov 2024 11:28:51 -0500 Subject: [PATCH] [lex.pptoken] Separate out keyword placeholders The _preprocessing-token_ term is defined as one of another set of grammar productions, with the three placeholder tokens standing out as specific single tokens. Moving the placeholder tokens into their own production simplifies both the _preprocessing-token_ and _keyword_ grammar productions, plus it guarantees a consistency for the set of placeholder between both places. It also makes it easier for [cpp] to directly cross-reference where needed. Pedantically, the term placeholder token itself is problematic as tokens are a phase 7 concern, where they become keywords. Within the earlier phases where these placeholders are used the correct term is a preprocessing token, so strictly these should be placeholder preprocessing tokens, which is a mouthful. So renaming the term to placeholder keyword, which is hopefully less of a concern as there are no keywords in phase 3--6 to confuse with phase 7 keywords. Finally, clean up the informative list in text of the various kinds of preprocessing tokens, rendering as a list in the source (one kind per line) and adding a cross-reference to each such item. --- source/lex.tex | 70 ++++++++++++++++++++++++----------------- source/preprocessor.tex | 4 +-- 2 files changed, 44 insertions(+), 30 deletions(-) diff --git a/source/lex.tex b/source/lex.tex index 0e93f84ea3..6378549939 100644 --- a/source/lex.tex +++ b/source/lex.tex @@ -532,17 +532,15 @@ \indextext{token!preprocessing|(}% \begin{bnf} \nontermdef{preprocessing-token}\br + keyword-placeholder\br header-name\br - import-keyword\br - module-keyword\br - export-keyword\br - identifier\br pp-number\br + preprocessing-op-or-punc\br + identifier\br character-literal\br user-defined-character-literal\br string-literal\br user-defined-string-literal\br - preprocessing-op-or-punc\br \textnormal{each non-whitespace character that cannot be one of the above} \end{bnf} @@ -552,12 +550,15 @@ In this document, glyphs are used to identify elements of the basic character set\iref{lex.charset}. -The categories of preprocessing token are: header names, -placeholder tokens produced by preprocessing \tcode{import} and \tcode{module} directives -(\grammarterm{import-keyword}, \grammarterm{module-keyword}, and \grammarterm{export-keyword}), -identifiers, preprocessing numbers, character literals (including user-defined character -literals), string literals (including user-defined string literals), preprocessing -operators and punctuators, and single non-whitespace characters that do not lexically +The categories of preprocessing token are: +keyword plaeholders\iref{lex.placeholder}, +header names\iref{lex.header}, +preprocessing numbers\iref{lex.ppnumber}, +preprocessing operators and punctuators\iref{lex.operators}, +identifiers\iref{lex.name}, +character literals\iref{lex.ccon} including user-defined character literals, +string literals\iref{lex.string} including user-defined string literals, +and single non-whitespace characters that do not lexically match the other preprocessing token categories. If a \unicode{0027}{apostrophe} or a \unicode{0022}{quotation mark} character matches the last category, the program is ill-formed. @@ -585,20 +586,6 @@ shall have the lexical form of a keyword, an identifier, a literal, or an operator or punctuator. -\pnum -\indexgrammar{\idxgram{import-keyword}}% -\indexgrammar{\idxgram{module-keyword}}% -\indexgrammar{\idxgram{export-keyword}}% -The \grammarterm{import-keyword} is produced -by processing an \keyword{import} directive\iref{cpp.import}, -the \grammarterm{module-keyword} is produced -by preprocessing a \keyword{module} directive\iref{cpp.module}, and -the \grammarterm{export-keyword} is produced -by preprocessing either of the previous two directives. -\begin{note} -None has any observable spelling. -\end{note} - \pnum If the input stream has been parsed into preprocessing tokens up to a given character: @@ -685,6 +672,35 @@ \end{example} \indextext{token!preprocessing|)} +\rSec1[lex.placeholder]{Keyword placeholders} +\indextext{placeholder!keyword|(}% +\indexgrammar{\idxgram{import-keyword}}% +\indexgrammar{\idxgram{module-keyword}}% +\indexgrammar{\idxgram{export-keyword}}% + +\begin{bnf} +\nontermdef{keyword-placeholder}\br + export-keyword\br + import-keyword\br + module-keyword +\end{bnf} + +\pnum +Keyword placeholders are produced by preprocessing \keyword{import} +and \keyword{module} directives. +The \grammarterm{import-keyword} is produced +by preprocessing an \keyword{import} directive\iref{cpp.import}, +the \grammarterm{module-keyword} is produced +by preprocessing a \keyword{module} directive\iref{cpp.module}, and +the \grammarterm{export-keyword} is produced +by preprocessing either of the previous two directives. + +\pnum +\begin{note} +Keyword placeholders do not have any observable spelling. +\end{note} +\indextext{placeholder!keyword|(}% + \rSec1[lex.header]{Header names} \indextext{header!name|(}% @@ -1021,9 +1037,7 @@ \begin{bnf} \nontermdef{keyword}\br \textnormal{any identifier listed in \tref{lex.key}}\br - \grammarterm{import-keyword}\br - \grammarterm{module-keyword}\br - \grammarterm{export-keyword} + \grammarterm{keyword-placeholder} \end{bnf} \pnum diff --git a/source/preprocessor.tex b/source/preprocessor.tex index b517a270a4..385b810156 100644 --- a/source/preprocessor.tex +++ b/source/preprocessor.tex @@ -1262,7 +1262,7 @@ \pnum The \tcode{module} and \tcode{export} (if it exists) preprocessing tokens are replaced by the \grammarterm{module-keyword} and -\grammarterm{export-keyword} preprocessing tokens respectively. +\grammarterm{export-keyword} keyword placeholders respectively\iref{lex.placeholder}. \begin{note} This makes the line no longer a directive so it is not removed at the end of phase 4. @@ -1320,7 +1320,7 @@ In all three forms of \grammarterm{pp-import}, the \tcode{import} and \tcode{export} (if it exists) preprocessing tokens are replaced by the \grammarterm{import-keyword} and -\grammarterm{export-keyword} preprocessing tokens respectively. +\grammarterm{export-keyword} keyword placeholders respectively\iref{lex.placeholder}. \begin{note} This makes the line no longer a directive so it is not removed at the end of phase 4.