Implement parser changes for the JSX automatic runtime (#739)

Progress toward #585

This change extends the parser to detect two JSX cases that are needed for the
automatic runtime. The new code is run unconditionally, even in the old
transform, since flagging it would have its own overhead and complexity.

The new JSX transform needs to distinguish children being static or non-static,
where static is equivalent to having at least two comma-separated children
emitted by the old transform or any child being a spread child. The main
challenge in getting this right is that JSX whitespace-trimming will sometimes
completely remove all content from a text range, in which case it shouldn't
count as a child for the purposes of counting the number of children.
Fortunately, there is a relatively simple algorithm to detect if a text range
is empty: it's empty if and only if it is entirely whitespace and has at least
one newline. To identify these "empty text" regions, I added a new token type
`jsxEmptyText` that is treated the same except for the purposes of counting
children. In the future, it likely is reasonable to not treat such a region as a
token in the first place.

The new JSX transform also needs to detect the pattern of a key appearing after
a prop spread. We don't do keyword detection on JSX prop names, so instead I
manually detect the name "key", but only if we have already seen a prop spread.

generator/generateTokenTypes.ts

@@ -112,6 +112,7 @@ const types = {
 
   jsxName: new TokenType("jsxName"),
   jsxText: new TokenType("jsxText"),
+  jsxEmptyText: new TokenType("jsxEmptyText"),
   jsxTagStart: new TokenType("jsxTagStart", {startsExpr}),
   jsxTagEnd: new TokenType("jsxTagEnd"),
   typeParameterStart: new TokenType("typeParameterStart", {startsExpr}),

src/parser/plugins/jsx/index.ts

@@ -3,6 +3,7 @@ import {
   finishToken,
   getTokenFromCode,
   IdentifierRole,
+  JSXRole,
   match,
   next,
   skipSpace,
@@ -16,34 +17,58 @@ import {charCodes} from "../../util/charcodes";
 import {IS_IDENTIFIER_CHAR, IS_IDENTIFIER_START} from "../../util/identifier";
 import {tsTryParseJSXTypeArgument} from "../typescript";
 
-// Reads inline JSX contents token.
+/**
+ * Read token with JSX contents.
+ *
+ * In addition to detecting jsxTagStart and also regular tokens that might be
+ * part of an expression, this code detects the start and end of text ranges
+ * within JSX children. In order to properly count the number of children, we
+ * distinguish jsxText from jsxEmptyText, which is a text range that simplifies
+ * to the empty string after JSX whitespace trimming.
+ *
+ * It turns out that a JSX text range will simplify to the empty string if and
+ * only if both of these conditions hold:
+ * - The range consists entirely of whitespace characters (only counting space,
+ *   tab, \r, and \n).
+ * - The range has at least one newline.
+ * This can be proven by analyzing any implementation of whitespace trimming,
+ * e.g. formatJSXTextLiteral in Sucrase or cleanJSXElementLiteralChild in Babel.
+ */
 function jsxReadToken(): void {
-  for (;;) {
+  let sawNewline = false;
+  let sawNonWhitespace = false;
+  while (true) {
     if (state.pos >= input.length) {
       unexpected("Unterminated JSX contents");
       return;
     }
 
     const ch = input.charCodeAt(state.pos);
-
-    switch (ch) {
-      case charCodes.lessThan:
-      case charCodes.leftCurlyBrace:
-        if (state.pos === state.start) {
-          if (ch === charCodes.lessThan) {
-            state.pos++;
-            finishToken(tt.jsxTagStart);
-            return;
-          }
-          getTokenFromCode(ch);
+    if (ch === charCodes.lessThan || ch === charCodes.leftCurlyBrace) {
+      if (state.pos === state.start) {
+        if (ch === charCodes.lessThan) {
+          state.pos++;
+          finishToken(tt.jsxTagStart);
           return;
         }
-        finishToken(tt.jsxText);
+        getTokenFromCode(ch);
         return;
+      }
+      if (sawNewline && !sawNonWhitespace) {
+        finishToken(tt.jsxEmptyText);
+      } else {
+        finishToken(tt.jsxText);
+      }
+      return;
+    }
 
-      default:
-        state.pos++;
+    // This is part of JSX text.
+    if (ch === charCodes.lineFeed) {
+      sawNewline = true;
+    } else if (ch !== charCodes.space && ch !== charCodes.carriageReturn && ch !== charCodes.tab) {
+      sawNonWhitespace = true;
     }
+    state.pos++;
   }
 }
 
@@ -116,7 +141,7 @@ function jsxParseAttributeValue(): void {
   switch (state.type) {
     case tt.braceL:
       next();
-      jsxParseExpressionContainer();
+      parseExpression();
       nextJSXTagToken();
       return;
 
@@ -134,47 +159,17 @@ function jsxParseAttributeValue(): void {
   }
 }
 
-function jsxParseEmptyExpression(): void {
-  // Do nothing.
-}
-
 // Parse JSX spread child, after already processing the {
 // Does not parse the closing }
 function jsxParseSpreadChild(): void {
   expect(tt.ellipsis);
   parseExpression();
 }
 
-// Parses JSX expression enclosed into curly brackets, after already processing the {
-// Does not parse the closing }
-function jsxParseExpressionContainer(): void {
-  if (match(tt.braceR)) {
-    jsxParseEmptyExpression();
-  } else {
-    parseExpression();
-  }
-}
-
-// Parses following JSX attribute name-value pair.
-function jsxParseAttribute(): void {
-  if (eat(tt.braceL)) {
-    expect(tt.ellipsis);
-    parseMaybeAssign();
-    // }
-    nextJSXTagToken();
-    return;
-  }
-  jsxParseNamespacedName(IdentifierRole.ObjectKey);
-  if (match(tt.eq)) {
-    nextJSXTagToken();
-    jsxParseAttributeValue();
-  }
-}
-
 // Parses JSX opening tag starting after "<".
 // Returns true if the tag was self-closing.
 // Does not parse the last token.
-function jsxParseOpeningElement(): boolean {
+function jsxParseOpeningElement(initialTokenIndex: number): boolean {
   if (match(tt.jsxTagEnd)) {
     // This is an open-fragment.
     return false;
@@ -183,8 +178,30 @@ function jsxParseOpeningElement(): boolean {
   if (isTypeScriptEnabled) {
     tsTryParseJSXTypeArgument();
   }
+  let hasSeenPropSpread = false;
   while (!match(tt.slash) && !match(tt.jsxTagEnd) && !state.error) {
-    jsxParseAttribute();
+    if (eat(tt.braceL)) {
+      hasSeenPropSpread = true;
+      expect(tt.ellipsis);
+      parseMaybeAssign();
+      // }
+      nextJSXTagToken();
+      continue;
+    }
+    if (
+      hasSeenPropSpread &&
+      state.end - state.start === 3 &&
+      input.charCodeAt(state.start) === charCodes.lowercaseK &&
+      input.charCodeAt(state.start + 1) === charCodes.lowercaseE &&
+      input.charCodeAt(state.start + 2) === charCodes.lowercaseY
+    ) {
+      state.tokens[initialTokenIndex].jsxRole = JSXRole.KeyAfterPropSpread;
+    }
+    jsxParseNamespacedName(IdentifierRole.ObjectKey);
+    if (match(tt.eq)) {
+      nextJSXTagToken();
+      jsxParseAttributeValue();
+    }
   }
   const isSelfClosing = match(tt.slash);
   if (isSelfClosing) {
@@ -208,7 +225,10 @@ function jsxParseClosingElement(): void {
 // (starting after "<"), attributes, contents and closing tag.
 // Does not parse the last token.
 function jsxParseElementAt(): void {
-  const isSelfClosing = jsxParseOpeningElement();
+  const initialTokenIndex = state.tokens.length - 1;
+  state.tokens[initialTokenIndex].jsxRole = JSXRole.Normal;
+  let numExplicitChildren = 0;
+  const isSelfClosing = jsxParseOpeningElement(initialTokenIndex);
   if (!isSelfClosing) {
     nextJSXExprToken();
     while (true) {
@@ -218,13 +238,26 @@ function jsxParseElementAt(): void {
           if (match(tt.slash)) {
             nextJSXTagToken();
             jsxParseClosingElement();
+            if (
+              numExplicitChildren > 1 &&
+              // Key after prop spread takes precedence precedence over static children.
+              state.tokens[initialTokenIndex].jsxRole !== JSXRole.KeyAfterPropSpread
+            ) {
+              state.tokens[initialTokenIndex].jsxRole = JSXRole.StaticChildren;
+            }
             return;
           }
+          numExplicitChildren++;
           jsxParseElementAt();
           nextJSXExprToken();
           break;
 
         case tt.jsxText:
+          numExplicitChildren++;
+          nextJSXExprToken();
+          break;
+
+        case tt.jsxEmptyText:
           nextJSXExprToken();
           break;
 
@@ -233,8 +266,17 @@ function jsxParseElementAt(): void {
           if (match(tt.ellipsis)) {
             jsxParseSpreadChild();
             nextJSXExprToken();
+            // Spread children are a mechanism to explicitly mark children as
+            // static, so count it as 2 children to satisfy the "more than one
+            // child" condition.
+            numExplicitChildren += 2;
           } else {
-            jsxParseExpressionContainer();
+            // If we see {}, this is an empty pseudo-expression that doesn't
+            // count as a child.
+            if (!match(tt.braceR)) {
+              numExplicitChildren++;
+              parseExpression();
+            }
             nextJSXExprToken();
           }
 

src/parser/tokenizer/index.ts

@@ -27,6 +27,19 @@ export enum IdentifierRole {
   ImportAccess,
 }
 
+/**
+ * Extra information on jsxTagStart tokens, used to determine which of the three
+ * jsx functions are called in the automatic transform.
+ */
+export enum JSXRole {
+  Normal,
+  // The element has at least two explicitly-specified children or has spread
+  // children.
+  StaticChildren,
+  // The element has a prop named "key" after a prop spread.
+  KeyAfterPropSpread,
+}
+
 export function isDeclaration(token: Token): boolean {
   const role = token.identifierRole;
   return (
@@ -97,6 +110,7 @@ export class Token {
     this.scopeDepth = state.scopeDepth;
     this.isType = state.isType;
     this.identifierRole = null;
+    this.jsxRole = null;
     this.shadowsGlobal = false;
     this.isAsyncOperation = false;
     this.contextId = null;
@@ -117,6 +131,7 @@ export class Token {
   scopeDepth: number;
   isType: boolean;
   identifierRole: IdentifierRole | null;
+  jsxRole: JSXRole | null;
   // Initially false for all tokens, then may be computed in a follow-up step that does scope
   // analysis.
   shadowsGlobal: boolean;