Skip to content

Commit

Permalink
Port babel-parser changes from 2022-10-26 to 2022-12-22 (#785)
Browse files Browse the repository at this point in the history 
Instructions: https://github.com/alangpierce/sucrase/wiki/Porting-changes-from-Babel's-parser

dfc4b61856 Parse import reflection (#14926)
✅ Implemented a basic version with tests, without any attempt at CJS support or type import support since the behavior appears unspecified at this point.

83009601f4 Parse `using` declaration (explicit resource management) (#14968)
✅ Implemented in a similar way with some simplifications since error handling isn't necessary.

04509a3be6 v7.20.0
🚫 Release only.

292e5d6b67 fix: parse `a satisfies b` as left value (#15096)
✅ Already worked, but added a test.

283720d252 fix: Parse re-declare var in class static body  (#15102)
🚫 Error handling bug that didn't affect Sucrase.

cfc453276a Fix(estree): Reset export's start after decorator (#15107)
🚫 estree not relevant to Sucrase.

1893249b67 v7.20.1
🚫 Release only.

b5a6931c23 fix: `parser` typings for plugins (#15094)
🚫 Babel-internal change.

12a58cb58c v7.20.2
🚫 Release only.

ce09a269d8 fix: support await as for-of-lhs (#15134)
🚫 Sucrase assumes modules, and `await` is a reserved word in modules.

7c9a8015fe v7.20.3
🚫 Release only.

5fde28b37e fix: Babel 8 types (#15109)
🚫 Babel-internal change.

4dedd57f64 fix: parse `import module, ...` (#15198)
✅ Fixed in implementation above.

f6546d79d4 Bump typescript to 4.9.3 (#15202)
🚫 Babel-internal change.

4c59d9fb2c Export `ParseResult` type (#15207)
🚫 Babel-internal change.

4369b33100 Parse `using[foo]` as computed member expression (#15225)
✅ Fixed with a simpler lookahead approach.

29a97b8ea2 v7.20.5
🚫 Release only.

f543b61bab refactor: remove ModuleDeclaration usage (#15236)
🚫 AST only.

362451b764 chore: Enable eslint rule `no-unnecessary-type-assertion` (#15260)
🚫 Babel-internal change.

1203f1858d chore: Clean up `parser` comments (#15252)
🚫 Babel-internal change.

3a71c79a56 fix: Support auto accessors with TypeScript annotations (#15209)
✅ This was already working in Sucrase, but added a test.

d414940e4c v7.20.7
🚫 Release only.
  • Loading branch information
@alangpierce
alangpierce authored Mar 20, 2023
1 parent bf0af52 commit c65dd7b
Show file tree
Hide file tree
Showing 8 changed files with 257 additions and 26 deletions.
1 change: 1 addition & 0 deletions generator/generateReadWordTree.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ const CONTEXTUAL_KEYWORDS = [
"symbol",
"type",
"unique",
"using",
];

const CODE = `\
Expand Down
2 changes: 1 addition & 1 deletion src/parser/plugins/typescript.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -965,7 +965,7 @@ function tsTryParseDeclare(): boolean {
case tt._var:
case tt._let: {
const oldIsType = pushTypeContext(1);
parseVarStatement(state.type);
parseVarStatement(state.type !== tt._var);
popTypeContext(oldIsType);
return true;
}
Expand Down
1 change: 1 addition & 0 deletions src/parser/tokenizer/keywords.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,5 @@ export enum ContextualKeyword {
_symbol,
_type,
_unique,
_using,
}
40 changes: 24 additions & 16 deletions src/parser/tokenizer/readWordTree.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ import {TokenType as tt} from "./types";
// prettier-ignore
export const READ_WORD_TREE = new Int32Array([
// ""
-1, 27, 783, 918, 1755, 2376, 2862, 3483, -1, 3699, -1, 4617, 4752, 4833, 5130, 5508, 5940, -1, 6480, 6939, 7749, 8181, 8343, 8505, -1, 8721, -1,
-1, 27, 783, 918, 1755, 2376, 2862, 3483, -1, 3699, -1, 4617, 4752, 4833, 5130, 5508, 5940, -1, 6480, 6939, 7749, 8181, 8451, 8613, -1, 8829, -1,
// "a"
-1, -1, 54, 243, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 432, -1, -1, -1, 675, -1, -1, -1,
// "ab"
Expand Down Expand Up @@ -611,7 +611,7 @@ export const READ_WORD_TREE = new Int32Array([
// "typeof"
(tt._typeof << 1) + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "u"
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8208, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8208, -1, -1, -1, -1, 8343, -1, -1, -1, -1, -1, -1, -1,
// "un"
-1, -1, -1, -1, -1, -1, -1, -1, -1, 8235, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "uni"
Expand All @@ -622,42 +622,50 @@ export const READ_WORD_TREE = new Int32Array([
-1, -1, -1, -1, -1, 8316, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "unique"
ContextualKeyword._unique << 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "us"
-1, -1, -1, -1, -1, -1, -1, -1, -1, 8370, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "usi"
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8397, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "usin"
-1, -1, -1, -1, -1, -1, -1, 8424, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "using"
ContextualKeyword._using << 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "v"
-1, 8370, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8424, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, 8478, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8532, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "va"
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8397, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8505, -1, -1, -1, -1, -1, -1, -1, -1,
// "var"
(tt._var << 1) + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "vo"
-1, -1, -1, -1, -1, -1, -1, -1, -1, 8451, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, 8559, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "voi"
-1, -1, -1, -1, 8478, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, 8586, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "void"
(tt._void << 1) + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "w"
-1, -1, -1, -1, -1, -1, -1, -1, 8532, 8640, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, 8640, 8748, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "wh"
-1, -1, -1, -1, -1, -1, -1, -1, -1, 8559, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, 8667, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "whi"
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8586, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8694, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "whil"
-1, -1, -1, -1, -1, 8613, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, 8721, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "while"
(tt._while << 1) + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "wi"
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8667, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8775, -1, -1, -1, -1, -1, -1,
// "wit"
-1, -1, -1, -1, -1, -1, -1, -1, 8694, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, 8802, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "with"
(tt._with << 1) + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "y"
-1, -1, -1, -1, -1, -1, -1, -1, -1, 8748, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, 8856, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "yi"
-1, -1, -1, -1, -1, 8775, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, 8883, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "yie"
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8802, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 8910, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "yiel"
-1, -1, -1, -1, 8829, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1, -1, 8937, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
// "yield"
(tt._yield << 1) + 1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
]);
87 changes: 79 additions & 8 deletions src/parser/traverser/statement.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,9 +79,11 @@ import {
eatContextual,
expect,
expectContextual,
hasFollowingLineBreak,
hasPrecedingLineBreak,
isContextual,
isLineTerminator,
isLookaheadContextual,
semicolon,
unexpected,
} from "./util";
Expand Down Expand Up @@ -173,7 +175,7 @@ function parseStatementContent(declaration: boolean): void {
if (!declaration) unexpected(); // NOTE: falls through to _var

case tt._var:
parseVarStatement(starttype);
parseVarStatement(starttype !== tt._var);
return;

case tt._while:
Expand Down Expand Up @@ -212,6 +214,15 @@ function parseStatementContent(declaration: boolean): void {
} else {
state.restoreFromSnapshot(snapshot);
}
} else if (
state.contextualKeyword === ContextualKeyword._using &&
!hasFollowingLineBreak() &&
// Statements like `using[0]` and `using in foo` aren't actual using
// declarations.
lookaheadType() === tt.name
) {
parseVarStatement(true);
return;
}
default:
// Do nothing.
Expand Down Expand Up @@ -308,6 +319,23 @@ function parseForStatement(): void {
state.scopeDepth--;
}

/**
* Determine if this token is a `using` declaration (explicit resource
* management) as part of a loop.
* https://github.com/tc39/proposal-explicit-resource-management
*/
function isUsingInLoop(): boolean {
if (!isContextual(ContextualKeyword._using)) {
return false;
}
// This must be `for (using of`, where `using` is the name of the loop
// variable.
if (isLookaheadContextual(ContextualKeyword._of)) {
return false;
}
return true;
}

// Disambiguating between a `for` and a `for`/`in` or `for`/`of`
// loop is non-trivial. Basically, we have to parse the init `var`
// statement or expression, disallowing the `in` operator (see
Expand All @@ -333,10 +361,9 @@ function parseAmbiguousForStatement(): void {
return;
}

if (match(tt._var) || match(tt._let) || match(tt._const)) {
const varKind = state.type;
if (match(tt._var) || match(tt._let) || match(tt._const) || isUsingInLoop()) {
next();
parseVar(true, varKind);
parseVar(true, state.type !== tt._var);
if (match(tt._in) || isContextual(ContextualKeyword._of)) {
parseForIn(forAwait);
return;
Expand Down Expand Up @@ -453,9 +480,9 @@ function parseTryStatement(): void {
}
}

export function parseVarStatement(kind: TokenType): void {
export function parseVarStatement(isBlockScope: boolean): void {
next();
parseVar(false, kind);
parseVar(false, isBlockScope);
semicolon();
}

Expand Down Expand Up @@ -543,9 +570,8 @@ function parseForIn(forAwait: boolean): void {

// Parse a list of variable declarations.

function parseVar(isFor: boolean, kind: TokenType): void {
function parseVar(isFor: boolean, isBlockScope: boolean): void {
while (true) {
const isBlockScope = kind === tt._const || kind === tt._let;
parseVarHead(isBlockScope);
if (eat(tt.eq)) {
const eqIndex = state.tokens.length - 1;
Expand Down Expand Up @@ -1083,6 +1109,50 @@ function parseExportSpecifier(): void {
}
}

/**
* Starting at the `module` token in an import, determine if it was truly an
* import reflection token or just looks like one.
*
* Returns true for:
* import module foo from "foo";
* import module from from "foo";
*
* Returns false for:
* import module from "foo";
* import module, {bar} from "foo";
*/
function isImportReflection(): boolean {
const snapshot = state.snapshot();
expectContextual(ContextualKeyword._module);
if (eatContextual(ContextualKeyword._from)) {
if (isContextual(ContextualKeyword._from)) {
state.restoreFromSnapshot(snapshot);
return true;
} else {
state.restoreFromSnapshot(snapshot);
return false;
}
} else if (match(tt.comma)) {
state.restoreFromSnapshot(snapshot);
return false;
} else {
state.restoreFromSnapshot(snapshot);
return true;
}
}

/**
* Eat the "module" token from the import reflection proposal.
* https://github.com/tc39/proposal-import-reflection
*/
function parseMaybeImportReflection(): void {
// isImportReflection does snapshot/restore, so only run it if we see the word
// "module".
if (isContextual(ContextualKeyword._module) && isImportReflection()) {
next();
}
}

// Parses import declaration.

export function parseImport(): void {
Expand Down Expand Up @@ -1117,6 +1187,7 @@ export function parseImport(): void {
if (match(tt.string)) {
parseExprAtom();
} else {
parseMaybeImportReflection();
parseImportSpecifiers();
expectContextual(ContextualKeyword._from);
parseExprAtom();
Expand Down
8 changes: 8 additions & 0 deletions src/transformers/ESMImportTransformer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,14 @@ export default class ESMImportTransformer extends Transformer {
return false;
}

// Skip the "module" token in import reflection.
if (
this.tokens.matchesContextual(ContextualKeyword._module) &&
this.tokens.matchesContextualAtIndex(this.tokens.currentIndex() + 2, ContextualKeyword._from)
) {
this.tokens.copyToken();
}

let foundNonTypeImport = false;
let needsComma = false;

Expand Down
92 changes: 92 additions & 0 deletions test/sucrase-test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1674,4 +1674,96 @@ describe("sucrase", () => {
{transforms: ["jsx", "typescript"]},
);
});

it("passes through import reflection", () => {
assertResult(
`
import module foo from './foo';
import module from from './foo';
import module from './foo';
import module, {a} from './foo';
`,
`
import module foo from './foo';
import module from from './foo';
import module from './foo';
import module, {a} from './foo';
`,
{transforms: []},
);
});

it("does not elide imported names when used in module reflection", () => {
assertResult(
`
import module foo from './foo';
import module bar from './bar';
console.log(foo);
`,
`
import module foo from './foo';
console.log(foo);
`,
{transforms: ["typescript"]},
);
});

it("passes through explicit resource management syntax", () => {
assertResult(
`
using lock = acquireLock();
for (using x of things()) {
console.log(x);
}
`,
`
using lock = acquireLock();
for (using x of things()) {
console.log(x);
}
`,
{transforms: []},
);
});

it("is not confused by the token `using` in other contexts", () => {
assertResult(
`
var using = 3;
using
let x = 1;
using[0];
using in x;
for (using of blah()) {
console.log(using);
}
`,
`
var using = 3;
using
let x = 1;
using[0];
using in x;
for (using of blah()) {
console.log(using);
}
`,
{transforms: []},
);
});

it("correctly parses for-of loops with external loop variable", () => {
assertResult(
`
let a;
for (a of b) {}
`,
`
let a;
for (a of b) {}
`,
{transforms: []},
);
});
});
Loading

0 comments on commit c65dd7b

Please # to comment.