From 56ccbd4cb97e34ed82ea5f32069b2dff2264a1a0 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Fri, 2 Aug 2024 14:57:16 +0200 Subject: [PATCH] Support @next/mdx with Turbopack (#68397) Closes PACK-3153 --- Cargo.lock | 5 +++++ crates/next-core/src/next_import_map.rs | 20 +++++++++---------- packages/next-mdx/index.js | 18 +++++++++++++++++ test/e2e/app-dir/mdx/mdx.test.ts | 6 +----- turbopack/crates/turbopack-node/Cargo.toml | 2 ++ .../js/src/transforms/webpack-loaders.ts | 2 +- .../turbopack-node/src/transforms/webpack.rs | 18 +++++++++++++++-- 7 files changed, 52 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ad33f314a129f..8e148c7125081 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2008,6 +2008,9 @@ name = "either" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" +dependencies = [ + "serde", +] [[package]] name = "encode_unicode" @@ -9076,6 +9079,7 @@ dependencies = [ "async-stream", "async-trait", "const_format", + "either", "futures", "futures-retry", "indexmap 1.9.3", @@ -9087,6 +9091,7 @@ dependencies = [ "regex", "serde", "serde_json", + "serde_with", "tokio", "tracing", "turbo-tasks", diff --git a/crates/next-core/src/next_import_map.rs b/crates/next-core/src/next_import_map.rs index 11c4ff0302a41..525e1efefba1c 100644 --- a/crates/next-core/src/next_import_map.rs +++ b/crates/next-core/src/next_import_map.rs @@ -805,17 +805,15 @@ async fn insert_next_shared_aliases( ) -> Result<()> { let package_root = next_js_fs().root(); - if next_config.mdx_rs().await?.is_some() { - insert_alias_to_alternatives( - import_map, - mdx_import_source_file(), - vec![ - request_to_import_mapping(project_path, "./mdx-components"), - request_to_import_mapping(project_path, "./src/mdx-components"), - request_to_import_mapping(project_path, "@mdx-js/react"), - ], - ); - } + insert_alias_to_alternatives( + import_map, + mdx_import_source_file(), + vec![ + request_to_import_mapping(project_path, "./mdx-components"), + request_to_import_mapping(project_path, "./src/mdx-components"), + request_to_import_mapping(project_path, "@mdx-js/react"), + ], + ); insert_package_alias( import_map, diff --git a/packages/next-mdx/index.js b/packages/next-mdx/index.js index 5d650c881acde..4d894b9af41d4 100644 --- a/packages/next-mdx/index.js +++ b/packages/next-mdx/index.js @@ -23,6 +23,24 @@ module.exports = } return Object.assign({}, nextConfig, { + experimental: Object.assign({}, nextConfig?.experimental, { + turbo: Object.assign({}, nextConfig?.experimental?.turbo, { + rules: Object.assign({}, nextConfig?.experimental?.turbo?.rules, { + '*.mdx': { + loaders: [loader], + as: '*.tsx', + }, + }), + resolveAlias: Object.assign( + {}, + nextConfig?.experimental?.turbo?.resolveAlias, + { + 'next-mdx-import-source-file': + '@vercel/turbopack-next/mdx-import-source', + } + ), + }), + }), webpack(config, options) { config.resolve.alias['next-mdx-import-source-file'] = [ 'private-next-root-dir/src/mdx-components', diff --git a/test/e2e/app-dir/mdx/mdx.test.ts b/test/e2e/app-dir/mdx/mdx.test.ts index af957f28e4fb6..516969ad9f27f 100644 --- a/test/e2e/app-dir/mdx/mdx.test.ts +++ b/test/e2e/app-dir/mdx/mdx.test.ts @@ -1,10 +1,6 @@ import { nextTestSetup } from 'e2e-utils' -for (const type of [ - 'with-mdx-rs', - // only mdx-rs should work with turbopack - ...(process.env.TURBOPACK ? [] : ['without-mdx-rs']), -]) { +for (const type of ['with-mdx-rs', 'without-mdx-rs']) { describe(`mdx ${type}`, () => { const { next } = nextTestSetup({ files: __dirname, diff --git a/turbopack/crates/turbopack-node/Cargo.toml b/turbopack/crates/turbopack-node/Cargo.toml index 220319306fe60..0b7ea08e374a3 100644 --- a/turbopack/crates/turbopack-node/Cargo.toml +++ b/turbopack/crates/turbopack-node/Cargo.toml @@ -21,6 +21,7 @@ anyhow = { workspace = true } async-stream = "0.3.4" async-trait = { workspace = true } const_format = "0.2.30" +either = { workspace = true, features = ["serde"] } futures = { workspace = true } futures-retry = { workspace = true } indexmap = { workspace = true, features = ["serde"] } @@ -33,6 +34,7 @@ regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } #serde_qs = { workspace = true } +serde_with = { workspace = true, features = ["base64"] } tokio = { workspace = true, features = ["full"] } tracing = { workspace = true } turbo-tasks = { workspace = true } diff --git a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts index ea0fb75b5cfbe..d598627cb099c 100644 --- a/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts +++ b/turbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts @@ -468,7 +468,7 @@ const transform = ( if (!result.result) return reject(new Error("No result from loaders")); const [source, map] = result.result; resolve({ - source, + source: Buffer.isBuffer(source) ? {binary: source.toString('base64')} : source, map: typeof map === "string" ? map diff --git a/turbopack/crates/turbopack-node/src/transforms/webpack.rs b/turbopack/crates/turbopack-node/src/transforms/webpack.rs index a218d3f21a699..c486cdaec5800 100644 --- a/turbopack/crates/turbopack-node/src/transforms/webpack.rs +++ b/turbopack/crates/turbopack-node/src/transforms/webpack.rs @@ -2,8 +2,10 @@ use std::mem::take; use anyhow::{bail, Context, Result}; use async_trait::async_trait; +use either::Either; use serde::{Deserialize, Serialize}; use serde_json::{json, Value as JsonValue}; +use serde_with::serde_as; use turbo_tasks::{ trace::TraceRawVcs, Completion, RcStr, TaskInput, TryJoinIterExt, Value, ValueToString, Vc, }; @@ -52,11 +54,20 @@ use crate::{ AssetsForSourceMapping, }; +#[serde_as] +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +struct BytesBase64 { + #[serde_as(as = "serde_with::base64::Base64")] + binary: Vec, +} + #[derive(Debug, Serialize, Deserialize, Clone)] #[serde(rename_all = "camelCase")] #[turbo_tasks::value(serialization = "custom")] struct WebpackLoadersProcessingResult { - source: RcStr, + #[serde(with = "either::serde_untagged")] + #[turbo_tasks(debug_ignore, trace_ignore)] + source: Either, map: Option, #[turbo_tasks(trace_ignore)] assets: Option>, @@ -250,7 +261,10 @@ impl WebpackLoadersProcessedAsset { } else { None }; - let file = File::from(processed.source); + let file = match processed.source { + Either::Left(str) => File::from(str), + Either::Right(bytes) => File::from(bytes.binary), + }; let assets = emitted_assets_to_virtual_sources(processed.assets); let content = AssetContent::File(FileContent::Content(file).cell()).cell(); Ok(ProcessWebpackLoadersResult {