From 7aca34b8963fd9ac8681485a44417e537e617245 Mon Sep 17 00:00:00 2001 From: QuietMisdreavus Date: Fri, 16 Feb 2018 16:47:00 -0600 Subject: [PATCH 01/11] add readme for librustdoc --- src/librustdoc/README.md | 170 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100644 src/librustdoc/README.md diff --git a/src/librustdoc/README.md b/src/librustdoc/README.md new file mode 100644 index 0000000000000..844c54c7aa88c --- /dev/null +++ b/src/librustdoc/README.md @@ -0,0 +1,170 @@ +# The walking tour of rustdoc + +Rustdoc is implemented entirely within the crate `librustdoc`. After partially compiling a crate to +get its AST (technically the HIR map) from rustc, librustdoc performs two major steps past that to +render a set of documentation: + +* "Clean" the AST into a form that's more suited to creating documentation (and slightly more + resistant to churn in the compiler). +* Use this cleaned AST to render a crate's documentation, one page at a time. + +Naturally, there's more than just this, and those descriptions simplify out lots of details, but +that's the high-level overview. + +(Side note: this is a library crate! The `rustdoc` binary is crated using the project in +`src/tools/rustdoc`. Note that literally all that does is call the `main()` that's in this crate's +`lib.rs`, though.) + +## Cheat sheet + +* Use `x.py build --stage 1 src/libstd src/tools/rustdoc` to make a useable rustdoc you can run on + other projects. + * Add `src/libtest` to be able to use `rustdoc --test`. + * If you've used `rustup toolchain link local /path/to/build/$TARGET/stage1` previously, then + after the previous build command, `cargo +local doc` will Just Work. +* Use `x.py doc --stage 1 src/libstd` to use this rustdoc to generate the standard library docs. + * The completed docs will be available in `build/$TARGET/doc/std`, though the bundle is meant to + be used as though you would copy out the `doc` folder to a web server, since that's where the + CSS/JS and landing page are. +* Most of the HTML printing code is in `html/format.rs` and `html/render.rs`. It's in a bunch of + `fmt::Display` implementations and supplementary functions. +* The types that operates on are defined in `clean/mod.rs`, right next to the custom `Clean` trait + used to process them out of the rustc HIR. +* The bits specific to using rustdoc as a test harness are in `test.rs`. +* The Markdown renderer is loaded up in `html/markdown.rs`, including functions for extracting + doctests from a given block of Markdown. +* The tests on rustdoc *output* are located in `src/test/rustdoc`, where they're handled by the test + runner of rustbuild and the supplementary script `src/etc/htmldocck.py`. +* Tests on search index generation are located in `src/test/rustdoc-js`, as a series of JavaScript + files that encode queries on the standard library search index and expected results. + +## From crate to clean + +In `core.rs` are two central items: the `DocContext` struct, and the `run_core` function. The latter +is where rustdoc calls out to rustc to compile a crate to the point where rustdoc can take over. The +former is a state container used when crawling through a crate to gather its documentation. + +The main process of crate crawling is done in `clean/mod.rs` through several implementations of the +`Clean` trait defined within. This is a conversion trait, which defines one method: + +```rust +pub trait Clean { + fn clean(&self, cx: &DocContext) -> T; +} +``` + +`clean/mod.rs` also defines the types for the "cleaned" AST used later on to render documentation +pages. Each usually accompanies an implementation of `Clean` that takes some AST or HIR type from +rustc and converts it into the appropriate "cleaned" type. "Big" items like modules or associated +items may have some extra processing in its `Clean` implementation, but for the most part these +impls are straightforward conversions. The "entry point" to this module is the `impl Clean +for visit_ast::RustdocVisitor`, which is called by `run_core` above. + +You see, I actually lied a little earlier: There's another AST transformation that happens before +the events in `clean/mod.rs`. In `visit_ast.rs` is the type `RustdocVisitor`, which *actually* +crawls a `hir::Crate` to get the first intermediate representation, defined in `doctree.rs`. This +pass is mainly to get a few intermediate wrappers around the HIR types and to process visibility +and inlining. This is where `#[doc(inline)]`, `#[doc(no_inline)]`, and `#[doc(hidden)]` are +processed, as well as the logic for whether a `pub use` should get the full page or a "Reexport" +line in the module page. + +Back in `clean/mod.rs`, the other major thing that happens here is the special collection of doc +comments (and basic `#[doc=""]` attributes) into a separate field in the `Attributes` struct from +the other attributes. The primary output of this process is a `clean::Crate` with a tree of `Item`s +which describe the publicly-documentable items in the target crate. + +### Hot potato + +Before moving on to the next major step, a few important "passes" occur over the documentation. +These do things like combine the separate "attributes" into a single string and strip leading +whitespace to make the document easier on the markdown parser, or drop items that are not public or +deliberately hidden with `#[doc(hidden)]`. These are all implemented in the `passes/` directory, one +file per pass. By default, all of these passes are run on a crate, but the ones regarding dropping +private/hidden items can be bypassed by passing `--document-private-items` to rustdoc. + +(Strictly speaking, you can fine-tune the passes run and even add your own, but [we're trying to +deprecate that][44136]. If you need finer-grain control over these passes, please let us know!) + +[44136]: https://github.com/rust-lang/rust/issues/44136 + +## From clean to crate + +This is where the "second phase" in rustdoc begins. This phase primarily lives in the `html/` +folder, and it all starts with `run()` in `html/render.rs`. This code is responsible for setting up +the `Context`, `SharedContext`, and `Cache` which are used during rendering, copying out the static +files which live in every rendered set of documentation (things like the fonts, CSS, and JavaScript +that live in `html/static/`), creating the search index, and printing out the source code rendering, +before beginning the process of rendering all the documentation for the crate. + +Several functions implemented directly on `Context` take the `clean::Crate` and set up some state +between rendering items or recursing on a module's child items. From here the "page rendering" +begins, via an enormous `write!()` call in `html/layout.rs`. The parts that actually generate HTML +from the items and documentation occurs within a series of `std::fmt::Display` implementations and +functions that pass around a `&mut std::fmt::Formatter`. The top-level implementation that writes +out the page body is the `impl<'a> fmt::Display for Item<'a>` in `html/render.rs`, which switches +out to one of several `item_*` functions based on the kind of `Item` being rendered. + +Depending on what kind of rendering code you're looking for, you'll probably find it either in +`html/render.rs` for major items like "what sections should I print for a struct page" or +`html/format.rs` for smaller component pieces like "how should I print a where clause as part of +some other item". + +Whenever rustdoc comes across an item that should print hand-written documentation alongside, it +calls out to `html/markdown.rs` which interfaces with the Markdown parser. This is exposed as a +series of types that wrap a string of Markdown, and implement `fmt::Display` to emit HTML text. It +takes special care to enable certain features like footnotes and tables and add syntax highlighting +to Rust code blocks (via `html/highlight.rs`) before running the Markdown parser. There's also a +function in here (`find_testable_code`) that specifically scans for Rust code blocks so the +test-runner code can find all the doctests in the crate. + +### From soup to nuts + +(alternate title: ["An unbroken thread that stretches from those first `Cell`s to us"][video]) + +[video]: https://www.youtube.com/watch?v=hOLAGYmUQV0 + +It's important to note that the AST cleaning can ask the compiler for information (crucially, +`DocContext` contains a `TyCtxt`), but page rendering cannot. The `clean::Crate` created within +`run_core` is passed outside the compiler context before being handed to `html::render::run`. This +means that a lot of the "supplementary data" that isn't immediately available inside an item's +definition, like which trait is the `Deref` trait used by the language, needs to be collected during +cleaning, stored in the `DocContext`, and passed along to the `SharedContext` during HTML rendering. +This manifests as a bunch of shared state, context variables, and `RefCell`s. + +Also of note is that some items that come from "asking the compiler" don't go directly into the +`DocContext` - for example, when loading items from a foreign crate, rustdoc will ask about trait +implementations and generate new `Item`s for the impls based on that information. This goes directly +into the returned `Crate` rather than roundabout through the `DocContext`. This way, these +implementations can be collected alongside the others, right before rendering the HTML. + +## Other tricks up its sleeve + +All this describes the process for generating HTML documentation from a Rust crate, but there are +couple other major modes that rustdoc runs in. It can also be run on a standalone Markdown file, or +it can run doctests on Rust code or standalone Markdown files. For the former, it shortcuts straight +to `html/markdown.rs`, optionally including a mode which inserts a Table of Contents to the output +HTML. + +For the latter, rustdoc runs a similar partial-compilation to get relevant documentation in +`test.rs`, but instead of going through the full clean and render process, it runs a much simpler +crate walk to grab *just* the hand-written documentation. Combined with the aforementioned +"`find_testable_code`" in `html/markdown.rs`, it builds up a collection of tests to run before +handing them off to the libtest test runner. One notable location in `test.rs` is the function +`make_test`, which is where hand-written doctests get transformed into something that can be +executed. + +## Dotting i's and crossing t's + +So that's rustdoc's code in a nutshell, but there's more things in the repo that deal with it. Since +we have the full `compiletest` suite at hand, there's a set of tests in `src/test/rustdoc` that make +sure the final HTML is what we expect in various situations. These tests also use a supplementary +script, `src/etc/htmldocck.py`, that allows it to look through the final HTML using XPath notation +to get a precise look at the output. The full description of all the commands available to rustdoc +tests is in `htmldocck.py`. + +In addition, there are separate tests for the search index and rustdoc's ability to query it. The +files in `src/test/rustdoc-js` each contain a different search query and the expected results, +broken out by search tab. These files are processed by a script in `src/tools/rustdoc-js` and the +Node.js runtime. These tests don't have as thorough of a writeup, but a broad example that features +results in all tabs can be found in `basic.js`. The basic idea is that you match a given `QUERY` +with a set of `EXPECTED` results, complete with the full item path of each item. From 8d893c1e9e136302a1640035b29e23023f87866e Mon Sep 17 00:00:00 2001 From: QuietMisdreavus Date: Tue, 20 Feb 2018 09:26:44 -0600 Subject: [PATCH 02/11] review nits --- src/librustdoc/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/librustdoc/README.md b/src/librustdoc/README.md index 844c54c7aa88c..b0a5ae3718df3 100644 --- a/src/librustdoc/README.md +++ b/src/librustdoc/README.md @@ -28,8 +28,8 @@ that's the high-level overview. CSS/JS and landing page are. * Most of the HTML printing code is in `html/format.rs` and `html/render.rs`. It's in a bunch of `fmt::Display` implementations and supplementary functions. -* The types that operates on are defined in `clean/mod.rs`, right next to the custom `Clean` trait - used to process them out of the rustc HIR. +* The types that got `Display` impls above are defined in `clean/mod.rs`, right next to the custom + `Clean` trait used to process them out of the rustc HIR. * The bits specific to using rustdoc as a test harness are in `test.rs`. * The Markdown renderer is loaded up in `html/markdown.rs`, including functions for extracting doctests from a given block of Markdown. @@ -68,10 +68,12 @@ and inlining. This is where `#[doc(inline)]`, `#[doc(no_inline)]`, and `#[doc(hi processed, as well as the logic for whether a `pub use` should get the full page or a "Reexport" line in the module page. -Back in `clean/mod.rs`, the other major thing that happens here is the special collection of doc -comments (and basic `#[doc=""]` attributes) into a separate field in the `Attributes` struct from -the other attributes. The primary output of this process is a `clean::Crate` with a tree of `Item`s -which describe the publicly-documentable items in the target crate. +The other major thing that happens in `clean/mod.rs` is the collection of doc comments and +`#[doc=""]` attributes into a separate field of the Attributes struct, present on anything that gets +hand-written documentation. This makes it easier to collect this documentation later in the process. + +The primary output of this process is a clean::Crate with a tree of Items which describe the +publicly-documentable items in the target crate. ### Hot potato From 8e4ad29cb9cc74bf44486f4507f36df70d495187 Mon Sep 17 00:00:00 2001 From: Seo Sanghyeon Date: Fri, 23 Feb 2018 21:48:41 +0900 Subject: [PATCH 03/11] Disable NEON on musl ARMv7 --- .../target/armv7_unknown_linux_musleabihf.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/librustc_back/target/armv7_unknown_linux_musleabihf.rs b/src/librustc_back/target/armv7_unknown_linux_musleabihf.rs index a36e26c0b7d5f..88f2b59675186 100644 --- a/src/librustc_back/target/armv7_unknown_linux_musleabihf.rs +++ b/src/librustc_back/target/armv7_unknown_linux_musleabihf.rs @@ -12,13 +12,7 @@ use LinkerFlavor; use target::{Target, TargetOptions, TargetResult}; pub fn target() -> TargetResult { - let mut base = super::linux_musl_base::opts(); - - // Most of these settings are copied from the armv7_unknown_linux_gnueabihf - // target. - base.features = "+v7,+vfp3,+neon".to_string(); - base.cpu = "cortex-a8".to_string(); - base.max_atomic_width = Some(64); + let base = super::linux_musl_base::opts(); Ok(Target { // It's important we use "gnueabihf" and not "musleabihf" here. LLVM // uses it to determine the calling convention and float ABI, and LLVM @@ -33,9 +27,15 @@ pub fn target() -> TargetResult { target_env: "musl".to_string(), target_vendor: "unknown".to_string(), linker_flavor: LinkerFlavor::Gcc, + + // Most of these settings are copied from the armv7_unknown_linux_gnueabihf + // target. options: TargetOptions { + features: "+v7,+vfp3,+d16,+thumb2,-neon".to_string(), + cpu: "generic".to_string(), + max_atomic_width: Some(64), abi_blacklist: super::arm_base::abi_blacklist(), .. base - }, + } }) } From ffb6291cd032e3b0577bf67a2b6d932d6ba0787c Mon Sep 17 00:00:00 2001 From: Phlosioneer Date: Mon, 26 Feb 2018 20:45:52 -0500 Subject: [PATCH 04/11] Improve --help performance for x.py Since compiling the bootstrap command doesn't require any submodules, we can skip updating submodules when a --help command is passed in. On my machine, this saves 1 minute if the submodules are already downloaded, and 10 minutes if run on a clean repo. This commit also adds a message before compiling/downloading anything when a --help command is passed in, to tell the user WHY --help takes so long to complete. It also points the user to the bootstrap README.md for faster help. Finally, this fixes one warning message that still referenced using make instead of x.py, even though x.py is now the standard way of building rust. --- src/bootstrap/bootstrap.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 603a97ddfd412..f5fcf4ba0f560 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -670,7 +670,7 @@ def set_dev_environment(self): self._download_url = 'https://dev-static.rust-lang.org' -def bootstrap(): +def bootstrap(help_triggered): """Configure, fetch, build and run the initial bootstrap""" parser = argparse.ArgumentParser(description='Build rust') parser.add_argument('--config') @@ -708,7 +708,7 @@ def bootstrap(): print(' and so in order to preserve your $HOME this will now') print(' use vendored sources by default. Note that if this') print(' does not work you should run a normal build first') - print(' before running a command like `sudo make install`') + print(' before running a command like `sudo ./x.py install`') if build.use_vendored_sources: if not os.path.exists('.cargo'): @@ -734,7 +734,10 @@ def bootstrap(): if 'dev' in data: build.set_dev_environment() - build.update_submodules() + # No help text depends on submodules. This check saves ~1 minute of git commands, even if + # all the submodules are present and downloaded! + if not help_triggered: + build.update_submodules() # Fetch/build the bootstrap build.build = args.build or build.build_triple() @@ -760,7 +763,13 @@ def main(): help_triggered = ( '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) try: - bootstrap() + # If the user is asking for help, let them know that the whole download-and-build + # process has to happen before anything is printed out. + if help_triggered: + print("NOTE: Downloading and compiling bootstrap requirements before processing") + print(" --help command. See src/bootstrap/README.md for help with common") + print(" commands.") + bootstrap(help_triggered) if not help_triggered: print("Build completed successfully in {}".format( format_build_time(time() - start_time))) From 5ac4f62f3832f2a4ed26ab95567c96a38a9f4f15 Mon Sep 17 00:00:00 2001 From: M Farkas-Dyck Date: Wed, 28 Feb 2018 20:57:56 -0800 Subject: [PATCH 05/11] impl Clone for ::std_unicode::char::{ToLowercase, ToUppercase} --- src/libstd_unicode/char.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libstd_unicode/char.rs b/src/libstd_unicode/char.rs index 844ff7a3c1252..5dd9c62775097 100644 --- a/src/libstd_unicode/char.rs +++ b/src/libstd_unicode/char.rs @@ -59,7 +59,7 @@ pub use version::UnicodeVersion; /// [`to_lowercase`]: ../../std/primitive.char.html#method.to_lowercase /// [`char`]: ../../std/primitive.char.html #[stable(feature = "rust1", since = "1.0.0")] -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ToLowercase(CaseMappingIter); #[stable(feature = "rust1", since = "1.0.0")] @@ -81,7 +81,7 @@ impl FusedIterator for ToLowercase {} /// [`to_uppercase`]: ../../std/primitive.char.html#method.to_uppercase /// [`char`]: ../../std/primitive.char.html #[stable(feature = "rust1", since = "1.0.0")] -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ToUppercase(CaseMappingIter); #[stable(feature = "rust1", since = "1.0.0")] @@ -95,7 +95,7 @@ impl Iterator for ToUppercase { #[unstable(feature = "fused", issue = "35602")] impl FusedIterator for ToUppercase {} -#[derive(Debug)] +#[derive(Debug, Clone)] enum CaseMappingIter { Three(char, char, char), Two(char, char), From 39d0b054cea73614a0c1dd06cd98d3d22eea6b10 Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Thu, 1 Mar 2018 14:47:20 +0200 Subject: [PATCH 06/11] Restore the download of rust-mingw The build might otherwise break due to mixing MinGW object files from rust-std and the local MinGW which might be newer/older than the version used to build rust-std. Fixes #48272 --- src/bootstrap/bootstrap.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 5966bb65df9c8..761cc94c8c66f 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -351,6 +351,11 @@ def download_stage0(self): with open(self.rustc_stamp(), 'w') as rust_stamp: rust_stamp.write(self.date) + if "pc-windows-gnu" in self.build: + filename = "rust-mingw-{}-{}.tar.gz".format( + rustc_channel, self.build) + self._download_stage0_helper(filename, "rust-mingw") + if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.program_out_of_date(self.cargo_stamp())): From 5332d9a8e8a66e22b8c81f3f302ce72734ba137f Mon Sep 17 00:00:00 2001 From: Segev Finer Date: Thu, 1 Mar 2018 18:33:14 +0200 Subject: [PATCH 07/11] Document why we download rust-mingw --- src/bootstrap/bootstrap.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 761cc94c8c66f..5d297570bc12c 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -351,6 +351,9 @@ def download_stage0(self): with open(self.rustc_stamp(), 'w') as rust_stamp: rust_stamp.write(self.date) + # This is required so that we don't mix incompatible MinGW + # libraries/binaries that are included in rust-std with + # the system MinGW ones. if "pc-windows-gnu" in self.build: filename = "rust-mingw-{}-{}.tar.gz".format( rustc_channel, self.build) From 2269ff521fe2a945e462d5156799011c5fce2966 Mon Sep 17 00:00:00 2001 From: Phlosioneer Date: Fri, 2 Mar 2018 03:29:35 -0500 Subject: [PATCH 08/11] Remove print_what_bootstrap_means It was an existing solution to tell the user why a --help command takes a long time to process. However, it would only print if the stage0 rust compiler needed to be downloaded, it came after update_submodules (which took a long time), and it was immediately followed by download messages and loading bars, meaning users could easily gloss over the message. This commit also moves the help message out of main(), and instead puts it at the top of bootstrap(). main() is intended to be minimal, only handling error messages. --- src/bootstrap/bootstrap.py | 35 ++++++++--------------------------- 1 file changed, 8 insertions(+), 27 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index f5fcf4ba0f560..327ae0cb65c88 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -314,7 +314,6 @@ def __init__(self): self.build_dir = os.path.join(os.getcwd(), "build") self.clean = False self.config_toml = '' - self.printed = False self.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) self.use_locked_deps = '' self.use_vendored_sources = '' @@ -336,7 +335,6 @@ def download_stage0(self): if self.rustc().startswith(self.bin_root()) and \ (not os.path.exists(self.rustc()) or self.program_out_of_date(self.rustc_stamp())): - self.print_what_bootstrap_means() if os.path.exists(self.bin_root()): shutil.rmtree(self.bin_root()) filename = "rust-std-{}-{}.tar.gz".format( @@ -354,7 +352,6 @@ def download_stage0(self): if self.cargo().startswith(self.bin_root()) and \ (not os.path.exists(self.cargo()) or self.program_out_of_date(self.cargo_stamp())): - self.print_what_bootstrap_means() filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build) self._download_stage0_helper(filename, "cargo") self.fix_executable("{}/bin/cargo".format(self.bin_root())) @@ -555,23 +552,6 @@ def exe_suffix(): return '.exe' return '' - def print_what_bootstrap_means(self): - """Prints more information about the build system""" - if hasattr(self, 'printed'): - return - self.printed = True - if os.path.exists(self.bootstrap_binary()): - return - if '--help' not in sys.argv or len(sys.argv) == 1: - return - - print('info: the build system for Rust is written in Rust, so this') - print(' script is now going to download a stage0 rust compiler') - print(' and then compile the build system itself') - print('') - print('info: in the meantime you can read more about rustbuild at') - print(' src/bootstrap/README.md before the download finishes') - def bootstrap_binary(self): """Return the path of the boostrap binary @@ -585,7 +565,6 @@ def bootstrap_binary(self): def build_bootstrap(self): """Build bootstrap""" - self.print_what_bootstrap_means() build_dir = os.path.join(self.build_dir, "bootstrap") if self.clean and os.path.exists(build_dir): shutil.rmtree(build_dir) @@ -672,6 +651,14 @@ def set_dev_environment(self): def bootstrap(help_triggered): """Configure, fetch, build and run the initial bootstrap""" + + # If the user is asking for help, let them know that the whole download-and-build + # process has to happen before anything is printed out. + if help_triggered: + print("info: Downloading and building bootstrap before processing --help") + print(" command. See src/bootstrap/README.md for help with common") + print(" commands.") + parser = argparse.ArgumentParser(description='Build rust') parser.add_argument('--config') parser.add_argument('--build') @@ -763,12 +750,6 @@ def main(): help_triggered = ( '-h' in sys.argv) or ('--help' in sys.argv) or (len(sys.argv) == 1) try: - # If the user is asking for help, let them know that the whole download-and-build - # process has to happen before anything is printed out. - if help_triggered: - print("NOTE: Downloading and compiling bootstrap requirements before processing") - print(" --help command. See src/bootstrap/README.md for help with common") - print(" commands.") bootstrap(help_triggered) if not help_triggered: print("Build completed successfully in {}".format( From 6250a47ea6147a60e11e59aded80ebb62c8e3ab1 Mon Sep 17 00:00:00 2001 From: Marc-Antoine Perennou Date: Fri, 2 Mar 2018 09:19:50 +0100 Subject: [PATCH 09/11] make codegen-backends directory name configurable This allows to parallel-install several versions of rust system-wide Fixes #48263 Signed-off-by: Marc-Antoine Perennou --- config.toml.example | 3 +++ src/bootstrap/builder.rs | 2 +- src/bootstrap/compile.rs | 3 ++- src/bootstrap/config.rs | 5 +++++ src/bootstrap/dist.rs | 3 ++- src/librustc_driver/lib.rs | 4 +++- 6 files changed, 16 insertions(+), 4 deletions(-) diff --git a/config.toml.example b/config.toml.example index 8d1fa3eec5cf2..3dfd25aade1e4 100644 --- a/config.toml.example +++ b/config.toml.example @@ -321,6 +321,9 @@ # bootstrap) #codegen-backends = ["llvm"] +# This is the name of the directory in which codegen backends will get installed +#codegen-backends-dir = "codegen-backends" + # Flag indicating whether `libstd` calls an imported function to handle basic IO # when targeting WebAssembly. Enable this to debug tests for the `wasm32-unknown-unknown` # target, as without this option the test output will not be captured. diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index b5946b44e05ef..2eec6c69739b7 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -462,7 +462,7 @@ impl<'a> Builder<'a> { pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf { self.sysroot_libdir(compiler, compiler.host) - .with_file_name("codegen-backends") + .with_file_name(self.build.config.rust_codegen_backends_dir.clone()) } /// Returns the compiler's libdir where it stores the dynamic libraries that diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 2c9f0ddb6c33d..f8e0a1f5a923c 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -514,7 +514,8 @@ fn rustc_cargo_env(build: &Build, cargo: &mut Command) { cargo.env("CFG_RELEASE", build.rust_release()) .env("CFG_RELEASE_CHANNEL", &build.config.channel) .env("CFG_VERSION", build.rust_version()) - .env("CFG_PREFIX", build.config.prefix.clone().unwrap_or_default()); + .env("CFG_PREFIX", build.config.prefix.clone().unwrap_or_default()) + .env("CFG_CODEGEN_BACKENDS_DIR", &build.config.rust_codegen_backends_dir); let libdir_relative = build.config.libdir_relative().unwrap_or(Path::new("lib")); cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 6bc20181a0330..361fc704bc07b 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -96,6 +96,7 @@ pub struct Config { pub rust_debuginfo_tests: bool, pub rust_dist_src: bool, pub rust_codegen_backends: Vec>, + pub rust_codegen_backends_dir: String, pub build: Interned, pub hosts: Vec>, @@ -289,6 +290,7 @@ struct Rust { test_miri: Option, save_toolstates: Option, codegen_backends: Option>, + codegen_backends_dir: Option, wasm_syscall: Option, } @@ -330,6 +332,7 @@ impl Config { config.rust_dist_src = true; config.test_miri = false; config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; + config.rust_codegen_backends_dir = "codegen-backends".to_owned(); config.rustc_error_format = flags.rustc_error_format; config.on_fail = flags.on_fail; @@ -488,6 +491,8 @@ impl Config { .collect(); } + set(&mut config.rust_codegen_backends_dir, rust.codegen_backends_dir.clone()); + match rust.codegen_units { Some(0) => config.rust_codegen_units = Some(num_cpus::get() as u32), Some(n) => config.rust_codegen_units = Some(n), diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e7aed7eb4fead..05630b8431fb5 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -590,7 +590,8 @@ impl Step for Std { let mut src = builder.sysroot_libdir(compiler, target).to_path_buf(); src.pop(); // Remove the trailing /lib folder from the sysroot_libdir cp_filtered(&src, &dst, &|path| { - path.file_name().and_then(|s| s.to_str()) != Some("codegen-backends") + path.file_name().and_then(|s| s.to_str()) != + Some(build.config.rust_codegen_backends_dir.as_str()) }); let mut cmd = rust_installer(builder); diff --git a/src/librustc_driver/lib.rs b/src/librustc_driver/lib.rs index 22c26b0643050..0f7a04e391bfb 100644 --- a/src/librustc_driver/lib.rs +++ b/src/librustc_driver/lib.rs @@ -303,7 +303,9 @@ fn get_trans_sysroot(backend_name: &str) -> fn() -> Box { let sysroot = sysroot_candidates.iter() .map(|sysroot| { let libdir = filesearch::relative_target_lib_path(&sysroot, &target); - sysroot.join(libdir).with_file_name("codegen-backends") + sysroot.join(libdir) + .with_file_name(option_env!("CFG_CODEGEN_BACKENDS_DIR") + .unwrap_or("codegen-backends")) }) .filter(|f| { info!("codegen backend candidate: {}", f.display()); From 900d511fdca20998e42634d5351f7033bbde1aed Mon Sep 17 00:00:00 2001 From: steveklabnik Date: Fri, 2 Mar 2018 13:52:13 -0500 Subject: [PATCH 10/11] Don't produce TOCs for doc markdown files Currently, we are producing headers for markdown files, which is generally not what we want. As such, passing this flag causes them to render normally. --- src/bootstrap/doc.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index 55d9723527e6d..a791dd13f0f4b 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -312,6 +312,7 @@ fn invoke_rustdoc(builder: &Builder, compiler: Compiler, target: Interned Date: Sat, 17 Feb 2018 17:23:19 -0800 Subject: [PATCH 11/11] std: Add `arch` and `simd` modules This commit imports the `stdsimd` crate into the standard library, creating an `arch` and `simd` module inside of both libcore and libstd. Both of these modules are **unstable** and will continue to be so until RFC 2335 is stabilized. As a brief recap, the modules are organized as so: * `arch` contains all current architectures with intrinsics, for example `std::arch::x86`, `std::arch::x86_64`, `std::arch::arm`, etc. These modules contain all of the intrinsics defined for the platform, like `_mm_set1_epi8`. * In the standard library, the `arch` module also exports a `is_target_feature_detected` macro which performs runtime detection to determine whether a target feature is available at runtime. * The `simd` module contains experimental versions of strongly-typed lane-aware SIMD primitives, to be fully fleshed out in a future RFC. The main purpose of this commit is to start pulling in all these intrinsics and such into the standard library on nightly and allow testing and such. This'll help allow users to easily kick the tires and see if intrinsics work as well as allow us to test out all the infrastructure for moving the intrinsics into the standard library. --- .gitmodules | 3 +++ src/libcore/lib.rs | 33 +++++++++++++++++++++++++++++---- src/libstd/lib.rs | 30 ++++++++++++++++++++++++++++++ src/stdsimd | 1 + src/tools/tidy/src/lib.rs | 1 + 5 files changed, 64 insertions(+), 4 deletions(-) create mode 160000 src/stdsimd diff --git a/.gitmodules b/.gitmodules index fc2f8bbc8a350..5b7fd48129929 100644 --- a/.gitmodules +++ b/.gitmodules @@ -50,3 +50,6 @@ [submodule "src/llvm-emscripten"] path = src/llvm-emscripten url = https://github.com/rust-lang/llvm +[submodule "src/stdsimd"] + path = src/stdsimd + url = https://github.com/rust-lang-nursery/stdsimd diff --git a/src/libcore/lib.rs b/src/libcore/lib.rs index 3dd30ee1c69e2..1efd605112dc2 100644 --- a/src/libcore/lib.rs +++ b/src/libcore/lib.rs @@ -68,16 +68,21 @@ #![feature(allow_internal_unstable)] #![feature(asm)] #![feature(associated_type_defaults)] +#![feature(attr_literals)] #![feature(cfg_target_feature)] #![feature(cfg_target_has_atomic)] #![feature(concat_idents)] #![feature(const_fn)] #![feature(custom_attribute)] +#![feature(doc_spotlight)] #![feature(fundamental)] #![feature(i128_type)] #![feature(inclusive_range_syntax)] #![feature(intrinsics)] +#![feature(iterator_flatten)] +#![feature(iterator_repeat_with)] #![feature(lang_items)] +#![feature(link_llvm_intrinsics)] #![feature(never_type)] #![feature(no_core)] #![feature(on_unimplemented)] @@ -85,15 +90,17 @@ #![feature(prelude_import)] #![feature(repr_simd, platform_intrinsics)] #![feature(rustc_attrs)] +#![feature(rustc_const_unstable)] +#![feature(simd_ffi)] #![feature(specialization)] #![feature(staged_api)] +#![feature(stmt_expr_attributes)] +#![feature(target_feature)] #![feature(unboxed_closures)] #![feature(untagged_unions)] #![feature(unwind_attributes)] -#![feature(doc_spotlight)] -#![feature(rustc_const_unstable)] -#![feature(iterator_repeat_with)] -#![feature(iterator_flatten)] + +#![cfg_attr(stage0, allow(unused_attributes))] #[prelude_import] #[allow(unused)] @@ -179,3 +186,21 @@ mod char_private; mod iter_private; mod tuple; mod unit; + +// Pull in the the `coresimd` crate directly into libcore. This is where all the +// architecture-specific (and vendor-specific) intrinsics are defined. AKA +// things like SIMD and such. Note that the actual source for all this lies in a +// different repository, rust-lang-nursery/stdsimd. That's why the setup here is +// a bit wonky. +#[path = "../stdsimd/coresimd/mod.rs"] +#[allow(missing_docs, missing_debug_implementations, dead_code)] +#[unstable(feature = "stdsimd", issue = "48556")] +#[cfg(not(stage0))] // allow changes to how stdsimd works in stage0 +mod coresimd; + +#[unstable(feature = "stdsimd", issue = "48556")] +#[cfg(not(stage0))] +pub use coresimd::simd; +#[unstable(feature = "stdsimd", issue = "48556")] +#[cfg(not(stage0))] +pub use coresimd::arch; diff --git a/src/libstd/lib.rs b/src/libstd/lib.rs index d7d856fe3ad06..a7e1c0ce732e0 100644 --- a/src/libstd/lib.rs +++ b/src/libstd/lib.rs @@ -299,6 +299,7 @@ #![feature(rand)] #![feature(raw)] #![feature(rustc_attrs)] +#![feature(stdsimd)] #![feature(sip_hash_13)] #![feature(slice_bytes)] #![feature(slice_concat_ext)] @@ -501,6 +502,35 @@ mod memchr; // compiler pub mod rt; +// Pull in the the `stdsimd` crate directly into libstd. This is the same as +// libcore's arch/simd modules where the source of truth here is in a different +// repository, but we pull things in here manually to get it into libstd. +// +// Note that the #[cfg] here is intended to do two things. First it allows us to +// change the rustc implementation of intrinsics in stage0 by not compiling simd +// intrinsics in stage0. Next it doesn't compile anything in test mode as +// stdsimd has tons of its own tests which we don't want to run. +#[path = "../stdsimd/stdsimd/mod.rs"] +#[allow(missing_debug_implementations, missing_docs, dead_code)] +#[unstable(feature = "stdsimd", issue = "48556")] +#[cfg(all(not(stage0), not(test)))] +mod stdsimd; + +// A "fake" module needed by the `stdsimd` module to compile, not actually +// exported though. +#[cfg(not(stage0))] +mod coresimd { + pub use core::arch; + pub use core::simd; +} + +#[unstable(feature = "stdsimd", issue = "48556")] +#[cfg(all(not(stage0), not(test)))] +pub use stdsimd::simd; +#[unstable(feature = "stdsimd", issue = "48556")] +#[cfg(all(not(stage0), not(test)))] +pub use stdsimd::arch; + // Include a number of private modules that exist solely to provide // the rustdoc documentation for primitive types. Using `include!` // because rustdoc only looks for these modules at the crate level. diff --git a/src/stdsimd b/src/stdsimd new file mode 160000 index 0000000000000..678cbd325c840 --- /dev/null +++ b/src/stdsimd @@ -0,0 +1 @@ +Subproject commit 678cbd325c84070c9dbe4303969fbd2734c0b4ee diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs index 4d89008d5ca54..1def3048ce071 100644 --- a/src/tools/tidy/src/lib.rs +++ b/src/tools/tidy/src/lib.rs @@ -71,6 +71,7 @@ fn filter_dirs(path: &Path) -> bool { "src/librustc/mir/interpret", "src/librustc_mir/interpret", "src/target", + "src/stdsimd", ]; skip.iter().any(|p| path.ends_with(p)) }