Skip to content

Commit

Permalink
Switch to api.ast.json.gz instead of api.ast.json
Browse files Browse the repository at this point in the history
api.ast.json consumes 2.5GB in the output directory, compressing it
saves 2.4GB of storage. The saving is useful in devhouse and for
github actions (linux SSD size 14GB).

Test: build bazel //... && bazel test //...
  • Loading branch information
ohodson committed Mar 10, 2023
1 parent d77187d commit 373a11b
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 5 deletions.
2 changes: 1 addition & 1 deletion build/cc_ast_dump.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ def _cc_ast_dump_impl(ctx):
inputs = depset(direct = [ctx.file.src], transitive = [cc_toolchain.all_files] + [cc_info.compilation_context.headers])
env = cc_common.get_environment_variables(feature_configuration = feature_configuration, action_name = CPP_COMPILE_ACTION_NAME, variables = variables)

command = " ".join([executable] + arguments + [">", ctx.outputs.out.path])
command = " ".join([executable] + arguments + ["|", "gzip", "-3", "-", ">", ctx.outputs.out.path])

# run_shell until https://github.com/bazelbuild/bazel/issues/5511 is fixed
ctx.actions.run_shell(
Expand Down
3 changes: 3 additions & 0 deletions rust-deps/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,9 @@ crates_vendor(
"clang-ast": crate.spec(
version = "0.1",
),
"flate2": crate.spec(
version = "1.0.24",
),
"serde": crate.spec(
version = "1.0",
features = ["default", "derive"],
Expand Down
6 changes: 6 additions & 0 deletions rust-deps/crates/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,12 @@ alias(
tags = ["manual"],
)

alias(
name = "flate2",
actual = "@crates_vendor__flate2-1.0.24//:flate2",
tags = ["manual"],
)

alias(
name = "libc",
actual = "@crates_vendor__libc-0.2.137//:libc",
Expand Down
3 changes: 2 additions & 1 deletion src/workerd/tools/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,7 @@ filegroup(
cc_ast_dump(
name = "dump_api_ast",
src = api_encoder_src,
out = "api.ast.json",
out = "api.ast.json.gz",
deps = [":api_encoder_lib"],
)

Expand All @@ -99,6 +99,7 @@ rust_binary(
deps = [
"@crates_vendor//:anyhow",
"@crates_vendor//:clang-ast",
"@crates_vendor//:flate2",
"@crates_vendor//:pico-args",
"@crates_vendor//:serde",
"@crates_vendor//:serde_json",
Expand Down
13 changes: 10 additions & 3 deletions src/workerd/tools/param-extractor.rs
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
use std::{
ffi::OsStr,
fs::File,
io::{BufReader, BufWriter, Write},
path::Path,
};

use anyhow::Result;
use flate2::read::GzDecoder;
use serde::{Deserialize, Serialize};

/// Contains the declarations we care about
Expand Down Expand Up @@ -50,10 +53,14 @@ type ClangNode = clang_ast::Node<Clang>;
fn main() -> Result<()> {
let mut args = pico_args::Arguments::from_env();

let clang_ast = args.value_from_os_str("--input", |path| {
let clang_ast = args.value_from_os_str("--input", |path_str| {
let path = Path::new(path_str);
let file = File::open(path)?;
let rdr = BufReader::new(file);
serde_json::from_reader(rdr).map_err(anyhow::Error::from)
let serde = match path.extension().and_then(OsStr::to_str) {
Some("gz") => serde_json::from_reader(BufReader::new(GzDecoder::new(file))),
_ => serde_json::from_reader(BufReader::new(file))
};
serde.map_err(anyhow::Error::from)
})?;

let value = get_parameter_names(clang_ast);
Expand Down

0 comments on commit 373a11b

Please # to comment.