Improve create_input and create_output (#115)

* Replace `brioche_test` module with `brioche_test_support` crate * Rewrite benchmarks from Criterion to Divan * Add more test cases around `Directory` * Add benchmark for `Directory::insert` * Tweak blob saving to use a mut ref for the permit * Add benchmark for saving blobs * Add input benchmark * Add `profiling` Cargo profile * Allow reusing buffers when creating blobs * Cache input paths to speed up `create_input` with resources * Update `save_blob_from_file` to use a blocking task for reading/hashing * Refactor resource creation in `create_input_inner` with a separate function * Add new benchmark for resources with common ancestor * Refactor `create_input` to use a graph for building artifacts * Update `create_input` to avoid redundant graph traversals * Update `create_input` to avoid traversing redundant resources * Update `create_input` to create blobs in parallel * Fix `create_input` removing resource files * Add more log messages to `create_input` * Remove unused functions in `input` module * Remove unused `buffer` param from `create_input_inner` * Update `create_output` to avoid trying to write resources multiple times * Fix handling of broken symlinks in `create_input` * Improve symlink handling in `create_input` * Fix `create_input` when a resource could not be found * Update `create_input` to remove input path if `remove_input` is set * Fix test failures from `create_output` changes * Add extra logging to `create_output` * Improve how `create_output` skips existing outputs * Combine `create_input_inner` into `create_input` * Add some comments to `create_input`
brioche-dev · Aug 26, 2024 · 0b8cbd6 · 0b8cbd6
1 parent 3517aec
commit 0b8cbd6
Show file tree

Hide file tree

Showing 41 changed files with 2,747 additions and 1,494 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,12 +1,16 @@
 [workspace]
-members = ["crates/brioche", "crates/brioche-core", "crates/brioche-pack"]
+members = ["crates/brioche", "crates/brioche-core", "crates/brioche-pack", "crates/brioche-test-support"]
 resolver = "2"
 
 [profile.dev]
 # Enable optimizations for debug builds because the JS runs slow without it.
 # Also, we end up overflowing the stack without optimizations when parsing JS
 opt-level = 2
 
+[profile.profiling]
+inherits = "release"
+debug = true
+
 [profile.release-tiny]
 inherits = "release"
 opt-level = "z"

diff --git a/crates/brioche-core/Cargo.toml b/crates/brioche-core/Cargo.toml
@@ -63,13 +63,14 @@ ulid = "1.1.0"
 url = { version = "2.5.0", features = ["serde"] }
 urlencoding = "2.1.3"
 walkdir = "2.5.0"
+petgraph = "0.6.5"
 
 [dev-dependencies]
 assert_matches = "1.5.0"
-criterion = { version = "0.5.1", features = ["async_tokio"] }
+brioche-test-support = { path = "../brioche-test-support" }
+divan = "0.1.14"
 mockito = "1.2.0"
 pretty_assertions = "1.4.0"
-tempdir = "0.3.7"
 zstd = "0.13.1"
 
 [target.'cfg(target_os = "linux")'.dependencies]
@@ -80,6 +81,18 @@ unshare = { git = "https://github.com/brioche-dev/unshare.git" }
 name = "bake"
 harness = false
 
+[[bench]]
+name = "blob"
+harness = false
+
+[[bench]]
+name = "directory"
+harness = false
+
+[[bench]]
+name = "input"
+harness = false
+
 [lints.clippy]
 # Temporarily ignore false positives from `tracing::instrument`, see:
 # https://github.com/rust-lang/rust-clippy/issues/12281

diff --git a/crates/brioche-core/benches/bake.rs b/crates/brioche-core/benches/bake.rs
@@ -2,10 +2,172 @@ use brioche_core::{
     recipe::{Directory, Recipe, WithMeta},
     Brioche,
 };
-use criterion::{criterion_group, criterion_main, Criterion};
-use futures::StreamExt as _;
 
-mod brioche_bench;
+fn main() {
+    divan::main();
+}
+
+#[divan::bench]
+fn cached_bake_deep_dir(bencher: divan::Bencher) {
+    let RecipesContext {
+        brioche,
+        runtime,
+        recipes,
+        ..
+    } = set_up_bench();
+
+    bencher.bench_local(|| {
+        runtime.block_on(async {
+            let deep_dir = WithMeta::without_meta(Recipe::from(recipes.deep_dir.clone()));
+            let _ = brioche_core::bake::bake(
+                &brioche,
+                deep_dir,
+                &brioche_core::bake::BakeScope::Anonymous,
+            )
+            .await
+            .unwrap();
+        });
+    })
+}
+
+#[divan::bench]
+fn cached_bake_wide_dir(bencher: divan::Bencher) {
+    let RecipesContext {
+        brioche,
+        runtime,
+        recipes,
+        ..
+    } = set_up_bench();
+
+    bencher.bench_local(|| {
+        runtime.block_on(async {
+            let wide_dir = WithMeta::without_meta(Recipe::from(recipes.wide_dir.clone()));
+            let _ = brioche_core::bake::bake(
+                &brioche,
+                wide_dir,
+                &brioche_core::bake::BakeScope::Anonymous,
+            )
+            .await
+            .unwrap();
+        });
+    })
+}
+
+#[divan::bench]
+fn cached_bake_deep_merge(bencher: divan::Bencher) {
+    let RecipesContext {
+        brioche,
+        runtime,
+        recipes,
+        ..
+    } = set_up_bench();
+
+    bencher.bench_local(|| {
+        runtime.block_on(async {
+            let merge_deep_dir = WithMeta::without_meta(recipes.merge_deep_dir.clone());
+            let _ = brioche_core::bake::bake(
+                &brioche,
+                merge_deep_dir,
+                &brioche_core::bake::BakeScope::Anonymous,
+            )
+            .await
+            .unwrap();
+        });
+    })
+}
+
+#[divan::bench]
+fn cached_bake_wide_merge(bencher: divan::Bencher) {
+    let RecipesContext {
+        brioche,
+        runtime,
+        recipes,
+        ..
+    } = set_up_bench();
+
+    bencher.bench_local(|| {
+        runtime.block_on(async {
+            let merge_wide_dir = WithMeta::without_meta(recipes.merge_wide_dir.clone());
+            let _ = brioche_core::bake::bake(
+                &brioche,
+                merge_wide_dir,
+                &brioche_core::bake::BakeScope::Anonymous,
+            )
+            .await
+            .unwrap();
+        });
+    })
+}
+
+fn set_up_bench() -> RecipesContext {
+    let runtime = tokio::runtime::Builder::new_current_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+    let (brioche, context) = runtime.block_on(brioche_test_support::brioche_test());
+
+    let deep_dir = runtime.block_on(make_deep_dir(&brioche, ""));
+    let _deep_dir_result = runtime
+        .block_on(brioche_core::bake::bake(
+            &brioche,
+            WithMeta::without_meta(Recipe::from(deep_dir.clone())),
+            &brioche_core::bake::BakeScope::Anonymous,
+        ))
+        .unwrap();
+
+    let wide_dir = runtime.block_on(make_wide_dir(&brioche, ""));
+    let _wide_dir_result = runtime
+        .block_on(brioche_core::bake::bake(
+            &brioche,
+            WithMeta::without_meta(Recipe::from(wide_dir.clone())),
+            &brioche_core::bake::BakeScope::Anonymous,
+        ))
+        .unwrap();
+
+    let merge_deep_dir = Recipe::Merge {
+        directories: (0..10)
+            .map(|n| {
+                let deep_dir = runtime.block_on(make_deep_dir(&brioche, &n.to_string()));
+                WithMeta::without_meta(Recipe::from(deep_dir))
+            })
+            .collect(),
+    };
+
+    let merge_wide_dir = Recipe::Merge {
+        directories: (0..10)
+            .map(|n| {
+                let wide_dir = runtime.block_on(make_wide_dir(&brioche, &n.to_string()));
+                WithMeta::without_meta(Recipe::from(wide_dir))
+            })
+            .collect(),
+    };
+
+    RecipesContext {
+        brioche,
+        _context: context,
+        runtime,
+        recipes: Recipes {
+            deep_dir,
+            wide_dir,
+            merge_deep_dir,
+            merge_wide_dir,
+        },
+    }
+}
+
+struct RecipesContext {
+    brioche: Brioche,
+    _context: brioche_test_support::TestContext,
+    runtime: tokio::runtime::Runtime,
+    recipes: Recipes,
+}
+
+struct Recipes {
+    deep_dir: Directory,
+    wide_dir: Directory,
+    merge_deep_dir: Recipe,
+    merge_wide_dir: Recipe,
+}
 
 async fn make_deep_dir(brioche: &Brioche, key: &str) -> Directory {
     let mut directory = Directory::default();
@@ -21,8 +183,8 @@ async fn make_deep_dir(brioche: &Brioche, key: &str) -> Directory {
                                     "{key}a{a}/{key}b{b}/{key}c{c}/{key}d{d}/{key}e{e}/file.txt"
                                 )
                                 .as_bytes(),
-                                Some(WithMeta::without_meta(brioche_bench::file(
-                                    brioche_bench::blob(
+                                Some(WithMeta::without_meta(brioche_test_support::file(
+                                    brioche_test_support::blob(
                                         brioche,
                                         format!("a={a},b={b},c={c},d={d},e={e}"),
                                     )
@@ -49,8 +211,8 @@ async fn make_wide_dir(brioche: &Brioche, key: &str) -> Directory {
                 .insert(
                     brioche,
                     format!("{key}a{a}/{key}b{b}/file.txt").as_bytes(),
-                    Some(WithMeta::without_meta(brioche_bench::file(
-                        brioche_bench::blob(brioche, format!("a={a},b={b}")).await,
+                    Some(WithMeta::without_meta(brioche_test_support::file(
+                        brioche_test_support::blob(brioche, format!("a={a},b={b}")).await,
                         false,
                     ))),
                 )
@@ -61,134 +223,3 @@ async fn make_wide_dir(brioche: &Brioche, key: &str) -> Directory {
 
     directory
 }
-
-fn run_bake_benchmark(c: &mut Criterion) {
-    let runtime = tokio::runtime::Builder::new_multi_thread()
-        .enable_all()
-        .build()
-        .expect("failed to build Tokio runtime");
-    let _runtime_guard = runtime.enter();
-
-    struct Recipes {
-        deep_dir: Directory,
-        wide_dir: Directory,
-        merge_deep_dir: Recipe,
-        merge_wide_dir: Recipe,
-    }
-
-    let (brioche, _context, recipes) = runtime.block_on(async {
-        let (brioche, context) = brioche_bench::brioche_test().await;
-
-        let deep_dir = make_deep_dir(&brioche, "").await;
-        let _deep_dir_result = brioche_core::bake::bake(
-            &brioche,
-            WithMeta::without_meta(Recipe::from(deep_dir.clone())),
-            &brioche_core::bake::BakeScope::Anonymous,
-        )
-        .await
-        .unwrap();
-
-        let wide_dir = make_wide_dir(&brioche, "").await;
-        let _wide_dir_result = brioche_core::bake::bake(
-            &brioche,
-            WithMeta::without_meta(Recipe::from(wide_dir.clone())),
-            &brioche_core::bake::BakeScope::Anonymous,
-        )
-        .await
-        .unwrap();
-
-        let merge_deep_dir = Recipe::Merge {
-            directories: futures::stream::iter(0..10)
-                .then(|n| {
-                    let brioche = brioche.clone();
-                    async move {
-                        WithMeta::without_meta(Recipe::from(
-                            make_deep_dir(&brioche, &n.to_string()).await,
-                        ))
-                    }
-                })
-                .collect()
-                .await,
-        };
-
-        let merge_wide_dir = Recipe::Merge {
-            directories: futures::stream::iter(0..10)
-                .then(|n| {
-                    let brioche = brioche.clone();
-                    async move {
-                        WithMeta::without_meta(Recipe::from(
-                            make_deep_dir(&brioche, &n.to_string()).await,
-                        ))
-                    }
-                })
-                .collect()
-                .await,
-        };
-
-        (
-            brioche,
-            context,
-            Recipes {
-                deep_dir,
-                wide_dir,
-                merge_deep_dir,
-                merge_wide_dir,
-            },
-        )
-    });
-
-    c.bench_function("cached bake deep dir", |b| {
-        b.to_async(&runtime).iter(|| async {
-            let deep_dir = WithMeta::without_meta(Recipe::from(recipes.deep_dir.clone()));
-            let _ = brioche_core::bake::bake(
-                &brioche,
-                deep_dir,
-                &brioche_core::bake::BakeScope::Anonymous,
-            )
-            .await
-            .unwrap();
-        })
-    });
-
-    c.bench_function("cached bake wide dir", |b| {
-        b.to_async(&runtime).iter(|| async {
-            let wide_dir = WithMeta::without_meta(Recipe::from(recipes.wide_dir.clone()));
-            let _ = brioche_core::bake::bake(
-                &brioche,
-                wide_dir,
-                &brioche_core::bake::BakeScope::Anonymous,
-            )
-            .await
-            .unwrap();
-        })
-    });
-
-    c.bench_function("cached bake deep merge", |b| {
-        b.to_async(&runtime).iter(|| async {
-            let merge_deep_dir = WithMeta::without_meta(recipes.merge_deep_dir.clone());
-            let _ = brioche_core::bake::bake(
-                &brioche,
-                merge_deep_dir,
-                &brioche_core::bake::BakeScope::Anonymous,
-            )
-            .await
-            .unwrap();
-        })
-    });
-
-    c.bench_function("cached bake wide merge", |b| {
-        b.to_async(&runtime).iter(|| async {
-            let merge_wide_dir = WithMeta::without_meta(recipes.merge_wide_dir.clone());
-            let _ = brioche_core::bake::bake(
-                &brioche,
-                merge_wide_dir,
-                &brioche_core::bake::BakeScope::Anonymous,
-            )
-            .await
-            .unwrap();
-        })
-    });
-}
-
-criterion_group!(benches, run_bake_benchmark);
-criterion_main!(benches);