diff --git a/crates/brioche-core/src/bake/download.rs b/crates/brioche-core/src/bake/download.rs index 40424e4..1506c7c 100644 --- a/crates/brioche-core/src/bake/download.rs +++ b/crates/brioche-core/src/bake/download.rs @@ -1,7 +1,3 @@ -use anyhow::Context as _; -use futures::TryStreamExt as _; -use tokio_util::compat::FuturesAsyncReadCompatExt as _; - use crate::{ recipe::{Directory, DownloadRecipe, File}, Brioche, @@ -9,76 +5,7 @@ use crate::{ #[tracing::instrument(skip(brioche, download), fields(url = %download.url))] pub async fn bake_download(brioche: &Brioche, download: DownloadRecipe) -> anyhow::Result { - // Acquire a permit to save the blob - let save_blob_permit = crate::blob::get_save_blob_permit().await?; - - // Acquire a permit to download - tracing::debug!("acquiring download semaphore permit"); - let _permit = brioche.download_semaphore.acquire().await?; - tracing::debug!("acquired download semaphore permit"); - - tracing::debug!(url = %download.url, "starting download"); - - let job_id = brioche.reporter.add_job(crate::reporter::NewJob::Download { - url: download.url.clone(), - }); - - let response = brioche - .download_client - .get(download.url.clone()) - .send() - .await?; - let response = response.error_for_status()?; - - let content_length = response.content_length().or_else(|| { - let content_length = response.headers().get(reqwest::header::CONTENT_LENGTH)?; - let content_length = content_length.to_str().ok()?.parse().ok()?; - if content_length == 0 { - None - } else { - Some(content_length) - } - }); - - let mut download_stream = response - .bytes_stream() - .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e)) - .into_async_read() - .compat(); - let download_stream = std::pin::pin!(download_stream); - - let save_blob_options = crate::blob::SaveBlobOptions::new() - .expected_hash(Some(download.hash)) - .on_progress(|bytes_read| { - if let Some(content_length) = content_length { - let progress_percent = (bytes_read as f64 / content_length as f64) * 100.0; - let progress_percent = progress_percent.round().min(99.0) as u8; - brioche.reporter.update_job( - job_id, - crate::reporter::UpdateJob::Download { - progress_percent: Some(progress_percent), - }, - ); - } - - Ok(()) - }); - - let blob_hash = crate::blob::save_blob_from_reader( - brioche, - save_blob_permit, - download_stream, - save_blob_options, - ) - .await - .context("failed to save blob")?; - - brioche.reporter.update_job( - job_id, - crate::reporter::UpdateJob::Download { - progress_percent: Some(100), - }, - ); + let blob_hash = crate::download::download(brioche, &download.url, Some(download.hash)).await?; Ok(File { content_blob: blob_hash, diff --git a/crates/brioche-core/src/download.rs b/crates/brioche-core/src/download.rs new file mode 100644 index 0000000..1566f5e --- /dev/null +++ b/crates/brioche-core/src/download.rs @@ -0,0 +1,81 @@ +use anyhow::Context as _; +use futures::TryStreamExt as _; +use tokio_util::compat::FuturesAsyncReadCompatExt as _; + +use crate::Brioche; + +#[tracing::instrument(skip(brioche, expected_hash))] +pub async fn download( + brioche: &Brioche, + url: &url::Url, + expected_hash: Option, +) -> anyhow::Result { + // Acquire a permit to save the blob + let save_blob_permit = crate::blob::get_save_blob_permit().await?; + + // Acquire a permit to download + tracing::debug!("acquiring download semaphore permit"); + let _permit = brioche.download_semaphore.acquire().await?; + tracing::debug!("acquired download semaphore permit"); + + tracing::debug!(%url, "starting download"); + + let job_id = brioche + .reporter + .add_job(crate::reporter::NewJob::Download { url: url.clone() }); + + let response = brioche.download_client.get(url.clone()).send().await?; + let response = response.error_for_status()?; + + let content_length = response.content_length().or_else(|| { + let content_length = response.headers().get(reqwest::header::CONTENT_LENGTH)?; + let content_length = content_length.to_str().ok()?.parse().ok()?; + if content_length == 0 { + None + } else { + Some(content_length) + } + }); + + let mut download_stream = response + .bytes_stream() + .map_err(|e| futures::io::Error::new(futures::io::ErrorKind::Other, e)) + .into_async_read() + .compat(); + let download_stream = std::pin::pin!(download_stream); + + let save_blob_options = crate::blob::SaveBlobOptions::new() + .expected_hash(expected_hash) + .on_progress(|bytes_read| { + if let Some(content_length) = content_length { + let progress_percent = (bytes_read as f64 / content_length as f64) * 100.0; + let progress_percent = progress_percent.round().min(99.0) as u8; + brioche.reporter.update_job( + job_id, + crate::reporter::UpdateJob::Download { + progress_percent: Some(progress_percent), + }, + ); + } + + Ok(()) + }); + + let blob_hash = crate::blob::save_blob_from_reader( + brioche, + save_blob_permit, + download_stream, + save_blob_options, + ) + .await + .context("failed to save blob")?; + + brioche.reporter.update_job( + job_id, + crate::reporter::UpdateJob::Download { + progress_percent: Some(100), + }, + ); + + Ok(blob_hash) +} diff --git a/crates/brioche-core/src/lib.rs b/crates/brioche-core/src/lib.rs index f459449..05cb1b5 100644 --- a/crates/brioche-core/src/lib.rs +++ b/crates/brioche-core/src/lib.rs @@ -12,6 +12,7 @@ use tokio::{ pub mod bake; pub mod blob; +pub mod download; pub mod encoding; pub mod fs_utils; pub mod input; @@ -312,6 +313,10 @@ pub enum Hasher { } impl Hasher { + pub fn new_sha256() -> Self { + Self::Sha256(sha2::Sha256::new()) + } + pub fn for_hash(hash: &Hash) -> Self { match hash { Hash::Sha256 { .. } => Self::Sha256(sha2::Sha256::new()), diff --git a/crates/brioche-core/src/project.rs b/crates/brioche-core/src/project.rs index 5b54602..4b86492 100644 --- a/crates/brioche-core/src/project.rs +++ b/crates/brioche-core/src/project.rs @@ -4,11 +4,11 @@ use std::{ sync::Arc, }; -use analyze::StaticQuery; +use analyze::{StaticOutput, StaticOutputKind, StaticQuery}; use anyhow::Context as _; use futures::{StreamExt as _, TryStreamExt as _}; use relative_path::{PathExt as _, RelativePath, RelativePathBuf}; -use tokio::io::AsyncWriteExt as _; +use tokio::io::{AsyncReadExt as _, AsyncWriteExt as _}; use crate::recipe::{Artifact, RecipeHash}; @@ -482,10 +482,11 @@ impl ProjectsInner { let Some(statics) = project.statics.get(&module_path) else { return Ok(None); }; - let Some(Some(static_)) = statics.get(static_) else { + let Some(Some(output)) = statics.get(static_) else { return Ok(None); }; - Ok(Some(*static_)) + let recipe_hash = static_.output_recipe_hash(output)?; + Ok(Some(recipe_hash)) } } @@ -682,7 +683,16 @@ async fn load_project_inner( for static_ in &module.statics { // Only resolve the static if we need a fully valid project if fully_valid { - let recipe_hash = resolve_static(brioche, &path, module, static_).await?; + let recipe_hash = resolve_static( + brioche, + &path, + module, + static_, + lockfile_required, + lockfile.as_ref(), + &mut new_lockfile, + ) + .await?; module_statics.insert(static_.clone(), Some(recipe_hash)); } else { module_statics.insert(static_.clone(), None); @@ -977,25 +987,32 @@ async fn fetch_project_from_registry( }) .await?; - let statics_recipes = project - .statics - .values() - .flat_map(|module_statics| module_statics.values().filter_map(|recipe| *recipe)) - .collect::>(); + let mut statics_recipes = HashSet::new(); + for (static_, output) in project.statics.values().flatten() { + let Some(output) = output else { + continue; + }; + + let recipe_hash = static_.output_recipe_hash(output)?; + statics_recipes.insert(recipe_hash); + } + crate::registry::fetch_recipes_deep(brioche, statics_recipes).await?; for (module_path, statics) in &project.statics { - for (static_, recipe_hash) in statics { - let Some(recipe_hash) = recipe_hash else { + for (static_, output) in statics { + let Some(output) = output else { continue; }; + let recipe_hash = static_.output_recipe_hash(output)?; + let module_path = module_path.to_logical_path(&temp_project_path); let module_dir = module_path.parent().context("no parent dir for module")?; match static_ { StaticQuery::Include(include) => { - let recipe = crate::recipe::get_recipe(brioche, *recipe_hash).await?; + let recipe = crate::recipe::get_recipe(brioche, recipe_hash).await?; let artifact: crate::recipe::Artifact = recipe.try_into().map_err(|_| { anyhow::anyhow!("included static recipe is not an artifact") })?; @@ -1014,7 +1031,7 @@ async fn fetch_project_from_registry( .await?; } StaticQuery::Glob { .. } => { - let recipe = crate::recipe::get_recipe(brioche, *recipe_hash).await?; + let recipe = crate::recipe::get_recipe(brioche, recipe_hash).await?; let artifact: crate::recipe::Artifact = recipe.try_into().map_err(|_| { anyhow::anyhow!("included static recipe is not an artifact") })?; @@ -1031,6 +1048,9 @@ async fn fetch_project_from_registry( ) .await?; } + StaticQuery::Download { .. } => { + // No need to do anything while fetching the project + } } } } @@ -1066,12 +1086,32 @@ async fn fetch_project_from_registry( .context("failed to copy blob")?; } + let dependencies = project + .dependencies + .iter() + .map(|(name, hash)| (name.clone(), *hash)) + .collect(); + let downloads = project + .statics + .values() + .flatten() + .filter_map(|(static_, output)| { + let url = match static_ { + StaticQuery::Download { url, .. } => url, + _ => return None, + }; + + let hash = match output { + Some(StaticOutput::Kind(StaticOutputKind::Download { hash })) => hash, + _ => return None, + }; + + Some((url.clone(), hash.clone())) + }) + .collect(); let lockfile = Lockfile { - dependencies: project - .dependencies - .iter() - .map(|(name, hash)| (name.clone(), *hash)) - .collect(), + dependencies, + downloads, }; let lockfile_path = temp_project_path.join("brioche.lock"); let lockfile_contents = @@ -1161,7 +1201,10 @@ async fn resolve_static( project_root: &Path, module: &analyze::ModuleAnalysis, static_: &analyze::StaticQuery, -) -> anyhow::Result { + lockfile_required: bool, + lockfile: Option<&Lockfile>, + new_lockfile: &mut Lockfile, +) -> anyhow::Result { match static_ { analyze::StaticQuery::Include(include) => { let module_path = module.project_subpath.to_path(project_root); @@ -1220,7 +1263,7 @@ async fn resolve_static( let recipe = crate::recipe::Recipe::from(artifact.value); crate::recipe::save_recipes(brioche, [&recipe]).await?; - Ok(artifact_hash) + Ok(StaticOutput::RecipeHash(artifact_hash)) } analyze::StaticQuery::Glob { patterns } => { let module_path = module.project_subpath.to_path(project_root); @@ -1302,7 +1345,97 @@ async fn resolve_static( crate::recipe::save_recipes(brioche, [&recipe]).await?; - Ok(recipe_hash) + Ok(StaticOutput::RecipeHash(recipe_hash)) + } + StaticQuery::Download { url } => { + let current_download_hash = lockfile.and_then(|lockfile| lockfile.downloads.get(url)); + + let download_hash: crate::Hash; + let blob_hash: Option; + + if let Some(hash) = current_download_hash { + // If we have the hash from the lockfile, use it to build + // the recipe. But, we don't have the blob hash yet + download_hash = hash.clone(); + blob_hash = None; + } else if lockfile_required { + // Error out if the download isn't in the lockfile but where + // updating the lockfile is disabled + anyhow::bail!("hash for download '{url}' not found in lockfile"); + } else { + // Download the URL as a blob + let new_blob_hash = crate::download::download(brioche, url, None).await?; + let blob_path = crate::blob::local_blob_path(brioche, new_blob_hash); + let mut blob = tokio::fs::File::open(&blob_path).await?; + + // Compute a hash to store in the lockfile + let mut hasher = crate::Hasher::new_sha256(); + let mut buffer = vec![0u8; 1024 * 1024]; + loop { + let length = blob + .read(&mut buffer) + .await + .context("failed to read blob")?; + if length == 0 { + break; + } + + hasher.update(&buffer[..length]); + } + + // Record both the hash for the recipe plus the output + // blob hash + download_hash = hasher.finish()?; + blob_hash = Some(new_blob_hash); + }; + + // Create the download recipe, which is equivalent to the URL + // we downloaded or the one recorded in the lockfile + let download_recipe = crate::recipe::Recipe::Download(crate::recipe::DownloadRecipe { + hash: download_hash.clone(), + url: url.clone(), + }); + let download_recipe_hash = download_recipe.hash(); + + if let Some(blob_hash) = blob_hash { + // If we downloaded the blob, save the recipe and the output + // artifact. This effectively caches the download + + let download_artifact = crate::recipe::Artifact::File(crate::recipe::File { + content_blob: blob_hash, + executable: false, + resources: Default::default(), + }); + + let download_recipe_json = serde_json::to_string(&download_recipe) + .context("failed to serialize download recipe")?; + let download_artifact_hash = download_artifact.hash(); + let download_artifact_json = serde_json::to_string(&download_artifact) + .context("failed to serialize download output artifact")?; + crate::bake::save_bake_result( + brioche, + download_recipe_hash, + &download_recipe_json, + download_artifact_hash, + &download_artifact_json, + ) + .await?; + } else { + // If we didn't download the blob, just save the recipe. This + // either means we've already cached the download before, + // or we haven't and we'll need to download it or fetch + // it from the registry + crate::recipe::save_recipes(brioche, &[download_recipe]).await?; + } + + // Update the new lockfile with the download hash + new_lockfile + .downloads + .insert(url.clone(), download_hash.clone()); + + Ok(StaticOutput::Kind(StaticOutputKind::Download { + hash: download_hash, + })) } } } @@ -1316,7 +1449,7 @@ pub struct Project { pub modules: HashMap, #[serde_as(as = "HashMap<_, Vec<(_, _)>>")] #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub statics: HashMap>>, + pub statics: HashMap>>, } impl Project { @@ -1533,4 +1666,7 @@ impl std::fmt::Display for WorkspaceMember { #[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct Lockfile { pub dependencies: BTreeMap, + + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub downloads: BTreeMap, } diff --git a/crates/brioche-core/src/project/analyze.rs b/crates/brioche-core/src/project/analyze.rs index 95ea175..95b390f 100644 --- a/crates/brioche-core/src/project/analyze.rs +++ b/crates/brioche-core/src/project/analyze.rs @@ -42,6 +42,44 @@ pub enum ImportAnalysis { pub enum StaticQuery { Include(StaticInclude), Glob { patterns: Vec }, + Download { url: url::Url }, +} + +impl StaticQuery { + pub fn output_recipe_hash( + &self, + output: &StaticOutput, + ) -> anyhow::Result { + let recipe_hash = match output { + StaticOutput::RecipeHash(hash) => *hash, + StaticOutput::Kind(StaticOutputKind::Download { hash }) => { + let download_url = match self { + StaticQuery::Download { url } => url, + _ => anyhow::bail!("expected download query"), + }; + let recipe = crate::recipe::Recipe::Download(crate::recipe::DownloadRecipe { + url: download_url.clone(), + hash: hash.clone(), + }); + recipe.hash() + } + }; + + Ok(recipe_hash) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(untagged)] +pub enum StaticOutput { + RecipeHash(crate::recipe::RecipeHash), + Kind(StaticOutputKind), +} + +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(tag = "kind")] +pub enum StaticOutputKind { + Download { hash: crate::Hash }, } #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, serde::Serialize, serde::Deserialize)] @@ -379,7 +417,7 @@ where return Ok(None); } - // Filter down to calls to the `.get()` method + // Filter down to calls to one of the static methods let Ok(callee_member) = callee.member() else { return Ok(None); }; @@ -463,6 +501,37 @@ where Ok(Some(StaticQuery::Glob { patterns: args })) } + "download" => { + // Get the arguments + let args = call_expr.arguments()?.args(); + let args = args + .iter() + .map(arg_to_string_literal) + .map(|arg| { + arg.with_context(|| { + format!("{location}: invalid arg to Brioche.download") + }) + }) + .collect::>>()?; + + // Ensure there's exactly one argument + let url = match &*args { + [url] => url.text(), + _ => { + anyhow::bail!( + "{location}: Brioche.download() must take exactly one argument", + ); + } + }; + + // Parse the URL + let url = url.parse().with_context(|| { + format!("{location}: invalid URL for Brioche.download") + })?; + + Ok(Some(StaticQuery::Download { url })) + + } _ => Ok(None), } }) diff --git a/crates/brioche-core/src/references.rs b/crates/brioche-core/src/references.rs index 19d3e7b..ef79d37 100644 --- a/crates/brioche-core/src/references.rs +++ b/crates/brioche-core/src/references.rs @@ -96,11 +96,12 @@ pub async fn project_references( } for (module_path, module_statics) in statics { - for static_recipe in module_statics.values() { - let static_recipe = static_recipe.with_context(|| { - format!("static recipe not loaded for module {module_path}") - })?; - new_recipes.insert(static_recipe); + for (static_, output) in module_statics { + let output = output + .as_ref() + .with_context(|| format!("static not loaded for module {module_path}"))?; + let static_recipe_hash = static_.output_recipe_hash(output)?; + new_recipes.insert(static_recipe_hash); } } } diff --git a/crates/brioche-core/src/script.rs b/crates/brioche-core/src/script.rs index c90dfa6..47313e0 100644 --- a/crates/brioche-core/src/script.rs +++ b/crates/brioche-core/src/script.rs @@ -283,6 +283,10 @@ pub async fn op_brioche_get_static( let patterns = patterns.iter().map(|pattern| lazy_format::lazy_format!("{pattern:?}")).join_with(", "); format!("failed to resolve Brioche.glob({patterns}) from {specifier}, were the patterns passed in as string literals?") } + StaticQuery::Download { url } => { + format!("failed to resolve Brioche.download({url:?}) from {specifier}, was the URL passed in as a string literal?") + } + })?; let recipe = crate::recipe::get_recipe(&brioche, recipe_hash).await?; Ok(recipe) diff --git a/crates/brioche-core/tests/project_load.rs b/crates/brioche-core/tests/project_load.rs index bb1c55c..9207c17 100644 --- a/crates/brioche-core/tests/project_load.rs +++ b/crates/brioche-core/tests/project_load.rs @@ -672,6 +672,81 @@ async fn test_project_load_with_remote_registry_dep_with_brioche_glob() -> anyho Ok(()) } +#[tokio::test] +async fn test_project_load_with_remote_registry_dep_with_brioche_download() -> anyhow::Result<()> { + let (brioche, mut context) = brioche_test::brioche_test().await; + + let mut server = mockito::Server::new(); + let server_url = server.url(); + + let hello = "hello"; + let hello_endpoint = server + .mock("GET", "/file.txt") + .with_body(hello) + .expect(1) + .create(); + + let download_url = format!("{server_url}/file.txt"); + + let foo_hash = context + .remote_registry_project(|path| async move { + tokio::fs::write( + path.join("project.bri"), + r#" + export const project = { + name: "foo", + }; + + export const hello = Brioche.download(""); + "# + .replace("", &download_url), + ) + .await + .unwrap(); + }) + .await; + let mock_foo_latest = context + .mock_registry_publish_tag("foo", "latest", foo_hash) + .create_async() + .await; + + let project_dir = context.mkdir("myproject").await; + context + .write_file( + "myproject/project.bri", + r#" + export const project = { + dependencies: { + foo: "*", + }, + }; + "#, + ) + .await; + + let (projects, project_hash) = brioche_test::load_project(&brioche, &project_dir).await?; + let project = projects.project(project_hash).unwrap(); + assert!(projects + .local_paths(project_hash) + .unwrap() + .contains(&project_dir)); + + let foo_dep_hash = project.dependency_hash("foo").unwrap(); + let foo_dep = projects.project(foo_dep_hash).unwrap(); + let foo_path = brioche.home.join("projects").join(foo_dep_hash.to_string()); + assert!(projects + .local_paths(foo_dep_hash) + .unwrap() + .contains(&foo_path)); + assert_eq!(foo_dep.dependencies().count(), 0); + + mock_foo_latest.assert_async().await; + + hello_endpoint.assert_async().await; + + Ok(()) +} + #[tokio::test] async fn test_project_load_with_remote_workspace_registry_dep() -> anyhow::Result<()> { let (brioche, mut context) = brioche_test::brioche_test().await; diff --git a/crates/brioche-core/tests/script_eval.rs b/crates/brioche-core/tests/script_eval.rs index dc544e8..386fca8 100644 --- a/crates/brioche-core/tests/script_eval.rs +++ b/crates/brioche-core/tests/script_eval.rs @@ -536,3 +536,80 @@ async fn test_eval_brioche_glob_submodule() -> anyhow::Result<()> { Ok(()) } + +#[tokio::test] +async fn test_eval_brioche_download() -> anyhow::Result<()> { + let (brioche, context) = brioche_test::brioche_test().await; + + let mut server = mockito::Server::new(); + let server_url = server.url(); + + let hello = "hello"; + let hello_blob = brioche_test::blob(&brioche, hello).await; + let hello_hash = brioche_test::sha256(hello); + let hello_endpoint = server + .mock("GET", "/file.txt") + .with_body(hello) + .expect(1) + .create(); + + let download_url = format!("{server_url}/file.txt"); + + let project_dir = context.mkdir("myproject").await; + context + .write_file( + "myproject/project.bri", + r#" + globalThis.Brioche = { + download: (url) => { + return { + briocheSerialize: async () => { + return Deno.core.ops.op_brioche_get_static( + import.meta.url, + { + type: "download", + url, + }, + ); + }, + }; + } + } + + export default () => { + return Brioche.download(""); + }; + "# + .replace("", &download_url), + ) + .await; + + let (projects, project_hash) = brioche_test::load_project(&brioche, &project_dir).await?; + + let resolved = evaluate(&brioche, &projects, project_hash, "default") + .await? + .value; + + assert_eq!( + resolved, + brioche_core::recipe::Recipe::Download(brioche_core::recipe::DownloadRecipe { + url: download_url.parse().unwrap(), + hash: hello_hash, + }) + ); + + // Bake the download, which ensures that the download was cached + let baked = brioche_test::bake_without_meta(&brioche, resolved).await?; + assert_eq!( + baked, + brioche_core::recipe::Artifact::File(brioche_core::recipe::File { + content_blob: hello_blob, + executable: false, + resources: Default::default(), + }) + ); + + hello_endpoint.assert_async().await; + + Ok(()) +}