diff --git a/Cargo.lock b/Cargo.lock index d51be041..dbcb6950 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1081,6 +1081,7 @@ dependencies = [ "matches", "ouroboros", "paste", + "rayon-cond", "regex", "rust-embed", "serde", @@ -3986,6 +3987,17 @@ dependencies = [ "rayon-core", ] +[[package]] +name = "rayon-cond" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +dependencies = [ + "either", + "itertools 0.11.0", + "rayon", +] + [[package]] name = "rayon-core" version = "1.12.0" diff --git a/cpclib-asm/Cargo.toml b/cpclib-asm/Cargo.toml index a36241f4..a0f55b09 100644 --- a/cpclib-asm/Cargo.toml +++ b/cpclib-asm/Cargo.toml @@ -49,7 +49,7 @@ smartstring.workspace = true indicatif = {workspace = true, optional=true} - +rayon-cond = "0.3.0" [build-dependencies] built.workspace = true diff --git a/cpclib-asm/src/assembler/delayed_command.rs b/cpclib-asm/src/assembler/delayed_command.rs index c242d22b..bd2dc823 100644 --- a/cpclib-asm/src/assembler/delayed_command.rs +++ b/cpclib-asm/src/assembler/delayed_command.rs @@ -5,6 +5,7 @@ use codespan_reporting::diagnostic::Severity; use cpclib_common::itertools::Itertools; #[cfg(not(target_arch = "wasm32"))] use cpclib_common::rayon::prelude::*; +use rayon_cond::CondIterator; use super::report::SavedFile; use super::save_command::SaveCommand; @@ -205,6 +206,14 @@ impl DelayedCommands { self.save_commands.keys().cloned().collect_vec() } + /// can save in parallel if all commands can be saved in parallel (we are strict because we miss lots of parallelism) + pub fn can_save_in_parallel(&self) -> bool { + self.save_commands.values() + .all(|s| + s.iter().all(|s| s.can_be_saved_in_parallel()) + ) + } + pub fn add_failed_assert_command(&mut self, command: FailedAssertCommand) { self.failed_assert_commands.push(command); } @@ -226,9 +235,9 @@ impl DelayedCommands { impl DelayedCommands { /// Execute the commands that correspond to the appropriate mmr configuration pub fn execute_save(&self, env: &Env, ga_mmr: u8) -> Result, AssemblerError> { - // we cannot save commands anymore in parallel, becaus each save command can change mmr + #[cfg(not(target_arch = "wasm32"))] - let iter = self.save_commands.par_iter(); + let iter = CondIterator::new(&self.save_commands, self.can_save_in_parallel()); #[cfg(target_arch = "wasm32")] let iter = self.save_commands.iter(); diff --git a/cpclib-asm/src/assembler/mod.rs b/cpclib-asm/src/assembler/mod.rs index 76421e4b..f3a23448 100644 --- a/cpclib-asm/src/assembler/mod.rs +++ b/cpclib-asm/src/assembler/mod.rs @@ -33,6 +33,7 @@ use cpclib_common::rayon::prelude::*; use cpclib_common::smallvec::SmallVec; use cpclib_sna::*; use cpclib_tokens::ToSimpleToken; +use rayon_cond::CondIterator; use self::function::{Function, FunctionBuilder, HardCodedFunction}; use self::listing_output::*; @@ -1104,7 +1105,11 @@ impl Env { self.ga_mmr = 0xC0; #[cfg(not(target_arch = "wasm32"))] - let iter = self.banks.par_iter(); + let iter = { + let can_save_in_parallel = self.banks.iter() + .all(|b| b.1.can_save_in_parallel()); + CondIterator::new(&self.banks, can_save_in_parallel) + }; #[cfg(target_arch = "wasm32")] let iter = self.banks.iter(); let mut saved = iter diff --git a/cpclib-asm/src/assembler/page_info.rs b/cpclib-asm/src/assembler/page_info.rs index 8c3a3b50..90cae838 100644 --- a/cpclib-asm/src/assembler/page_info.rs +++ b/cpclib-asm/src/assembler/page_info.rs @@ -61,6 +61,7 @@ impl PageInformation { pub fn failed_assert_commands(&self) -> &[FailedAssertCommand] ; pub fn failed_assert_commands_mut(&mut self) -> &mut[FailedAssertCommand] ; + pub fn can_save_in_parallel(&self) -> bool; pub fn get_save_mmrs(&self) -> Vec; pub fn execute_save(&self, env: &Env, mmr: u8) -> Result, AssemblerError>; pub fn nb_files_to_save(&self) -> usize; diff --git a/cpclib-asm/src/assembler/save_command.rs b/cpclib-asm/src/assembler/save_command.rs index 077ed5c3..1f304982 100644 --- a/cpclib-asm/src/assembler/save_command.rs +++ b/cpclib-asm/src/assembler/save_command.rs @@ -44,6 +44,10 @@ impl SaveCommand { self.ga_mmr } + pub fn can_be_saved_in_parallel(&self) -> bool { + (&self.dsk_filename).is_none() + } + /// Really make the save - Prerequisit : the page is properly selected pub fn execute_on(&self, env: &Env) -> Result { assert_eq!(env.ga_mmr, self.ga_mmr); diff --git a/cpclib-basm/tests/real_tests.rs b/cpclib-basm/tests/real_tests.rs index 650a354f..f0cb2e22 100644 --- a/cpclib-basm/tests/real_tests.rs +++ b/cpclib-basm/tests/real_tests.rs @@ -1,4 +1,5 @@ use std::process::Command; +use std::sync::Mutex; use cpclib_asm::assembler::Env; use cpclib_asm::error::AssemblerError; @@ -9,6 +10,20 @@ use pretty_assertions::assert_eq; use regex::Regex; use test_generator::test_resources; +lazy_static::lazy_static!{ + static ref LOCK: Mutex<()> = Mutex::default(); +} + + +fn manual_cleanup() { + for fname in &["hello.dsk"] { + let p = std::path::Path::new(fname); + if p.exists() { + std::fs::remove_file(p).unwrap() + } + } +} + fn command_for_generated_test( fname: &str, output: &str @@ -48,6 +63,8 @@ fn test_roudoudou_generated_code() { #[test_resources("cpclib-basm/tests/asm/warning_*.asm")] fn expect_warning_but_success(real_fname: &str) { + let _lock = LOCK.lock().unwrap(); + let fname = &real_fname["cpclib-basm/tests/asm/".len()..]; let output_file = tempfile::NamedTempFile::new().expect("Unable to build temporary file"); @@ -137,6 +154,9 @@ fn expect_one_line_success(real_fname: &str) { { return; } + let _lock = LOCK.lock().unwrap(); + + manual_cleanup() ; let fname = &real_fname["cpclib-basm/tests/asm/".len()..]; @@ -219,6 +239,10 @@ fn expect_several_empty_lines_success(real_fname: &str) { if real_fname.contains("basic") { return; } + let _lock = LOCK.lock().unwrap(); + + manual_cleanup() ; + let fname = &real_fname["cpclib-basm/tests/asm/".len()..]; @@ -274,6 +298,10 @@ fn expect_several_empty_lines_success(real_fname: &str) { /// TODO write tests specifics for this purpose fn expect_listing_success(fname: &str) { let fname = &fname["cpclib-basm/tests/asm/".len()..]; + let _lock = LOCK.lock().unwrap(); + + manual_cleanup() ; + let output_file = tempfile::NamedTempFile::new().expect("Unable to build temporary file"); let output_fname = output_file.path().as_os_str().to_str().unwrap(); @@ -307,6 +335,11 @@ fn expect_listing_success(fname: &str) { //#[test_resources("basm/tests/asm/good_*.sym")] /// TODO write tests specifics for this purpose fn expect_symbols_success(fname: &str) { + let _lock = LOCK.lock().unwrap(); + + manual_cleanup() ; + + let sym_gt = &fname["cpclib-basm/tests/asm/".len()..]; let fname = sym_gt.replace(".sym", ".asm"); @@ -346,6 +379,10 @@ fn expect_symbols_success(fname: &str) { #[test_resources("cpclib-basm/tests/asm/good_*.asm")] fn expect_success(fname: &str) { + let _lock = LOCK.lock().unwrap(); + + manual_cleanup() ; + eprintln!("{}", fname); let fname = &fname["cpclib-basm/tests/asm/".len()..]; @@ -396,6 +433,11 @@ fn expect_success(fname: &str) { #[test_resources("cpclib-basm/tests/asm/bad_*.asm")] fn expect_failure(fname: &str) { + let _lock = LOCK.lock().unwrap(); + + manual_cleanup() ; + + let fname = &fname["cpclib-basm/tests/asm/".len()..]; let output_file = tempfile::NamedTempFile::new().expect("Unable to build temporary file"); diff --git a/cpclib-disc/src/edsk.rs b/cpclib-disc/src/edsk.rs index cb973ee8..0ef6340c 100644 --- a/cpclib-disc/src/edsk.rs +++ b/cpclib-disc/src/edsk.rs @@ -7,7 +7,7 @@ use std::path::Path; use std::string::ToString; use cpclib_common::bitflags::bitflags; -use cpclib_common::itertools::zip; +use cpclib_common::itertools::{zip, Itertools}; use delegate::delegate; use getset::Getters; @@ -1018,7 +1018,12 @@ impl ExtendedDsk { } let mut manager = AmsdosManager::new_from_disc(self, head); + + eprint!("{:?}", manager.catalog().all_entries().collect_vec()); manager.add_file(&file, system, read_only)?; + eprint!("{:?}", manager.catalog().all_entries().collect_vec()); + + Ok(()) }