Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

piecrust: improve bootstrap #398

Merged
merged 4 commits into from
Oct 18, 2024
Merged

piecrust: improve bootstrap #398

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
0630995
piecrust: remove useless string clone
herr-seppia Oct 18, 2024
f879f92
piecrust: move "calculating root" logs
herr-seppia Oct 18, 2024
09b50c9
piecrust: remove `merkle.root()` calculation during startup
herr-seppia Oct 18, 2024
c56bfc6
piecrust: add logs for new vm instantiation
herr-seppia Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
105 changes: 50 additions & 55 deletions piecrust/src/store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,9 @@ impl ContractStore {
let (call, calls) = mpsc::channel();
let commit_store = self.commit_store.clone();

read_all_commits(&self.engine, self.root_dir.clone(), commit_store)?;
tracing::trace!("before read_all_commit");
read_all_commits(&self.engine, &self.root_dir, commit_store)?;
tracing::trace!("after read_all_commit");

let commit_store = self.commit_store.clone();

Expand Down Expand Up @@ -252,19 +254,21 @@ fn read_all_commits<P: AsRef<Path>>(
let root_dir = root_dir.as_ref();

let root_dir = root_dir.join(MAIN_DIR);
fs::create_dir_all(root_dir.clone())?;
fs::create_dir_all(&root_dir)?;

for entry in fs::read_dir(root_dir)? {
let entry = entry?;
if entry.path().is_dir() {
let filename = entry.file_name().to_string_lossy().to_string();
let filename = entry.file_name();
if filename == MEMORY_DIR
|| filename == BYTECODE_DIR
|| filename == LEAF_DIR
{
continue;
}
tracing::trace!("before read_commit");
let commit = read_commit(engine, entry.path())?;
tracing::trace!("before read_commit");
let root = *commit.root();
commit_store.lock().unwrap().insert_commit(root, commit);
}
Expand Down Expand Up @@ -350,8 +354,10 @@ fn commit_from_dir<P: AsRef<Path>>(

// let contracts_merkle_path = dir.join(MERKLE_FILE);
let leaf_dir = main_dir.join(LEAF_DIR);
tracing::trace!("before index_merkle_from_path");
let (index, contracts_merkle) =
index_merkle_from_path(main_dir, leaf_dir, &maybe_hash)?;
tracing::trace!("after index_merkle_from_path");

let bytecode_dir = main_dir.join(BYTECODE_DIR);
let memory_dir = main_dir.join(MEMORY_DIR);
Expand Down Expand Up @@ -390,7 +396,7 @@ fn commit_from_dir<P: AsRef<Path>>(
let path = ContractSession::find_page(
*page_index,
maybe_hash,
contract_memory_dir.clone(),
&contract_memory_dir,
main_dir,
);
let found = path.map(|p| p.is_file()).unwrap_or(false);
Expand Down Expand Up @@ -426,18 +432,18 @@ fn index_merkle_from_path(
for entry in fs::read_dir(leaf_dir)? {
let entry = entry?;
if entry.path().is_dir() {
let contract_id_hex =
entry.file_name().to_string_lossy().to_string();
let contract_id = contract_id_from_hex(contract_id_hex.clone());
let contract_id_hex = entry.file_name();
let contract_id =
contract_id_from_hex(contract_id_hex.to_string_lossy());
let contract_leaf_path = leaf_dir.join(contract_id_hex);
let element_path = ContractSession::find_element(
*maybe_commit_id,
contract_leaf_path.clone(),
main_path.as_ref(),
&contract_leaf_path,
&main_path,
)
.unwrap_or(contract_leaf_path.join(ELEMENT_FILE));
if element_path.is_file() {
let element_bytes = fs::read(element_path.clone())?;
let element_bytes = fs::read(&element_path)?;
let element: ContractIndexElement =
rkyv::from_bytes(&element_bytes).map_err(|err| {
tracing::trace!(
Expand All @@ -457,7 +463,6 @@ fn index_merkle_from_path(
element.int_pos.expect("int pos should be present"),
h,
);
let _ = merkle.root();
}
index.insert_contract_index(&contract_id, element);
}
Expand Down Expand Up @@ -563,7 +568,10 @@ impl Commit {
}

pub fn root(&self) -> Ref<Hash> {
self.contracts_merkle.root()
tracing::trace!("calculating root started");
let ret = self.contracts_merkle.root();
tracing::trace!("calculating root finished");
ret
}
}

Expand Down Expand Up @@ -798,10 +806,8 @@ fn write_commit<P: AsRef<Path>>(
}
}

tracing::trace!("calculating root started");
let root = *commit.root();
let root_hex = hex::encode(root);
tracing::trace!("calculating root finished");

// Don't write the commit if it already exists on disk. This may happen if
// the same transactions on the same base commit for example.
Expand Down Expand Up @@ -869,12 +875,9 @@ fn write_commit_inner<P: AsRef<Path>, S: AsRef<str>>(
let mut dirty = false;
// Write dirty pages and keep track of the page indices.
for (dirty_page, _, page_index) in contract_data.memory.dirty_pages() {
let page_path: PathBuf = page_path_main(
&memory_main_dir,
*page_index,
commit_id.as_ref(),
)?;
fs::write(page_path.clone(), dirty_page)?;
let page_path: PathBuf =
page_path_main(&memory_main_dir, *page_index, &commit_id)?;
fs::write(page_path, dirty_page)?;
pages.insert(*page_index);
dirty = true;
}
Expand Down Expand Up @@ -910,15 +913,15 @@ fn write_commit_inner<P: AsRef<Path>, S: AsRef<str>>(
.join(hex::encode(contract_id.as_bytes()))
.join(commit_id.as_ref());
let element_file_path = element_dir_path.join(ELEMENT_FILE);
create_dir_all(element_dir_path.clone())?;
create_dir_all(element_dir_path)?;
let element_bytes =
rkyv::to_bytes::<_, 128>(element).map_err(|err| {
io::Error::new(
io::ErrorKind::InvalidData,
format!("Failed serializing element file: {err}"),
)
})?;
fs::write(element_file_path.clone(), element_bytes)?;
fs::write(&element_file_path, element_bytes)?;
}
}
tracing::trace!("persisting index finished");
Expand All @@ -932,7 +935,7 @@ fn write_commit_inner<P: AsRef<Path>, S: AsRef<str>>(
format!("Failed serializing base info file: {err}"),
)
})?;
fs::write(base_main_path.clone(), base_info_bytes)?;
fs::write(base_main_path, base_info_bytes)?;

Ok(())
}
Expand All @@ -944,24 +947,22 @@ fn delete_commit_dir<P: AsRef<Path>>(
) -> io::Result<()> {
let root = hex::encode(root);
let root_main_dir = root_dir.as_ref().join(MAIN_DIR);
let commit_dir = root_main_dir.join(root.clone());
let commit_dir = root_main_dir.join(&root);
if commit_dir.exists() {
let base_info_path = commit_dir.join(BASE_FILE);
let base_info = base_from_path(base_info_path.clone())?;
let base_info = base_from_path(base_info_path)?;
for contract_hint in base_info.contract_hints {
let contract_hex = hex::encode(contract_hint);
let commit_mem_path = root_main_dir
.join(MEMORY_DIR)
.join(contract_hex.clone())
.join(root.clone());
fs::remove_dir_all(commit_mem_path.clone())?;
let commit_leaf_path = root_main_dir
.join(LEAF_DIR)
.join(contract_hex.clone())
.join(root.clone());
fs::remove_dir_all(commit_leaf_path.clone())?;
.join(&contract_hex)
.join(&root);
fs::remove_dir_all(&commit_mem_path)?;
let commit_leaf_path =
root_main_dir.join(LEAF_DIR).join(&contract_hex).join(&root);
fs::remove_dir_all(&commit_leaf_path)?;
}
fs::remove_dir_all(commit_dir.clone())?;
fs::remove_dir_all(&commit_dir)?;
}
Ok(())
}
Expand All @@ -974,44 +975,38 @@ fn finalize_commit<P: AsRef<Path>>(
) -> io::Result<()> {
let main_dir = root_dir.as_ref().join(MAIN_DIR);
let root = hex::encode(root);
let commit_path = main_dir.join(root.clone());
let commit_path = main_dir.join(&root);
let base_info_path = commit_path.join(BASE_FILE);
let base_info = base_from_path(base_info_path.clone())?;
let base_info = base_from_path(&base_info_path)?;
for contract_hint in base_info.contract_hints {
let contract_hex = hex::encode(contract_hint);
// MEMORY
let src_path = main_dir
.join(MEMORY_DIR)
.join(contract_hex.clone())
.join(root.clone());
let dst_path =
main_dir.clone().join(MEMORY_DIR).join(contract_hex.clone());
for entry in fs::read_dir(src_path.clone())? {
let entry = entry?;
let filename = entry.file_name().to_string_lossy().to_string();
let src_file_path = src_path.join(filename.clone());
let dst_file_path = dst_path.join(filename);
let src_path =
main_dir.join(MEMORY_DIR).join(&contract_hex).join(&root);
let dst_path = main_dir.join(MEMORY_DIR).join(&contract_hex);
for entry in fs::read_dir(&src_path)? {
let filename = entry?.file_name();
let src_file_path = src_path.join(&filename);
let dst_file_path = dst_path.join(&filename);
if src_file_path.is_file() {
fs::rename(src_file_path, dst_file_path)?;
}
}
fs::remove_dir(src_path.clone())?;
fs::remove_dir(&src_path)?;
// LEAF
let src_leaf_path = main_dir
.join(LEAF_DIR)
.join(contract_hex.clone())
.join(root.clone());
let dst_leaf_path = main_dir.clone().join(LEAF_DIR).join(contract_hex);
let src_leaf_path =
main_dir.join(LEAF_DIR).join(&contract_hex).join(&root);
let dst_leaf_path = main_dir.join(LEAF_DIR).join(contract_hex);
let src_leaf_file_path = src_leaf_path.join(ELEMENT_FILE);
let dst_leaf_file_path = dst_leaf_path.join(ELEMENT_FILE);
if src_leaf_file_path.is_file() {
fs::rename(src_leaf_file_path, dst_leaf_file_path)?;
}
fs::remove_dir(src_leaf_path.clone())?;
fs::remove_dir(src_leaf_path)?;
}

fs::remove_file(base_info_path)?;
fs::remove_dir(commit_path.clone())?;
fs::remove_dir(commit_path)?;

Ok(())
}
24 changes: 8 additions & 16 deletions piecrust/src/store/session.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ impl ContractSession {
let hash_hex = hex::encode(hash.as_bytes());
let path = memory_path
.as_ref()
.join(hash_hex.clone())
.join(&hash_hex)
.join(format!("{page_index}"));
if path.is_file() {
Some(path)
Expand All @@ -191,8 +191,8 @@ impl ContractSession {
Self::find_page(
page_index,
index.maybe_base,
memory_path.as_ref(),
main_path.as_ref(),
memory_path,
main_path,
)
}
}
Expand All @@ -212,21 +212,15 @@ impl ContractSession {
None => None,
Some(hash) => {
let hash_hex = hex::encode(hash.as_bytes());
let path = leaf_path
.as_ref()
.join(hash_hex.clone())
.join(ELEMENT_FILE);
let path =
leaf_path.as_ref().join(&hash_hex).join(ELEMENT_FILE);
if path.is_file() {
Some(path)
} else {
let base_info_path =
main_path.as_ref().join(hash_hex).join(BASE_FILE);
let index = base_from_path(base_info_path).ok()?;
Self::find_element(
index.maybe_base,
leaf_path.as_ref(),
main_path.as_ref(),
)
Self::find_element(index.maybe_base, leaf_path, main_path)
}
}
}
Expand Down Expand Up @@ -278,8 +272,6 @@ impl ContractSession {
Some(elem) => {
let page_indices =
elem.page_indices.clone();
let memory_path = memory_path.clone();

Memory::from_files(
module.is_64(),
move |page_index: usize| {
Expand All @@ -290,8 +282,8 @@ impl ContractSession {
Self::find_page(
page_index,
commit_id,
memory_path.clone(),
base_dir.clone(),
&memory_path,
&base_dir,
)
.unwrap_or(
memory_path.join(
Expand Down
4 changes: 4 additions & 0 deletions piecrust/src/vm.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,17 +138,21 @@ impl VM {
/// # Errors
/// If the directory contains unparseable or inconsistent data.
pub fn new<P: AsRef<Path>>(root_dir: P) -> Result<Self, Error> {
tracing::trace!("vm::new");
let config = config();

let engine = Engine::new(&config).expect(
"Configuration should be valid since its set at compile time",
);

tracing::trace!("before ContractStore::new");
let mut store = ContractStore::new(engine.clone(), root_dir)
.map_err(|err| PersistenceError(Arc::new(err)))?;
tracing::trace!("before ContractStore::finish_new");
store
.finish_new()
.map_err(|err| PersistenceError(Arc::new(err)))?;
tracing::trace!("after ContractStore::finish_new");

Ok(Self {
engine,
Expand Down