From bf798b4538ec8e5b732d0a8f0fa4dfa1e29d078c Mon Sep 17 00:00:00 2001 From: Konrad Stepniak Date: Wed, 9 Oct 2024 12:13:56 +0300 Subject: [PATCH] fix(polka-storage-proofs): add sector padding (#434) --- Cargo.lock | 1 + .../src/commands/utils.rs | 3 +- lib/polka-storage-proofs/Cargo.toml | 1 + lib/polka-storage-proofs/src/porep/sealer.rs | 180 ++++++++++++++++-- 4 files changed, 168 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bca87aaa..f46d4db2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9383,6 +9383,7 @@ dependencies = [ "primitives-proofs", "rand", "rand_xorshift", + "rstest", "scale-info", "storage-proofs-core", "storage-proofs-porep", diff --git a/cli/polka-storage-provider/src/commands/utils.rs b/cli/polka-storage-provider/src/commands/utils.rs index 15f99fc9..52a2a404 100644 --- a/cli/polka-storage-provider/src/commands/utils.rs +++ b/cli/polka-storage-provider/src/commands/utils.rs @@ -176,7 +176,7 @@ impl UtilsCommand { println!("Creating sector..."); let sealer = Sealer::new(seal_proof.0); - sealer + let piece_infos = sealer .create_sector( vec![(piece_file, piece_info.clone())], unsealed_sector.as_file_mut(), @@ -189,7 +189,6 @@ impl UtilsCommand { let prover_id = [0u8; 32]; let ticket = [12u8; 32]; let seed = [13u8; 32]; - let piece_infos = vec![piece_info]; println!("Precommitting..."); let cache_directory = diff --git a/lib/polka-storage-proofs/Cargo.toml b/lib/polka-storage-proofs/Cargo.toml index 447b94dd..431e07e2 100644 --- a/lib/polka-storage-proofs/Cargo.toml +++ b/lib/polka-storage-proofs/Cargo.toml @@ -32,6 +32,7 @@ scale-info = { workspace = true, features = ["derive"], optional = true } [dev-dependencies] rand = { workspace = true, features = ["alloc"] } +rstest = { workspace = true } [lints] workspace = true diff --git a/lib/polka-storage-proofs/src/porep/sealer.rs b/lib/polka-storage-proofs/src/porep/sealer.rs index ebd3800d..7fcfc8bc 100644 --- a/lib/polka-storage-proofs/src/porep/sealer.rs +++ b/lib/polka-storage-proofs/src/porep/sealer.rs @@ -5,8 +5,8 @@ use blstrs::Bls12; use filecoin_hashers::Domain; use filecoin_proofs::{ add_piece, as_safe_commitment, parameters::setup_params, DefaultBinaryTree, DefaultPieceDomain, - DefaultPieceHasher, PoRepConfig, SealCommitPhase1Output, SealPreCommitOutput, - SealPreCommitPhase1Output, UnpaddedBytesAmount, + DefaultPieceHasher, PaddedBytesAmount, PoRepConfig, SealCommitPhase1Output, + SealPreCommitOutput, SealPreCommitPhase1Output, UnpaddedBytesAmount, }; use primitives_proofs::{RegisteredSealProof, SectorNumber}; use storage_proofs_core::{compound_proof, compound_proof::CompoundProof}; @@ -26,7 +26,6 @@ impl Sealer { } } - // TODO(@th7nder,#420,02/10/2024): this ain't working properly. it only works when pieces.len() == 1 and piece_size == 2032 == sector_size /// Takes all of the pieces and puts them in a sector with padding. /// # Arguments /// @@ -41,28 +40,24 @@ impl Sealer { &self, pieces: Vec<(R, PieceInfo)>, mut unsealed_sector: W, - ) -> Result<(), PoRepError> { + ) -> Result, PoRepError> { if pieces.len() == 0 { return Err(PoRepError::EmptySector); } + let mut result_pieces: Vec = Vec::new(); let mut piece_lengths: Vec = Vec::new(); - if pieces.len() > 1 - || UnpaddedBytesAmount(pieces[0].1.size as u64) != self.porep_config.sector_size.into() - { - todo!( - "known bug: issue#420 piece_size {} != 2032", - pieces[0].1.size - ); - } - + let mut unpadded_occupied_space: UnpaddedBytesAmount = UnpaddedBytesAmount(0); for (idx, (reader, piece)) in pieces.into_iter().enumerate() { let piece: filecoin_proofs::PieceInfo = piece.into(); - let (calculated_piece_info, _) = + let (calculated_piece_info, written_bytes) = add_piece(reader, &mut unsealed_sector, piece.size, &piece_lengths).unwrap(); piece_lengths.push(piece.size); + // We need to add `written_bytes` not `piece.size`, as `add_piece` adds padding. + unpadded_occupied_space = unpadded_occupied_space + written_bytes; + if piece.commitment != calculated_piece_info.commitment { return Err(PoRepError::InvalidPieceCid( idx, @@ -70,9 +65,15 @@ impl Sealer { calculated_piece_info.commitment, )); } + + result_pieces.push(piece.into()); } - Ok(()) + let sector_size: UnpaddedBytesAmount = self.porep_config.sector_size.into(); + let padding_pieces = filler_pieces(sector_size - unpadded_occupied_space); + result_pieces.extend(padding_pieces.into_iter().map(Into::into)); + + Ok(result_pieces) } /// Takes the data contained in `unsealed_sector`, seals it and puts it into `sealed_sector`. @@ -234,6 +235,57 @@ impl Sealer { } } +/// Takes remaining space to be filled with zero-byte pieces and generates filler pieces. +/// Sector's CommD is calculated in two ways: from pieces and from the Sector file. +/// During computation from the Sector file, when the sector is not full, zero-bytes are used +/// as padding to make the sector match the necessary node size for the Binary Merkle Tree calculation. +/// To match this logic when calculating CommD out of the Piece Infos we need to generate dummy pieces. +/// Returns dummy pieces with appropriate sizes and commitments. +/// +/// Pre-condition: +/// * `remaining_space == (unpadded_sector_size - real_pieces.map(|p| p.unpadded_piece_length).sum())` +/// +/// References: +/// * +/// * +/// * +fn filler_pieces(remaining_space: UnpaddedBytesAmount) -> Vec { + // We convert it to `PaddedBytesAmount` as it makes calculations (on the powers of 2) easier (see below). + let mut remaining_space: PaddedBytesAmount = remaining_space.into(); + + // All of the piece sizes need to be Padded (i.e. Fr32 padded, because we use BLS12-381 cryptography, where each field element is 254 bits) + // All of the piece sizes need to be a power of 2 as well, because we use Binary Merkle Tree for CommD/CommP computation. + // Considering the binary representation of remaining_space, each set bit represents a valid piece size. + let pieces = remaining_space.0.count_ones() as usize; + let mut piece_infos: Vec = vec![]; + for _ in 0..pieces { + // We create pieces from smaller to bigger, because Merkle Proof is computed from leaves to root. + // e.g. + // sector_size = 2048.to_unpadded() = 2032 + // piece_size = 256.to_unpadded() = 254 (Piece 0) + // remaining = 2032 - 254 = 1778.to_padded() = 1792 + // 1792 == 0b11100000000 + // Piece 1 | trailing_zeros = 8 | piece_size = 1 << 8 = 256 + // remaining ^= (1 << 8) = 0b11000000000 + // Piece 2 | trailing_zeros = 9 | piece_size = 1 << 9 = 512 + // remaining ^= (1 << 9) = 0b10000000000 + // Piece 3 | trailing|zeros = 10 | piece_size = 1 << 10 = 1024 + // And then, out of the pieces, CommD: + // hash(Piece 0[256], Piece 1 [256]) + // hash(Piece 0|1, Piece 2) + // hash(Piece 0|1|2, Piece 3) + let next = remaining_space.0.trailing_zeros(); + let psize = PaddedBytesAmount(1 << next); + + remaining_space.0 ^= psize.0; + + piece_infos.push(filecoin_proofs::pieces::zero_padding(psize.into()).unwrap()); + } + + // We return filler piece infos as this is what `filecoin_proofs::seal_pre_commit_phase1` accepts + piece_infos +} + /// Public inputs for a PoRep. /// References: /// * ] @@ -243,3 +295,101 @@ pub struct PreCommitOutput { pub comm_r: Commitment, pub comm_d: Commitment, } + +#[cfg(test)] +mod test { + use std::io::Cursor; + + use rand::{RngCore, SeedableRng}; + use rand_xorshift::XorShiftRng; + use rstest::rstest; + + use super::*; + + #[test] + fn filler_pieces_for_sizes() { + assert_eq!( + Vec::::new(), + filler_pieces(PaddedBytesAmount(0).into()) + ); + + assert_eq!( + vec![ + // Piece Sizes need to be Power of Twos, at least 128. + filecoin_proofs::pieces::zero_padding(PaddedBytesAmount(128).into()).unwrap(), + filecoin_proofs::pieces::zero_padding(PaddedBytesAmount(256).into()).unwrap(), + filecoin_proofs::pieces::zero_padding(PaddedBytesAmount(512).into()).unwrap(), + filecoin_proofs::pieces::zero_padding(PaddedBytesAmount(1024).into()).unwrap(), + ], + filler_pieces((PaddedBytesAmount(2048) - PaddedBytesAmount(128)).into()) + ); + } + + #[rstest] + // Only one 'real' small piece is added, rest should be padding pieces. + #[case(vec![128])] + // Not aligned to the left pieces + #[case(vec![128, 256])] + // Not aligned to the right pieces + #[case(vec![1024, 256])] + // Biggest possible piece size + #[case(vec![2048])] + fn padding_for_sector(#[case] piece_sizes: Vec) { + let sealer = Sealer::new(RegisteredSealProof::StackedDRG2KiBV1P1); + // Create a file-like sector where non-occupied bytes are 0 + let sector_size = sealer.porep_config.sector_size.0 as usize; + let mut staged_sector = vec![0u8; sector_size]; + + let piece_infos: Vec<(Cursor>, PieceInfo)> = piece_sizes + .into_iter() + .map(|size| { + let (piece_bytes, piece_info) = + piece_with_random_data(PaddedBytesAmount(size as u64)); + + (Cursor::new(piece_bytes), piece_info.into()) + }) + .collect(); + + let pieces: Vec = sealer + .create_sector(piece_infos, Cursor::new(&mut staged_sector)) + .unwrap() + .into_iter() + .map(|p| p.into()) + .collect(); + + let pieces_commd = + filecoin_proofs::compute_comm_d(sealer.porep_config.sector_size, &pieces).unwrap(); + let data_commd = compute_data_comm_d(sealer.porep_config.sector_size, &staged_sector); + assert_eq!(data_commd, pieces_commd) + } + + /// Generates a piece of `size` and a PieceInfo for it + fn piece_with_random_data(size: PaddedBytesAmount) -> (Vec, filecoin_proofs::PieceInfo) { + let rng = &mut XorShiftRng::from_seed(filecoin_proofs::TEST_SEED); + + let piece_size: UnpaddedBytesAmount = size.into(); + let mut piece_bytes = vec![0u8; piece_size.0 as usize]; + rng.fill_bytes(&mut piece_bytes); + let piece_info = + filecoin_proofs::generate_piece_commitment(Cursor::new(&mut piece_bytes), piece_size) + .unwrap(); + + (piece_bytes, piece_info) + } + + /// Computes CommD from the raw data, not from the pieces. + fn compute_data_comm_d( + sector_size: filecoin_proofs::SectorSize, + data: &[u8], + ) -> filecoin_proofs::Commitment { + let data_tree: filecoin_proofs::DataTree = + storage_proofs_core::merkle::create_base_merkle_tree::( + None, + sector_size.0 as usize / storage_proofs_core::util::NODE_SIZE, + data, + ) + .expect("failed to create data tree"); + let comm_d_root: blstrs::Scalar = data_tree.root().into(); + filecoin_proofs::commitment_from_fr(comm_d_root) + } +}