fix(storage-proofs-porep): fix graph generation

- expander: divide before casting to u32
- drg: move predecessor to the first position

fil-proofs-tooling/Cargo.toml

@@ -22,7 +22,7 @@ regex = "1.3.7"
 commandspec = "0.12.2"
 chrono = { version = "0.4.7", features = ["serde"] }
 memmap = "0.7.0"
-bellperson = { version = "0.11", default-features = false }
+bellperson = { version = "0.12", default-features = false }
 rand = "0.7"
 tempfile = "3.0.8"
 cpu-time = "1.0.0"

fil-proofs-tooling/src/bin/gen_graph_cache/main.rs

@@ -131,6 +131,41 @@ fn main() -> Result<()> {
                 0, 0, 0, 0,
             ],
         ),
+        (
+            SECTOR_SIZE_2_KIB, // v1.1
+            [
+                5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0,
+            ],
+        ),
+        (
+            SECTOR_SIZE_8_MIB, // v1.1
+            [
+                6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0,
+            ],
+        ),
+        (
+            SECTOR_SIZE_512_MIB, // v1.1
+            [
+                7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0,
+            ],
+        ),
+        (
+            SECTOR_SIZE_32_GIB, // v1.1
+            [
+                8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0,
+            ],
+        ),
+        (
+            SECTOR_SIZE_64_GIB, // v1.1
+            [
+                9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+                0, 0, 0, 0,
+            ],
+        ),
     ];
 
     let supported_sector_sizes = sector_sizes_and_porep_ids

filecoin-proofs/Cargo.toml

@@ -23,7 +23,7 @@ serde_json = "1.0"
 regex = "1.3.7"
 ff = { version = "0.2.3", package = "fff" }
 blake2b_simd = "0.5"
-bellperson = { version = "0.11", default-features = false }
+bellperson = { version = "0.12", default-features = false }
 clap = "2"
 log = "0.4.7"
 fil_logger = "0.1"
@@ -41,7 +41,7 @@ sha2 = "0.9.1"
 typenum = "1.11.2"
 bitintr = "0.3.0"
 gperftools = { version = "0.2", optional = true }
-phase2 = { version = "0.10", package = "phase21", default-features = false }
+phase2 = { version = "0.11", package = "phase21", default-features = false }
 simplelog = "0.8.0"
 rand_chacha = "0.2.1"
 dialoguer = "0.7.1"

filecoin-proofs/src/constants.rs

@@ -4,6 +4,7 @@ use std::sync::RwLock;
 use lazy_static::lazy_static;
 use storage_proofs::hasher::Hasher;
 use storage_proofs::util::NODE_SIZE;
+use storage_proofs::MAX_LEGACY_POREP_REGISTERED_PROOF_ID;
 use typenum::{U0, U2, U8};
 
 use crate::types::UnpaddedBytesAmount;
@@ -27,6 +28,8 @@ pub const WINDOW_POST_CHALLENGE_COUNT: usize = 10;
 pub const DRG_DEGREE: usize = storage_proofs::drgraph::BASE_DEGREE;
 pub const EXP_DEGREE: usize = storage_proofs::porep::stacked::EXP_DEGREE;
 
+pub const MAX_LEGACY_REGISTERED_SEAL_PROOF_ID: u64 = MAX_LEGACY_POREP_REGISTERED_PROOF_ID;
+
 /// Sector sizes for which parameters have been published.
 pub const PUBLISHED_SECTOR_SIZES: [u64; 10] = [
     SECTOR_SIZE_2_KIB,

filecoin-proofs/tests/api.rs

@@ -15,7 +15,7 @@ use tempfile::NamedTempFile;
 
 use filecoin_proofs::*;
 
-// Use a fixed PoRep ID, so that the parents cache can be re-used between different tests
+// Use a fixed PoRep ID, so that the parents cache can be re-used between some tests.
 const ARBITRARY_POREP_ID: [u8; 32] = [28; 32];
 
 static INIT_LOGGER: Once = Once::new();
@@ -31,55 +31,114 @@ const TEST_SEED: [u8; 16] = [
 
 #[test]
 #[ignore]
-fn test_seal_lifecycle_2kib_base_8() -> Result<()> {
-    seal_lifecycle::<SectorShape2KiB>(SECTOR_SIZE_2_KIB)
+fn test_seal_lifecycle_2kib_porep_id_v1_base_8() -> Result<()> {
+    let porep_id_v1: u64 = 0; // This is a RegisteredSealProof value
+
+    let mut porep_id = [0u8; 32];
+    porep_id[..8].copy_from_slice(&porep_id_v1.to_le_bytes());
+    seal_lifecycle::<SectorShape2KiB>(SECTOR_SIZE_2_KIB, &porep_id)
+}
+
+#[test]
+#[ignore]
+fn test_seal_lifecycle_2kib_porep_id_v1_1_base_8() -> Result<()> {
+    let porep_id_v1_1: u64 = 5; // This is a RegisteredSealProof value
+
+    let mut porep_id = [0u8; 32];
+    porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes());
+    seal_lifecycle::<SectorShape2KiB>(SECTOR_SIZE_2_KIB, &porep_id)
 }
 
 #[test]
 #[ignore]
 fn test_seal_lifecycle_4kib_sub_8_2() -> Result<()> {
-    seal_lifecycle::<SectorShape4KiB>(SECTOR_SIZE_4_KIB)
+    seal_lifecycle::<SectorShape4KiB>(SECTOR_SIZE_4_KIB, &ARBITRARY_POREP_ID)
 }
 
 #[test]
 #[ignore]
 fn test_seal_lifecycle_16kib_sub_8_2() -> Result<()> {
-    seal_lifecycle::<SectorShape16KiB>(SECTOR_SIZE_16_KIB)
+    seal_lifecycle::<SectorShape16KiB>(SECTOR_SIZE_16_KIB, &ARBITRARY_POREP_ID)
 }
 
 #[test]
 #[ignore]
 fn test_seal_lifecycle_32kib_top_8_8_2() -> Result<()> {
-    seal_lifecycle::<SectorShape32KiB>(SECTOR_SIZE_32_KIB)
+    seal_lifecycle::<SectorShape32KiB>(SECTOR_SIZE_32_KIB, &ARBITRARY_POREP_ID)
 }
 
 // These tests are good to run, but take a long time.
 
 //#[test]
 //#[ignore]
-//fn test_seal_lifecycle_512mib_top_8_0_0() -> Result<()> {
-//    seal_lifecycle::<SectorShape512MiB>(SECTOR_SIZE_512_MIB)
+//fn test_seal_lifecycle_512mib_porep_id_v1_top_8_0_0() -> Result<()> {
+//    let porep_id_v1: u64 = 2; // This is a RegisteredSealProof value
+//
+//    let mut porep_id = [0u8; 32];
+//    porep_id[..8].copy_from_slice(&porep_id_v1.to_le_bytes());
+//    seal_lifecycle::<SectorShape512MiB>(SECTOR_SIZE_512_MIB, &porep_id)
+//}
+
+//#[test]
+//#[ignore]
+//fn test_seal_lifecycle_512mib_porep_id_v1_1_top_8_0_0() -> Result<()> {
+//    let porep_id_v1_1: u64 = 7; // This is a RegisteredSealProof value
+//
+//    let mut porep_id = [0u8; 32];
+//    porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes());
+//    seal_lifecycle::<SectorShape512MiB>(SECTOR_SIZE_512_MIB, &porep_id)
+//}
+
+//#[test]
+//#[ignore]
+//fn test_seal_lifecycle_32gib_porep_id_v1_top_8_8_0() -> Result<()> {
+//    let porep_id_v1: u64 = 3; // This is a RegisteredSealProof value
+//
+//    let mut porep_id = [0u8; 32];
+//    porep_id[..8].copy_from_slice(&porep_id_v1.to_le_bytes());
+//    seal_lifecycle::<SectorShape32GiB>(SECTOR_SIZE_32_GIB, &porep_id)
+//}
+
+//#[test]
+//#[ignore]
+//fn test_seal_lifecycle_32gib_porep_id_v1_1_top_8_8_0() -> Result<()> {
+//    let porep_id_v1_1: u64 = 8; // This is a RegisteredSealProof value
+//
+//    let mut porep_id = [0u8; 32];
+//    porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes());
+//    seal_lifecycle::<SectorShape32GiB>(SECTOR_SIZE_32_GIB, &porep_id)
 //}
 
 //#[test]
 //#[ignore]
-//fn test_seal_lifecycle_32gib_top_8_8_0() -> Result<()> {
-//    seal_lifecycle::<SectorShape32GiB>(SECTOR_SIZE_32_GIB)
+//fn test_seal_lifecycle_64gib_porep_id_v1_top_8_8_2() -> Result<()> {
+//    let porep_id_v1: u64 = 4; // This is a RegisteredSealProof value
+//
+//    let mut porep_id = [0u8; 32];
+//    porep_id[..8].copy_from_slice(&porep_id_v1.to_le_bytes());
+//    seal_lifecycle::<SectorShape64GiB>(SECTOR_SIZE_64_GIB, &porep_id)
 //}
 
 //#[test]
 //#[ignore]
-//fn test_seal_lifecycle_64gib_top_8_8_2() -> Result<()> {
-//    seal_lifecycle::<SectorShape64GiB>(SECTOR_SIZE_64_GIB)
+//fn test_seal_lifecycle_64gib_porep_id_v1_1_top_8_8_2() -> Result<()> {
+//    let porep_id_v1_1: u64 = 9; // This is a RegisteredSealProof value
+//
+//    let mut porep_id = [0u8; 32];
+//    porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes());
+//    seal_lifecycle::<SectorShape64GiB>(SECTOR_SIZE_64_GIB, &porep_id)
 //}
 
-fn seal_lifecycle<Tree: 'static + MerkleTreeTrait>(sector_size: u64) -> Result<()> {
+fn seal_lifecycle<Tree: 'static + MerkleTreeTrait>(
+    sector_size: u64,
+    porep_id: &[u8; 32],
+) -> Result<()> {
     let rng = &mut XorShiftRng::from_seed(TEST_SEED);
     let prover_fr: DefaultTreeDomain = Fr::random(rng).into();
     let mut prover_id = [0u8; 32];
     prover_id.copy_from_slice(AsRef::<[u8]>::as_ref(&prover_fr));
 
-    create_seal::<_, Tree>(rng, sector_size, prover_id, false)?;
+    create_seal::<_, Tree>(rng, sector_size, prover_id, false, porep_id)?;
     Ok(())
 }
 
@@ -116,23 +175,56 @@ fn clear_cache_dir_keep_data_layer(cache_dir: &tempfile::TempDir) {
 }
 
 #[test]
-fn test_resumable_seal_skip_proofs() {
-    run_resumable_seal::<SectorShape2KiB>(true, 0);
-    run_resumable_seal::<SectorShape2KiB>(true, 1);
+fn test_resumable_seal_skip_proofs_v1() {
+    let porep_id_v1: u64 = 0; // This is a RegisteredSealProof value
+
+    let mut porep_id = [0u8; 32];
+    porep_id[..8].copy_from_slice(&porep_id_v1.to_le_bytes());
+    run_resumable_seal::<SectorShape2KiB>(true, 0, &porep_id);
+    run_resumable_seal::<SectorShape2KiB>(true, 1, &porep_id);
+}
+
+#[test]
+fn test_resumable_seal_skip_proofs_v1_1() {
+    let porep_id_v1_1: u64 = 5; // This is a RegisteredSealProof value
+
+    let mut porep_id = [0u8; 32];
+    porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes());
+    run_resumable_seal::<SectorShape2KiB>(true, 0, &porep_id);
+    run_resumable_seal::<SectorShape2KiB>(true, 1, &porep_id);
 }
 
 #[test]
 #[ignore]
-fn test_resumable_seal() {
-    run_resumable_seal::<SectorShape2KiB>(false, 0);
-    run_resumable_seal::<SectorShape2KiB>(false, 1);
+fn test_resumable_seal_v1() {
+    let porep_id_v1: u64 = 0; // This is a RegisteredSealProof value
+
+    let mut porep_id = [0u8; 32];
+    porep_id[..8].copy_from_slice(&porep_id_v1.to_le_bytes());
+    run_resumable_seal::<SectorShape2KiB>(false, 0, &porep_id);
+    run_resumable_seal::<SectorShape2KiB>(false, 1, &porep_id);
+}
+
+#[test]
+#[ignore]
+fn test_resumable_seal_v1_1() {
+    let porep_id_v1_1: u64 = 5; // This is a RegisteredSealProof value
+
+    let mut porep_id = [0u8; 32];
+    porep_id[..8].copy_from_slice(&porep_id_v1_1.to_le_bytes());
+    run_resumable_seal::<SectorShape2KiB>(false, 0, &porep_id);
+    run_resumable_seal::<SectorShape2KiB>(false, 1, &porep_id);
 }
 
 /// Create a seal, delete a layer and resume
 ///
 /// The current code works on two layers only. The `layer_to_delete` specifies (zero-based) which
 /// layer should be deleted.
-fn run_resumable_seal<Tree: 'static + MerkleTreeTrait>(skip_proofs: bool, layer_to_delete: usize) {
+fn run_resumable_seal<Tree: 'static + MerkleTreeTrait>(
+    skip_proofs: bool,
+    layer_to_delete: usize,
+    porep_id: &[u8; 32],
+) {
     init_logger();
 
     let sector_size = SECTOR_SIZE_2_KIB;
@@ -146,7 +238,7 @@ fn run_resumable_seal<Tree: 'static + MerkleTreeTrait>(skip_proofs: bool, layer_
     let sealed_sector_file = NamedTempFile::new().expect("failed to created sealed sector file");
     let cache_dir = tempfile::tempdir().expect("failed to create temp dir");
 
-    let config = porep_config(sector_size, ARBITRARY_POREP_ID);
+    let config = porep_config(sector_size, *porep_id);
     let ticket = rng.gen();
     let sector_id = rng.gen::<u64>().into();
 
@@ -271,7 +363,8 @@ fn test_winning_post_empty_sector_challenge() -> Result<()> {
     let sector_count = 0;
     let sector_size = SECTOR_SIZE_2_KIB;
 
-    let (_, _, _, _) = create_seal::<_, SectorShape2KiB>(rng, sector_size, prover_id, true)?;
+    let (_, _, _, _) =
+        create_seal::<_, SectorShape2KiB>(rng, sector_size, prover_id, true, &ARBITRARY_POREP_ID)?;
 
     let random_fr: DefaultTreeDomain = Fr::random(rng).into();
     let mut randomness = [0u8; 32];
@@ -304,9 +397,9 @@ fn winning_post<Tree: 'static + MerkleTreeTrait>(sector_size: u64, fake: bool) -
     prover_id.copy_from_slice(AsRef::<[u8]>::as_ref(&prover_fr));
 
     let (sector_id, replica, comm_r, cache_dir) = if fake {
-        create_fake_seal::<_, Tree>(rng, sector_size)?
+        create_fake_seal::<_, Tree>(rng, sector_size, &ARBITRARY_POREP_ID)?
     } else {
-        create_seal::<_, Tree>(rng, sector_size, prover_id, true)?
+        create_seal::<_, Tree>(rng, sector_size, prover_id, true, &ARBITRARY_POREP_ID)?
     };
     let sector_count = WINNING_POST_SECTOR_COUNT;
 
@@ -496,9 +589,9 @@ fn window_post<Tree: 'static + MerkleTreeTrait>(
 
     for _ in 0..total_sector_count {
         let (sector_id, replica, comm_r, cache_dir) = if fake {
-            create_fake_seal::<_, Tree>(rng, sector_size)?
+            create_fake_seal::<_, Tree>(rng, sector_size, &ARBITRARY_POREP_ID)?
         } else {
-            create_seal::<_, Tree>(rng, sector_size, prover_id, true)?
+            create_seal::<_, Tree>(rng, sector_size, prover_id, true, &ARBITRARY_POREP_ID)?
         };
         priv_replicas.insert(
             sector_id,
@@ -719,14 +812,15 @@ fn create_seal<R: Rng, Tree: 'static + MerkleTreeTrait>(
     sector_size: u64,
     prover_id: ProverId,
     skip_proof: bool,
+    porep_id: &[u8; 32],
 ) -> Result<(SectorId, NamedTempFile, Commitment, tempfile::TempDir)> {
     init_logger();
 
     let (mut piece_file, piece_bytes) = generate_piece_file(sector_size)?;
     let sealed_sector_file = NamedTempFile::new()?;
     let cache_dir = tempfile::tempdir().expect("failed to create temp dir");
 
-    let config = porep_config(sector_size, ARBITRARY_POREP_ID);
+    let config = porep_config(sector_size, *porep_id);
     let ticket = rng.gen();
     let seed = rng.gen();
     let sector_id = rng.gen::<u64>().into();
@@ -776,12 +870,19 @@ fn create_seal<R: Rng, Tree: 'static + MerkleTreeTrait>(
 fn create_fake_seal<R: rand::Rng, Tree: 'static + MerkleTreeTrait>(
     mut rng: &mut R,
     sector_size: u64,
+    porep_id: &[u8; 32],
 ) -> Result<(SectorId, NamedTempFile, Commitment, tempfile::TempDir)> {
     init_logger();
 
     let sealed_sector_file = NamedTempFile::new()?;
 
-    let config = porep_config(sector_size, ARBITRARY_POREP_ID);
+    let config = PoRepConfig {
+        sector_size: SectorSize(sector_size),
+        partitions: PoRepProofPartitions(
+            *POREP_PARTITIONS.read().unwrap().get(&sector_size).unwrap(),
+        ),
+        porep_id: *porep_id,
+    };
 
     let cache_dir = tempfile::tempdir().unwrap();
 

storage-proofs/Cargo.toml

@@ -22,4 +22,3 @@ measurements = ["storage-proofs-core/measurements"]
 profile = ["measurements"]
 pairing = ["storage-proofs-core/pairing", "storage-proofs-post/pairing", "storage-proofs-porep/pairing"]
 blst = ["storage-proofs-core/blst", "storage-proofs-post/blst", "storage-proofs-porep/blst"]
-

storage-proofs/core/Cargo.toml

@@ -30,15 +30,15 @@ blake2b_simd = "0.5"
 blake2s_simd = "0.5"
 toml = "0.5"
 ff = { version = "0.2.3", package = "fff" }
-bellperson = { version = "0.11", default-features = false }
+bellperson = { version = "0.12", default-features = false }
 serde_json = "1.0"
 log = "0.4.7"
 rand_chacha = "0.2.1"
 hex = "0.4.0"
 generic-array = "0.14.4"
 anyhow = "1.0.23"
 thiserror = "1.0.6"
-neptune = { version = "2.1.0", default-features = false, features = ["gpu"] }
+neptune = { version = "2.2.0", default-features = false, features = ["gpu"] }
 cpu-time = { version = "1.0", optional = true }
 gperftools = { version = "0.2", optional = true }
 num_cpus = "1.10.1"