From 0301a68b66b19b154f8eba67b1c99d253219f33c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandar=20Terenti=C4=87?= Date: Fri, 2 Dec 2022 12:57:53 +0100 Subject: [PATCH 1/4] Add reconstruct columns function. --- kate/recovery/src/com.rs | 47 ++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/kate/recovery/src/com.rs b/kate/recovery/src/com.rs index 9fa3c33b..6174d77e 100644 --- a/kate/recovery/src/com.rs +++ b/kate/recovery/src/com.rs @@ -1,7 +1,4 @@ -use std::{ - collections::HashMap, - convert::{TryFrom, TryInto}, -}; +use std::{collections::HashMap, convert::TryFrom}; use codec::Decode; use dusk_bytes::Serializable; @@ -132,6 +129,32 @@ pub fn reconstruct_extrinsics( unflatten_padded_data(ranges, data).map_err(ReconstructionError::DataDecodingError) } +pub fn reconstruct_columns( + dimensions: &matrix::Dimensions, + cells: Vec, +) -> Result)>, ReconstructionError> { + let columns = map_cells(dimensions, cells)?; + + columns + .iter() + .map(|(&col, cells)| { + if cells.len() < dimensions.rows().into() { + return Err(ReconstructionError::InvalidColumn(col)); + } + + let cells = cells.values().cloned().collect::>(); + + let column = reconstruct_column(dimensions.extended_rows(), &cells) + .map_err(ReconstructionError::ColumnReconstructionError)? + .iter() + .flat_map(BlsScalar::to_bytes) + .collect::>(); + + Ok((col, column)) + }) + .collect::>() +} + fn reconstruct_available( dimensions: &matrix::Dimensions, cells: Vec, @@ -146,12 +169,8 @@ fn reconstruct_available( return Err(ReconstructionError::InvalidColumn(col)); } let cells = column_cells.values().cloned().collect::>(); - let row_count: u16 = dimensions - .extended_rows() - .try_into() - .map_err(|_| ReconstructionError::RowCountExceeded)?; - reconstruct_column(row_count, &cells) + reconstruct_column(dimensions.extended_rows(), &cells) .map(|scalars| scalars.into_iter().map(Some).collect::>()) .map_err(ReconstructionError::ColumnReconstructionError) }, @@ -396,7 +415,7 @@ pub type AppDataRange = std::ops::Range; // performing one round of ifft should reveal original data which were // coded together pub fn reconstruct_column( - row_count: u16, + row_count: u32, cells: &[data::DataCell], ) -> Result, String> { // just ensures all rows are from same column ! @@ -410,10 +429,10 @@ pub fn reconstruct_column( // given row index in column of interest, finds it if present // and returns back wrapped in `Some`, otherwise returns `None` - fn find_row_by_index(idx: u16, cells: &[data::DataCell]) -> Option { + fn find_row_by_index(idx: u32, cells: &[data::DataCell]) -> Option { cells .iter() - .find(|cell| cell.position.row == idx as u32) + .find(|cell| cell.position.row == idx) .map(|cell| { <[u8; BlsScalar::SIZE]>::try_from(&cell.data[..]) .expect("didn't find u8 array of length 32") @@ -774,7 +793,7 @@ mod tests { } } - fn build_coded_eval_domain() -> (Vec, u16, u16) { + fn build_coded_eval_domain() -> (Vec, u32, u16) { let domain_size = 1u16 << 2; let row_count = domain_size.checked_mul(2).unwrap(); let eval_domain = EvaluationDomain::new(domain_size as usize).unwrap(); @@ -793,7 +812,7 @@ mod tests { let coded = eval_domain.fft(&src); assert!(coded.len() == row_count as usize); - (coded, row_count, domain_size) + (coded, row_count.into(), domain_size) } // Following test cases attempt to figure out any loop holes From eecc7f71e7069cdf325e18ad3eb9cb98aab711fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandar=20Terenti=C4=87?= Date: Tue, 6 Dec 2022 15:59:39 +0100 Subject: [PATCH 2/4] Add column positions function. --- Cargo.lock | 62 +++++++++++++++++++++++++++++++++++-- kate/recovery/Cargo.toml | 1 + kate/recovery/src/com.rs | 33 ++++++++++++++++++-- kate/recovery/src/matrix.rs | 11 +++++++ 4 files changed, 103 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 09c98333..633d2a4c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1682,6 +1682,7 @@ dependencies = [ "dusk-plonk", "getrandom 0.2.7", "hex", + "num", "once_cell", "parity-scale-codec 3.1.5", "rand 0.8.5", @@ -1921,6 +1922,20 @@ dependencies = [ "tiny-keccak", ] +[[package]] +name = "num" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606" +dependencies = [ + "num-bigint 0.4.3", + "num-complex", + "num-integer", + "num-iter", + "num-rational 0.4.1", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.2.6" @@ -1932,6 +1947,26 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ae39348c8bc5fbd7f40c727a9925f03517afd2ab27d46702108b6a7e5414c19" +dependencies = [ + "num-traits", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -1942,6 +1977,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + [[package]] name = "num-rational" version = "0.2.4" @@ -1949,7 +1995,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c000134b5dbf44adc5cb772486d335293351644b801551abe8f75c84cfa4aef" dependencies = [ "autocfg", - "num-bigint", + "num-bigint 0.2.6", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0638a1c9d0a3c0914158145bc76cff373a75a627e6ecbfb71cbe6f453a5a19b0" +dependencies = [ + "autocfg", + "num-bigint 0.4.3", "num-integer", "num-traits", ] @@ -3772,7 +3830,7 @@ dependencies = [ "downcast-rs", "libc", "memory_units", - "num-rational", + "num-rational 0.2.4", "num-traits", "parity-wasm", "wasmi-validation", diff --git a/kate/recovery/Cargo.toml b/kate/recovery/Cargo.toml index a787fa84..d1032a3e 100644 --- a/kate/recovery/Cargo.toml +++ b/kate/recovery/Cargo.toml @@ -10,6 +10,7 @@ derive_more = "0.99.17" dusk-bytes = "0.1.6" dusk-plonk = { git = "https://github.com/maticnetwork/plonk.git", tag = "v0.12.0-polygon-2" } getrandom = { version = "0.2", features = ["js"] } +num = "0.4.0" once_cell = { version = "1.9.0", default-features = false } rand = "0.8.4" rand_chacha = "0.3" diff --git a/kate/recovery/src/com.rs b/kate/recovery/src/com.rs index 6174d77e..32858dee 100644 --- a/kate/recovery/src/com.rs +++ b/kate/recovery/src/com.rs @@ -1,8 +1,13 @@ -use std::{collections::HashMap, convert::TryFrom}; - use codec::Decode; use dusk_bytes::Serializable; use dusk_plonk::{fft::EvaluationDomain, prelude::BlsScalar}; +use num::ToPrimitive; +use rand::seq::SliceRandom; +use std::{ + collections::{BTreeSet, HashMap}, + convert::TryFrom, + iter::FromIterator, +}; use thiserror::Error; use crate::{config, data, index, matrix}; @@ -25,6 +30,30 @@ pub enum ReconstructionError { RowCountExceeded, } +/// From given positions, constructs related columns positions, up to given factor. +/// E.g. if factor is 0.66, 66% of matched columns will be returned. +/// Positions in columns are random. +/// Function panics if factor is above 1.0. +pub fn columns_positions( + dimensions: &matrix::Dimensions, + positions: Vec, + factor: f64, +) -> Vec { + assert!(factor <= 1.0); + + let cells = (factor * dimensions.extended_rows() as f64) + .to_usize() + .expect("result is lesser than usize maximum"); + + let rng = &mut rand::thread_rng(); + + BTreeSet::from_iter(positions.iter().map(|position| position.col)) + .into_iter() + .map(|col| dimensions.col_positions(col)) + .flat_map(|col| col.choose_multiple(rng, cells).cloned().collect::>()) + .collect::>() +} + /// Creates hash map of columns, each being hash map of cells, from vector of cells. /// Intention is to be able to find duplicates and to group cells by column. fn map_cells( diff --git a/kate/recovery/src/matrix.rs b/kate/recovery/src/matrix.rs index 733ccb84..9c2d21d4 100644 --- a/kate/recovery/src/matrix.rs +++ b/kate/recovery/src/matrix.rs @@ -98,6 +98,17 @@ impl Dimensions { .collect::>() } + /// Cell positions for given column in extended matrix. + /// Empty if column index is not valid. + pub fn col_positions(&self, col: u16) -> Vec { + if self.cols() <= col { + return vec![]; + } + (0..self.extended_rows()) + .map(|row| Position { col, row }) + .collect::>() + } + /// Cell positions for given rows in extended matrix. /// Empty if row index is not valid. fn extended_row_positions(&self, row: u32) -> Vec { From ac8fae44d673e8395f61a50616e45f9318ec770a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandar=20Terenti=C4=87?= Date: Wed, 7 Dec 2022 13:16:09 +0100 Subject: [PATCH 3/4] Return hash map of columns in reconstruct_columns. --- kate/recovery/src/com.rs | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/kate/recovery/src/com.rs b/kate/recovery/src/com.rs index 32858dee..b661b298 100644 --- a/kate/recovery/src/com.rs +++ b/kate/recovery/src/com.rs @@ -4,13 +4,16 @@ use dusk_plonk::{fft::EvaluationDomain, prelude::BlsScalar}; use num::ToPrimitive; use rand::seq::SliceRandom; use std::{ - collections::{BTreeSet, HashMap}, + collections::{HashMap, HashSet}, convert::TryFrom, iter::FromIterator, }; use thiserror::Error; -use crate::{config, data, index, matrix}; +use crate::{ + config::{self, CHUNK_SIZE}, + data, index, matrix, +}; #[derive(Debug, Error)] pub enum ReconstructionError { @@ -36,7 +39,7 @@ pub enum ReconstructionError { /// Function panics if factor is above 1.0. pub fn columns_positions( dimensions: &matrix::Dimensions, - positions: Vec, + positions: &[matrix::Position], factor: f64, ) -> Vec { assert!(factor <= 1.0); @@ -47,7 +50,9 @@ pub fn columns_positions( let rng = &mut rand::thread_rng(); - BTreeSet::from_iter(positions.iter().map(|position| position.col)) + let columns: HashSet = HashSet::from_iter(positions.iter().map(|position| position.col)); + + columns .into_iter() .map(|col| dimensions.col_positions(col)) .flat_map(|col| col.choose_multiple(rng, cells).cloned().collect::>()) @@ -158,10 +163,17 @@ pub fn reconstruct_extrinsics( unflatten_padded_data(ranges, data).map_err(ReconstructionError::DataDecodingError) } +/// Reconstructs columns for given cells. +/// +/// # Arguments +/// +/// * `dimensions` - Extended matrix dimensions +/// * `cells` - Cells from required columns, at least 50% cells per column pub fn reconstruct_columns( dimensions: &matrix::Dimensions, - cells: Vec, -) -> Result)>, ReconstructionError> { + cells: &[data::Cell], +) -> Result>, ReconstructionError> { + let cells: Vec = cells.iter().cloned().map(Into::into).collect::>(); let columns = map_cells(dimensions, cells)?; columns @@ -176,8 +188,8 @@ pub fn reconstruct_columns( let column = reconstruct_column(dimensions.extended_rows(), &cells) .map_err(ReconstructionError::ColumnReconstructionError)? .iter() - .flat_map(BlsScalar::to_bytes) - .collect::>(); + .map(BlsScalar::to_bytes) + .collect::>(); Ok((col, column)) }) From d52b51882a229103507b231ee18022f8c5da5bb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Aleksandar=20Terenti=C4=87?= Date: Thu, 8 Dec 2022 12:09:38 +0100 Subject: [PATCH 4/4] Update recovery crate version. --- Cargo.lock | 2 +- kate/recovery/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 633d2a4c..590e5d62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1675,7 +1675,7 @@ dependencies = [ [[package]] name = "kate-recovery" -version = "0.6.1" +version = "0.7.0" dependencies = [ "derive_more", "dusk-bytes", diff --git a/kate/recovery/Cargo.toml b/kate/recovery/Cargo.toml index d1032a3e..d1dbb00e 100644 --- a/kate/recovery/Cargo.toml +++ b/kate/recovery/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "kate-recovery" -version = "0.6.1" +version = "0.7.0" authors = ["Denis Ermolin "] edition = "2018"