diff --git a/CHANGELOG.md b/CHANGELOG.md index 79e77e40..44e9bc65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Seeds to multiple algorithims that depend on random number generation. - Added feature `js` to use WASM in browser -## BREAKING CHANGE +## BREAKING CHANGES +- SVM algorithms now use an enum type of Kernel, rather than distinct structs with a shared trait, so that they may be compared in a single hyperparameter search. - Added a new parameter to `train_test_split` to define the seed. ## [0.2.1] - 2022-05-10 diff --git a/src/model_selection/hyper_tuning.rs b/src/model_selection/hyper_tuning.rs index cb69da18..fea8dd0e 100644 --- a/src/model_selection/hyper_tuning.rs +++ b/src/model_selection/hyper_tuning.rs @@ -60,9 +60,9 @@ where #[cfg(test)] mod tests { - use crate::linear::logistic_regression::{ - LogisticRegression, LogisticRegressionSearchParameters, -}; + use crate::linear::logistic_regression::{ + LogisticRegression, LogisticRegressionSearchParameters, + }; #[test] fn test_grid_search() { @@ -113,5 +113,31 @@ mod tests { .unwrap(); assert!([0., 1.].contains(&results.parameters.alpha)); - } + } + + #[test] + fn svm_check() { + let breast_cancer = crate::dataset::breast_cancer::load_dataset(); + let y = breast_cancer.target; + let x = DenseMatrix::from_array( + breast_cancer.num_samples, + breast_cancer.num_features, + &breast_cancer.data, + ); + let kernels = vec![ + Kernel::Linear, + Kernel::RBF { gamma: 0.001 }, + Kernel::RBF { gamma: 0.0001 }, + ]; + let parameters = SVCSearchParameters { + kernel: kernels, + c: vec![0., 10., 100., 1000.], + ..Default::default() + }; + let cv = KFold { + n_splits: 5, + ..KFold::default() + }; + grid_search(SVC::fit, &x, &y, parameters.into_iter(), cv, &recall).unwrap(); + } } diff --git a/src/svm/mod.rs b/src/svm/mod.rs index 4c71b3f2..e9957885 100644 --- a/src/svm/mod.rs +++ b/src/svm/mod.rs @@ -32,126 +32,60 @@ use serde::{Deserialize, Serialize}; use crate::linalg::BaseVector; use crate::math::num::RealNumber; -/// Defines a kernel function -pub trait Kernel>: Clone { - /// Apply kernel function to x_i and x_j - fn apply(&self, x_i: &V, x_j: &V) -> T; -} - -/// Pre-defined kernel functions -pub struct Kernels {} - -impl Kernels { +/// Kernel functions +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum Kernel { /// Linear kernel - pub fn linear() -> LinearKernel { - LinearKernel {} - } - + Linear, /// Radial basis function kernel (Gaussian) - pub fn rbf(gamma: T) -> RBFKernel { - RBFKernel { gamma } - } - - /// Polynomial kernel - /// * `degree` - degree of the polynomial - /// * `gamma` - kernel coefficient - /// * `coef0` - independent term in kernel function - pub fn polynomial(degree: T, gamma: T, coef0: T) -> PolynomialKernel { - PolynomialKernel { - degree, - gamma, - coef0, - } - } - + RBF { + /// kernel coefficient + gamma: T, + }, + /// Sigmoid kernel + Sigmoid { + /// kernel coefficient + gamma: T, + /// independent term in kernel function + coef0: T, + }, /// Polynomial kernel - /// * `degree` - degree of the polynomial - /// * `n_features` - number of features in vector - pub fn polynomial_with_degree( + Polynomial { + /// kernel coefficient + gamma: T, + /// independent term in kernel function + coef0: T, + /// degree of the polynomial degree: T, - n_features: usize, - ) -> PolynomialKernel { - let coef0 = T::one(); - let gamma = T::one() / T::from_usize(n_features).unwrap(); - Kernels::polynomial(degree, gamma, coef0) - } - - /// Sigmoid kernel - /// * `gamma` - kernel coefficient - /// * `coef0` - independent term in kernel function - pub fn sigmoid(gamma: T, coef0: T) -> SigmoidKernel { - SigmoidKernel { gamma, coef0 } - } - - /// Sigmoid kernel - /// * `gamma` - kernel coefficient - pub fn sigmoid_with_gamma(gamma: T) -> SigmoidKernel { - SigmoidKernel { - gamma, - coef0: T::one(), - } - } + }, } -/// Linear Kernel -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct LinearKernel {} - -/// Radial basis function (Gaussian) kernel -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RBFKernel { - /// kernel coefficient - pub gamma: T, -} - -/// Polynomial kernel -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct PolynomialKernel { - /// degree of the polynomial - pub degree: T, - /// kernel coefficient - pub gamma: T, - /// independent term in kernel function - pub coef0: T, -} - -/// Sigmoid (hyperbolic tangent) kernel -#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct SigmoidKernel { - /// kernel coefficient - pub gamma: T, - /// independent term in kernel function - pub coef0: T, -} - -impl> Kernel for LinearKernel { - fn apply(&self, x_i: &V, x_j: &V) -> T { - x_i.dot(x_j) +impl Default for Kernel { + fn default() -> Self { + Kernel::Linear } } -impl> Kernel for RBFKernel { - fn apply(&self, x_i: &V, x_j: &V) -> T { - let v_diff = x_i.sub(x_j); - (-self.gamma * v_diff.mul(&v_diff).sum()).exp() - } -} - -impl> Kernel for PolynomialKernel { - fn apply(&self, x_i: &V, x_j: &V) -> T { - let dot = x_i.dot(x_j); - (self.gamma * dot + self.coef0).powf(self.degree) - } -} - -impl> Kernel for SigmoidKernel { - fn apply(&self, x_i: &V, x_j: &V) -> T { - let dot = x_i.dot(x_j); - (self.gamma * dot + self.coef0).tanh() +fn apply>(kernel: &Kernel, x_i: &V, x_j: &V) -> T { + match kernel { + Kernel::Polynomial { + degree, + gamma, + coef0, + } => { + let dot = x_i.dot(x_j); + (*gamma * dot + *coef0).powf(*degree) + } + Kernel::Sigmoid { gamma, coef0 } => { + let dot = x_i.dot(x_j); + (*gamma * dot + *coef0).tanh() + } + Kernel::RBF { gamma } => { + let v_diff = x_i.sub(x_j); + (-*gamma * v_diff.mul(&v_diff).sum()).exp() + } + Kernel::Linear => x_i.dot(x_j), } } @@ -165,7 +99,7 @@ mod tests { let v1 = vec![1., 2., 3.]; let v2 = vec![4., 5., 6.]; - assert_eq!(32f64, Kernels::linear().apply(&v1, &v2)); + assert_eq!(32f64, apply(&Kernel::Linear, &v1, &v2)); } #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] @@ -174,7 +108,7 @@ mod tests { let v1 = vec![1., 2., 3.]; let v2 = vec![4., 5., 6.]; - assert!((0.2265f64 - Kernels::rbf(0.055).apply(&v1, &v2)).abs() < 1e-4); + assert!((0.2265f64 - apply(&Kernel::RBF { gamma: 0.055 }, &v1, &v2)).abs() < 1e-4); } #[cfg_attr(target_arch = "wasm32", wasm_bindgen_test::wasm_bindgen_test)] @@ -184,7 +118,17 @@ mod tests { let v2 = vec![4., 5., 6.]; assert!( - (4913f64 - Kernels::polynomial(3.0, 0.5, 1.0).apply(&v1, &v2)).abs() + (4913f64 + - apply( + &Kernel::Polynomial { + gamma: 0.5, + coef0: 1.0, + degree: 3.0 + }, + &v1, + &v2 + )) + .abs() < std::f64::EPSILON ); } @@ -195,6 +139,18 @@ mod tests { let v1 = vec![1., 2., 3.]; let v2 = vec![4., 5., 6.]; - assert!((0.3969f64 - Kernels::sigmoid(0.01, 0.1).apply(&v1, &v2)).abs() < 1e-4); + assert!( + (0.3969f64 + - apply( + &Kernel::Sigmoid { + gamma: 0.01, + coef0: 0.1 + }, + &v1, + &v2 + )) + .abs() + < 1e-4 + ); } } diff --git a/src/svm/svc.rs b/src/svm/svc.rs index 97b91de3..9b861e9e 100644 --- a/src/svm/svc.rs +++ b/src/svm/svc.rs @@ -28,7 +28,7 @@ //! //! ``` //! use smartcore::linalg::naive::dense_matrix::*; -//! use smartcore::svm::Kernels; +//! use smartcore::svm::Kernel; //! use smartcore::svm::svc::{SVC, SVCParameters}; //! //! // Iris dataset @@ -85,12 +85,12 @@ use crate::linalg::BaseVector; use crate::linalg::Matrix; use crate::math::num::RealNumber; use crate::rand::get_rng_impl; -use crate::svm::{Kernel, Kernels, LinearKernel}; +use crate::svm::Kernel; #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Debug, Clone)] /// SVC Parameters -pub struct SVCParameters, K: Kernel> { +pub struct SVCParameters> { #[cfg_attr(feature = "serde", serde(default))] /// Number of epochs. pub epoch: usize, @@ -102,8 +102,7 @@ pub struct SVCParameters, K: Kernel pub tol: T, #[cfg_attr(feature = "serde", serde(default))] /// The kernel function. - pub kernel: K, - #[cfg_attr(feature = "serde", serde(default))] + pub kernel: Kernel, /// Unused parameter. m: PhantomData, #[cfg_attr(feature = "serde", serde(default))] @@ -114,7 +113,7 @@ pub struct SVCParameters, K: Kernel /// SVC grid search parameters #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Debug, Clone)] -pub struct SVCSearchParameters, K: Kernel> { +pub struct SVCSearchParameters> { #[cfg_attr(feature = "serde", serde(default))] /// Number of epochs. pub epoch: Vec, @@ -126,7 +125,7 @@ pub struct SVCSearchParameters, K: Kernel, #[cfg_attr(feature = "serde", serde(default))] /// The kernel function. - pub kernel: Vec, + pub kernel: Vec>, #[cfg_attr(feature = "serde", serde(default))] /// Unused parameter. m: PhantomData, @@ -136,8 +135,8 @@ pub struct SVCSearchParameters, K: Kernel, K: Kernel> { - svc_search_parameters: SVCSearchParameters, +pub struct SVCSearchParametersIterator> { + svc_search_parameters: SVCSearchParameters, current_epoch: usize, current_c: usize, current_tol: usize, @@ -145,11 +144,9 @@ pub struct SVCSearchParametersIterator, K: Kernel, K: Kernel> IntoIterator - for SVCSearchParameters -{ - type Item = SVCParameters; - type IntoIter = SVCSearchParametersIterator; +impl> IntoIterator for SVCSearchParameters { + type Item = SVCParameters; + type IntoIter = SVCSearchParametersIterator; fn into_iter(self) -> Self::IntoIter { SVCSearchParametersIterator { @@ -163,22 +160,20 @@ impl, K: Kernel> IntoIterator } } -impl, K: Kernel> Iterator - for SVCSearchParametersIterator -{ - type Item = SVCParameters; +impl> Iterator for SVCSearchParametersIterator { + type Item = SVCParameters; fn next(&mut self) -> Option { if self.current_epoch == self.svc_search_parameters.epoch.len() && self.current_c == self.svc_search_parameters.c.len() && self.current_tol == self.svc_search_parameters.tol.len() && self.current_kernel == self.svc_search_parameters.kernel.len() - && self.current_seed == self.svc_search_parameters.kernel.len() + && self.current_seed == self.svc_search_parameters.seed.len() { return None; } - let next = SVCParameters:: { + let next = SVCParameters:: { epoch: self.svc_search_parameters.epoch[self.current_epoch], c: self.svc_search_parameters.c[self.current_c], tol: self.svc_search_parameters.tol[self.current_tol], @@ -201,7 +196,7 @@ impl, K: Kernel> Iterator self.current_c = 0; self.current_tol = 0; self.current_kernel += 1; - } else if self.current_kernel + 1 < self.svc_search_parameters.kernel.len() { + } else if self.current_seed + 1 < self.svc_search_parameters.seed.len() { self.current_epoch = 0; self.current_c = 0; self.current_tol = 0; @@ -219,9 +214,9 @@ impl, K: Kernel> Iterator } } -impl> Default for SVCSearchParameters { +impl> Default for SVCSearchParameters { fn default() -> Self { - let default_params: SVCParameters = SVCParameters::default(); + let default_params: SVCParameters = SVCParameters::default(); SVCSearchParameters { epoch: vec![default_params.epoch], @@ -239,14 +234,14 @@ impl> Default for SVCSearchParameters, K: Kernel> { +pub struct SVC> { classes: Vec, - kernel: K, + kernel: Kernel, instances: Vec, w: Vec, b: T, @@ -264,27 +259,27 @@ struct SupportVector> { k: T, } -struct Cache<'a, T: RealNumber, M: Matrix, K: Kernel> { - kernel: &'a K, +struct Cache<'a, T: RealNumber, M: Matrix> { + kernel: &'a Kernel, data: HashMap<(usize, usize), T>, phantom: PhantomData, } -struct Optimizer<'a, T: RealNumber, M: Matrix, K: Kernel> { +struct Optimizer<'a, T: RealNumber, M: Matrix> { x: &'a M, y: &'a M::RowVector, - parameters: &'a SVCParameters, + parameters: &'a SVCParameters, svmin: usize, svmax: usize, gmin: T, gmax: T, tau: T, sv: Vec>, - kernel: &'a K, + kernel: &'a Kernel, recalculate_minmax_grad: bool, } -impl, K: Kernel> SVCParameters { +impl> SVCParameters { /// Number of epochs. pub fn with_epoch(mut self, epoch: usize) -> Self { self.epoch = epoch; @@ -301,7 +296,7 @@ impl, K: Kernel> SVCParameters>(&self, kernel: KK) -> SVCParameters { + pub fn with_kernel(&self, kernel: Kernel) -> SVCParameters { SVCParameters { epoch: self.epoch, c: self.c, @@ -319,36 +314,34 @@ impl, K: Kernel> SVCParameters> Default for SVCParameters { +impl> Default for SVCParameters { fn default() -> Self { SVCParameters { epoch: 2, c: T::one(), tol: T::from_f64(1e-3).unwrap(), - kernel: Kernels::linear(), + kernel: Kernel::default(), m: PhantomData, seed: None, } } } -impl, K: Kernel> - SupervisedEstimator> for SVC +impl> SupervisedEstimator> + for SVC { - fn fit(x: &M, y: &M::RowVector, parameters: SVCParameters) -> Result { + fn fit(x: &M, y: &M::RowVector, parameters: SVCParameters) -> Result { SVC::fit(x, y, parameters) } } -impl, K: Kernel> Predictor - for SVC -{ +impl> Predictor for SVC { fn predict(&self, x: &M) -> Result { self.predict(x) } } -impl, K: Kernel> SVC { +impl> SVC { /// Fits SVC to your data. /// * `x` - _NxM_ matrix with _N_ observations and _M_ features in each observation. /// * `y` - class labels @@ -356,8 +349,8 @@ impl, K: Kernel> SVC { pub fn fit( x: &M, y: &M::RowVector, - parameters: SVCParameters, - ) -> Result, Failed> { + parameters: SVCParameters, + ) -> Result, Failed> { let (n, _) = x.shape(); if n != y.len() { @@ -434,14 +427,14 @@ impl, K: Kernel> SVC { let mut f = self.b; for i in 0..self.instances.len() { - f += self.w[i] * self.kernel.apply(&x, &self.instances[i]); + f += self.w[i] * crate::svm::apply(&self.kernel, &x, &self.instances[i]); } f } } -impl, K: Kernel> PartialEq for SVC { +impl> PartialEq for SVC { fn eq(&self, other: &Self) -> bool { if (self.b - other.b).abs() > T::epsilon() * T::two() || self.w.len() != other.w.len() @@ -465,8 +458,8 @@ impl, K: Kernel> PartialEq for SVC< } impl> SupportVector { - fn new>(i: usize, x: V, y: T, g: T, c: T, k: &K) -> SupportVector { - let k_v = k.apply(&x, &x); + fn new(i: usize, x: V, y: T, g: T, c: T, k: &Kernel) -> SupportVector { + let k_v = crate::svm::apply(k, &x, &x); let (cmin, cmax) = if y > T::zero() { (T::zero(), c) } else { @@ -484,8 +477,8 @@ impl> SupportVector { } } -impl<'a, T: RealNumber, M: Matrix, K: Kernel> Cache<'a, T, M, K> { - fn new(kernel: &'a K) -> Cache<'a, T, M, K> { +impl<'a, T: RealNumber, M: Matrix> Cache<'a, T, M> { + fn new(kernel: &'a Kernel) -> Cache<'a, T, M> { Cache { kernel, data: HashMap::new(), @@ -497,10 +490,10 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Cache<'a, T, M let idx_i = i.index; let idx_j = j.index; #[allow(clippy::or_fun_call)] - let entry = self - .data - .entry((idx_i, idx_j)) - .or_insert(self.kernel.apply(&i.x, &j.x)); + let entry = + self.data + .entry((idx_i, idx_j)) + .or_insert(crate::svm::apply(self.kernel, &i.x, &j.x)); *entry } @@ -513,13 +506,13 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Cache<'a, T, M } } -impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, T, M, K> { +impl<'a, T: RealNumber, M: Matrix> Optimizer<'a, T, M> { fn new( x: &'a M, y: &'a M::RowVector, - kernel: &'a K, - parameters: &'a SVCParameters, - ) -> Optimizer<'a, T, M, K> { + kernel: &'a Kernel, + parameters: &'a SVCParameters, + ) -> Optimizer<'a, T, M> { let (n, _) = x.shape(); Optimizer { @@ -575,7 +568,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, (support_vectors, w, b) } - fn initialize(&mut self, cache: &mut Cache<'_, T, M, K>) { + fn initialize(&mut self, cache: &mut Cache<'_, T, M>) { let (n, _) = self.x.shape(); let few = 5; let mut cp = 0; @@ -599,7 +592,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, } } - fn process(&mut self, i: usize, x: M::RowVector, y: T, cache: &mut Cache<'_, T, M, K>) -> bool { + fn process(&mut self, i: usize, x: M::RowVector, y: T, cache: &mut Cache<'_, T, M>) -> bool { for j in 0..self.sv.len() { if self.sv[j].index == i { return true; @@ -611,7 +604,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, let mut cache_values: Vec<((usize, usize), T)> = Vec::new(); for v in self.sv.iter() { - let k = self.kernel.apply(&v.x, &x); + let k = crate::svm::apply(self.kernel, &v.x, &x); cache_values.push(((i, v.index), k)); g -= v.alpha * k; } @@ -642,13 +635,13 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, true } - fn reprocess(&mut self, tol: T, cache: &mut Cache<'_, T, M, K>) -> bool { + fn reprocess(&mut self, tol: T, cache: &mut Cache<'_, T, M>) -> bool { let status = self.smo(None, None, tol, cache); self.clean(cache); status } - fn finish(&mut self, cache: &mut Cache<'_, T, M, K>) { + fn finish(&mut self, cache: &mut Cache<'_, T, M>) { let mut max_iter = self.sv.len(); while self.smo(None, None, self.parameters.tol, cache) && max_iter > 0 { @@ -683,7 +676,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, self.recalculate_minmax_grad = false } - fn clean(&mut self, cache: &mut Cache<'_, T, M, K>) { + fn clean(&mut self, cache: &mut Cache<'_, T, M>) { self.find_min_max_gradient(); let gmax = self.gmax; @@ -717,7 +710,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, &mut self, idx_1: Option, idx_2: Option, - cache: &mut Cache<'_, T, M, K>, + cache: &mut Cache<'_, T, M>, ) -> Option<(usize, usize, T)> { match (idx_1, idx_2) { (None, None) => { @@ -759,7 +752,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, idx_1, idx_2, k_v_12.unwrap_or_else(|| { - self.kernel.apply(&self.sv[idx_1].x, &self.sv[idx_2].x) + crate::svm::apply(self.kernel, &self.sv[idx_1].x, &self.sv[idx_2].x) }), ) }) @@ -797,7 +790,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, idx_1, idx_2, k_v_12.unwrap_or_else(|| { - self.kernel.apply(&self.sv[idx_1].x, &self.sv[idx_2].x) + crate::svm::apply(self.kernel, &self.sv[idx_1].x, &self.sv[idx_2].x) }), ) }) @@ -805,7 +798,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, (Some(idx_1), Some(idx_2)) => Some(( idx_1, idx_2, - self.kernel.apply(&self.sv[idx_1].x, &self.sv[idx_2].x), + crate::svm::apply(self.kernel, &self.sv[idx_1].x, &self.sv[idx_2].x), )), } } @@ -815,7 +808,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, idx_1: Option, idx_2: Option, tol: T, - cache: &mut Cache<'_, T, M, K>, + cache: &mut Cache<'_, T, M>, ) -> bool { match self.select_pair(idx_1, idx_2, cache) { Some((idx_1, idx_2, k_v_12)) => { @@ -854,7 +847,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, } } - fn update(&mut self, v1: usize, v2: usize, step: T, cache: &mut Cache<'_, T, M, K>) { + fn update(&mut self, v1: usize, v2: usize, step: T, cache: &mut Cache<'_, T, M>) { self.sv[v1].alpha -= step; self.sv[v2].alpha += step; @@ -879,19 +872,26 @@ mod tests { #[test] fn search_parameters() { - let parameters: SVCSearchParameters, LinearKernel> = - SVCSearchParameters { - epoch: vec![10, 100], - kernel: vec![LinearKernel {}], - ..Default::default() - }; + let linear = Kernel::Linear; + let rbf = Kernel::RBF { gamma: 0.001 }; + let parameters: SVCSearchParameters> = SVCSearchParameters { + epoch: vec![10, 100], + kernel: vec![linear, rbf], + ..Default::default() + }; let mut iter = parameters.into_iter(); let next = iter.next().unwrap(); assert_eq!(next.epoch, 10); - assert_eq!(next.kernel, LinearKernel {}); + assert_eq!(next.kernel, Kernel::Linear); + let next = iter.next().unwrap(); + assert_eq!(next.epoch, 100); + assert_eq!(next.kernel, Kernel::Linear); + let next = iter.next().unwrap(); + assert_eq!(next.epoch, 10); + assert_eq!(next.kernel, Kernel::RBF { gamma: 0.001 }); let next = iter.next().unwrap(); assert_eq!(next.epoch, 100); - assert_eq!(next.kernel, LinearKernel {}); + assert_eq!(next.kernel, Kernel::RBF { gamma: 0.001 }); assert!(iter.next().is_none()); } @@ -930,7 +930,7 @@ mod tests { &y, SVCParameters::default() .with_c(200.0) - .with_kernel(Kernels::linear()) + .with_kernel(Kernel::Linear) .with_seed(Some(100)), ) .and_then(|lr| lr.predict(&x)) @@ -965,7 +965,7 @@ mod tests { &y, SVCParameters::default() .with_c(200.0) - .with_kernel(Kernels::linear()), + .with_kernel(Kernel::Linear), ) .and_then(|lr| lr.decision_function(&x2)) .unwrap(); @@ -1019,7 +1019,7 @@ mod tests { &y, SVCParameters::default() .with_c(1.0) - .with_kernel(Kernels::rbf(0.7)), + .with_kernel(Kernel::RBF { gamma: 0.7 }), ) .and_then(|lr| lr.predict(&x)) .unwrap(); @@ -1066,7 +1066,7 @@ mod tests { let svc = SVC::fit(&x, &y, Default::default()).unwrap(); - let deserialized_svc: SVC, LinearKernel> = + let deserialized_svc: SVC> = serde_json::from_str(&serde_json::to_string(&svc).unwrap()).unwrap(); assert_eq!(svc, deserialized_svc); diff --git a/src/svm/svr.rs b/src/svm/svr.rs index 25326d4c..a6eeccbb 100644 --- a/src/svm/svr.rs +++ b/src/svm/svr.rs @@ -76,12 +76,12 @@ use crate::error::Failed; use crate::linalg::BaseVector; use crate::linalg::Matrix; use crate::math::num::RealNumber; -use crate::svm::{Kernel, Kernels, LinearKernel}; +use crate::svm::Kernel; #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Debug, Clone)] /// SVR Parameters -pub struct SVRParameters, K: Kernel> { +pub struct SVRParameters> { /// Epsilon in the epsilon-SVR model. pub eps: T, /// Regularization parameter. @@ -89,7 +89,7 @@ pub struct SVRParameters, K: Kernel /// Tolerance for stopping criterion. pub tol: T, /// The kernel function. - pub kernel: K, + pub kernel: Kernel, /// Unused parameter. m: PhantomData, } @@ -97,7 +97,7 @@ pub struct SVRParameters, K: Kernel /// SVR grid search parameters #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] #[derive(Debug, Clone)] -pub struct SVRSearchParameters, K: Kernel> { +pub struct SVRSearchParameters> { /// Epsilon in the epsilon-SVR model. pub eps: Vec, /// Regularization parameter. @@ -105,25 +105,23 @@ pub struct SVRSearchParameters, K: Kernel, /// The kernel function. - pub kernel: Vec, + pub kernel: Vec>, /// Unused parameter. m: PhantomData, } /// SVR grid search iterator -pub struct SVRSearchParametersIterator, K: Kernel> { - svr_search_parameters: SVRSearchParameters, +pub struct SVRSearchParametersIterator> { + svr_search_parameters: SVRSearchParameters, current_eps: usize, current_c: usize, current_tol: usize, current_kernel: usize, } -impl, K: Kernel> IntoIterator - for SVRSearchParameters -{ - type Item = SVRParameters; - type IntoIter = SVRSearchParametersIterator; +impl> IntoIterator for SVRSearchParameters { + type Item = SVRParameters; + type IntoIter = SVRSearchParametersIterator; fn into_iter(self) -> Self::IntoIter { SVRSearchParametersIterator { @@ -136,10 +134,8 @@ impl, K: Kernel> IntoIterator } } -impl, K: Kernel> Iterator - for SVRSearchParametersIterator -{ - type Item = SVRParameters; +impl> Iterator for SVRSearchParametersIterator { + type Item = SVRParameters; fn next(&mut self) -> Option { if self.current_eps == self.svr_search_parameters.eps.len() @@ -150,7 +146,7 @@ impl, K: Kernel> Iterator return None; } - let next = SVRParameters:: { + let next = SVRParameters:: { eps: self.svr_search_parameters.eps[self.current_eps], c: self.svr_search_parameters.c[self.current_c], tol: self.svr_search_parameters.tol[self.current_tol], @@ -183,9 +179,9 @@ impl, K: Kernel> Iterator } } -impl> Default for SVRSearchParameters { +impl> Default for SVRSearchParameters { fn default() -> Self { - let default_params: SVRParameters = SVRParameters::default(); + let default_params: SVRParameters = SVRParameters::default(); SVRSearchParameters { eps: vec![default_params.eps], @@ -202,14 +198,14 @@ impl> Default for SVRSearchParameters, K: Kernel> { - kernel: K, +pub struct SVR> { + kernel: Kernel, instances: Vec, w: Vec, b: T, @@ -226,7 +222,7 @@ struct SupportVector> { } /// Sequential Minimal Optimization algorithm -struct Optimizer<'a, T: RealNumber, M: Matrix, K: Kernel> { +struct Optimizer<'a, T: RealNumber, M: Matrix> { tol: T, c: T, svmin: usize, @@ -237,14 +233,14 @@ struct Optimizer<'a, T: RealNumber, M: Matrix, K: Kernel> { gmaxindex: usize, tau: T, sv: Vec>, - kernel: &'a K, + kernel: &'a Kernel, } struct Cache { data: Vec>>>, } -impl, K: Kernel> SVRParameters { +impl> SVRParameters { /// Epsilon in the epsilon-SVR model. pub fn with_eps(mut self, eps: T) -> Self { self.eps = eps; @@ -261,7 +257,7 @@ impl, K: Kernel> SVRParameters>(&self, kernel: KK) -> SVRParameters { + pub fn with_kernel(&self, kernel: Kernel) -> SVRParameters { SVRParameters { eps: self.eps, c: self.c, @@ -272,35 +268,33 @@ impl, K: Kernel> SVRParameters> Default for SVRParameters { +impl> Default for SVRParameters { fn default() -> Self { SVRParameters { eps: T::from_f64(0.1).unwrap(), c: T::one(), tol: T::from_f64(1e-3).unwrap(), - kernel: Kernels::linear(), + kernel: Kernel::Linear, m: PhantomData, } } } -impl, K: Kernel> - SupervisedEstimator> for SVR +impl> SupervisedEstimator> + for SVR { - fn fit(x: &M, y: &M::RowVector, parameters: SVRParameters) -> Result { + fn fit(x: &M, y: &M::RowVector, parameters: SVRParameters) -> Result { SVR::fit(x, y, parameters) } } -impl, K: Kernel> Predictor - for SVR -{ +impl> Predictor for SVR { fn predict(&self, x: &M) -> Result { self.predict(x) } } -impl, K: Kernel> SVR { +impl> SVR { /// Fits SVR to your data. /// * `x` - _NxM_ matrix with _N_ observations and _M_ features in each observation. /// * `y` - target values @@ -309,8 +303,8 @@ impl, K: Kernel> SVR { pub fn fit( x: &M, y: &M::RowVector, - parameters: SVRParameters, - ) -> Result, Failed> { + parameters: SVRParameters, + ) -> Result, Failed> { let (n, _) = x.shape(); if n != y.len() { @@ -349,14 +343,14 @@ impl, K: Kernel> SVR { let mut f = self.b; for i in 0..self.instances.len() { - f += self.w[i] * self.kernel.apply(&x, &self.instances[i]); + f += self.w[i] * crate::svm::apply(&self.kernel, &x, &self.instances[i]); } f } } -impl, K: Kernel> PartialEq for SVR { +impl> PartialEq for SVR { fn eq(&self, other: &Self) -> bool { if (self.b - other.b).abs() > T::epsilon() * T::two() || self.w.len() != other.w.len() @@ -380,8 +374,8 @@ impl, K: Kernel> PartialEq for SVR< } impl> SupportVector { - fn new>(i: usize, x: V, y: T, eps: T, k: &K) -> SupportVector { - let k_v = k.apply(&x, &x); + fn new(i: usize, x: V, y: T, eps: T, k: &Kernel) -> SupportVector { + let k_v = crate::svm::apply(k, &x, &x); SupportVector { index: i, x, @@ -392,13 +386,13 @@ impl> SupportVector { } } -impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, T, M, K> { +impl<'a, T: RealNumber, M: Matrix> Optimizer<'a, T, M> { fn new( x: &M, y: &M::RowVector, - kernel: &'a K, - parameters: &SVRParameters, - ) -> Optimizer<'a, T, M, K> { + kernel: &'a Kernel, + parameters: &SVRParameters, + ) -> Optimizer<'a, T, M> { let (n, _) = x.shape(); let mut support_vectors: Vec> = Vec::with_capacity(n); @@ -479,7 +473,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, let k1 = cache.get(self.sv[v1].index, || { self.sv .iter() - .map(|vi| self.kernel.apply(&self.sv[v1].x, &vi.x)) + .map(|vi| crate::svm::apply(self.kernel, &self.sv[v1].x, &vi.x)) .collect() }); @@ -526,7 +520,7 @@ impl<'a, T: RealNumber, M: Matrix, K: Kernel> Optimizer<'a, let k2 = cache.get(self.sv[v2].index, || { self.sv .iter() - .map(|vi| self.kernel.apply(&self.sv[v2].x, &vi.x)) + .map(|vi| crate::svm::apply(self.kernel, &self.sv[v2].x, &vi.x)) .collect() }); @@ -641,19 +635,18 @@ mod tests { #[test] fn search_parameters() { - let parameters: SVRSearchParameters, LinearKernel> = - SVRSearchParameters { - eps: vec![0., 1.], - kernel: vec![LinearKernel {}], - ..Default::default() - }; + let parameters: SVRSearchParameters> = SVRSearchParameters { + eps: vec![0., 1.], + kernel: vec![Kernel::Linear {}], + ..Default::default() + }; let mut iter = parameters.into_iter(); let next = iter.next().unwrap(); assert_eq!(next.eps, 0.); - assert_eq!(next.kernel, LinearKernel {}); + assert_eq!(next.kernel, Kernel::Linear {}); let next = iter.next().unwrap(); assert_eq!(next.eps, 1.); - assert_eq!(next.kernel, LinearKernel {}); + assert_eq!(next.kernel, Kernel::Linear {}); assert!(iter.next().is_none()); } @@ -721,7 +714,7 @@ mod tests { let svr = SVR::fit(&x, &y, Default::default()).unwrap(); - let deserialized_svr: SVR, LinearKernel> = + let deserialized_svr: SVR> = serde_json::from_str(&serde_json::to_string(&svr).unwrap()).unwrap(); assert_eq!(svr, deserialized_svr);