Gradient Descent

benches/gradient_descent.rs

@@ -0,0 +1,31 @@
+use criterion::{black_box, criterion_group, criterion_main, Criterion, BenchmarkId};
+use bfgs::settings::MinimizationAlg;
+
+// Global minimum: [0., 0.., ...]
+pub fn sphere(x: &Vec<f64>, _g: &Vec<f64>, f: &mut f64, d: i32) {
+    *f = 0.;
+    for i in 0..d as usize {
+        *f += x[i] * x[i];
+    }
+}
+
+fn bench_gradient_descent(c: &mut Criterion) {
+
+    let dims = vec![2, 6, 20, 60, 200];
+
+    let mut settings: bfgs::settings::Settings = Default::default();
+    settings.minimization = MinimizationAlg::GradientDescent;
+
+    let mut group = c.benchmark_group("Gradient Descent");
+
+    for d in dims {
+        group.bench_with_input(BenchmarkId::from_parameter(d), &d, |b, &d| {
+            b.iter(|| bfgs::get_minimum(&sphere, black_box(&mut vec![1.7; d]), &settings));
+        });
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, bench_gradient_descent);
+criterion_main!(benches);

src/gradient_descent.rs

@@ -0,0 +1,66 @@
+use crate::Settings;
+
+pub(crate) fn gradient_descent<Ef, Gf>(fn_function: &Ef, fn_gradient: &Gf, x: &mut Vec<f64>, settings: &Settings)
+                                                  -> Option<f64>
+    where
+        Ef: Fn(&Vec<f64>, &Vec<f64>, &mut f64, i32),
+        Gf: Fn(&Vec<f64>, &mut Vec<f64>, &f64, i32)
+{
+    // Settings
+    let iter_max = settings.iter_max;
+    let verbose = settings.verbose;
+
+    // Get the dimension
+    let d = x.len() as i32;
+
+    // Function update evaluations
+    let mut eval: usize = 0;
+
+    // Energy definition
+    let mut f: f64 = 0.;
+
+    // Gradient definition
+    let mut g: Vec<f64> = vec![0.; d as usize];
+
+    // Update energy and gradient
+    fn_function(x, &g, &mut f, d);
+    fn_gradient(x, &mut g, &f, d);
+    eval += 1;
+
+    // Iteration counter
+    let mut iter: usize = 0;
+
+    // Learning rate
+    let alpha = 0.1;
+
+    // Iteration
+    while iter < iter_max {
+        // Update the iteration counter
+        iter += 1;
+
+        // Update the position
+        unsafe{ cblas::daxpy(d, - alpha, &*g, 1, &mut *x, 1);}
+
+        // Update energy and gradient
+        fn_function(x, &g, &mut f, d);
+        fn_gradient(x, &mut g, &f, d);
+        eval += 1;
+
+        let g_norm = unsafe { cblas::dnrm2(d, &*g, 1) };
+        if g_norm < settings.gtol {
+            if verbose {
+                println!("Exit condition reached:");
+                println!(" - Iterations: {} ", iter);
+                println!(" - Function evaluations: {}", eval);
+            }
+            return Some(f);
+        }
+    }
+
+    if verbose {
+        println!("Maximum number of iterations reached");
+    }
+    return None;
+}
+
+

src/lib.rs

@@ -10,6 +10,7 @@ mod lbfgs;
 mod line_search;
 mod exit_condition;
 mod log;
+pub mod gradient_descent;
 
 use crate::settings::Settings;
 use crate::settings::MinimizationAlg;
@@ -124,6 +125,10 @@ fn do_bfgs<Function, Gradient>(fn_function: &Function, fn_gradient: &Gradient, x
 
     // Handle different minimization methods
     match settings.minimization {
+        MinimizationAlg::GradientDescent => {
+            use crate::gradient_descent::gradient_descent;
+            gradient_descent(fn_function, fn_gradient, x, settings)
+        }
         MinimizationAlg::Bfgs => {
             use crate::bfgs::bfgs;
             bfgs(fn_function, fn_gradient, x, settings)

src/settings.rs

@@ -1,5 +1,6 @@
 /// Enumerator for minimization algorithm
 pub enum MinimizationAlg {
+    GradientDescent,
     Bfgs,
     Lbfgs,
     /// Use L-BFGS algorithm if BFGS fails

tests/test_functions.rs

@@ -1,4 +1,5 @@
-// allow(unused) flag to avoid compiler warning
+// Minimization functions from https://www.sfu.ca/~ssurjano/optimization.html
+// Note: allow(unused) flag to avoid compiler warning
 
 #[allow(unused)]
 // Global minimum: [0., 0.., ...]

tests/test_gradient_descent.rs

@@ -0,0 +1,27 @@
+use rand::{Rng, thread_rng};
+use bfgs::settings::MinimizationAlg;
+
+mod test_functions;
+mod test_utils;
+
+#[test]
+fn test_sphere_function() {
+    use bfgs;
+
+    // Create settings with default parameters
+    let mut settings: bfgs::settings::Settings = Default::default();
+    // Select the minimization algorithm
+    settings.minimization = MinimizationAlg::GradientDescent;
+    settings.iter_max = 1000;
+    settings.verbose = true;
+
+    let dims = vec![2, 6, 20, 100, 1000];
+
+    for d in dims {
+        let mut x = vec![(); d].into_iter().map(|_| thread_rng().gen_range(-10.0..10.0)).collect();
+        let result = bfgs::get_minimum(&test_functions::sphere, &mut x, &settings);
+        assert_ne!(result, None, "Result not found");
+        let cmp = vec![0.; d];
+        test_utils::check_result(x, cmp);
+    }
+}