diff --git a/rust/Cargo.lock b/rust/Cargo.lock index fe11398ed..cd7b35112 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -54,9 +54,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6748e8def348ed4d14996fa801f4122cd763fff530258cdc03f64b25f89d3a5a" +checksum = "0c378d78423fdad8089616f827526ee33c19f2fddbd5de1629152c9593ba4783" dependencies = [ "memchr", ] @@ -84,24 +84,23 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" [[package]] name = "anstyle-parse" @@ -123,9 +122,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -171,9 +170,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "base64ct" @@ -181,6 +180,34 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" +[[package]] +name = "bincode" +version = "1.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" +dependencies = [ + "serde", +] + +[[package]] +name = "bincode" +version = "2.0.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f11ea1a0346b94ef188834a65c068a03aec181c94896d481d7a0a40d85b0ce95" +dependencies = [ + "bincode_derive", + "serde", +] + +[[package]] +name = "bincode_derive" +version = "2.0.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e30759b3b99a1b802a7a3aa21c85c3ded5c28e1c83170d82d70f08bbf7f3e4c" +dependencies = [ + "virtue", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -278,14 +305,14 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.26" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "95ed24df0632f708f5f6d8082675bef2596f7084dee3dd55f632290bf35bfe0f" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "winapi", + "windows-targets", ] [[package]] @@ -327,20 +354,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.23" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03aef18ddf7d879c15ce20f04826ef8418101c7e528014c3eeea13321047dca3" +checksum = "6a13b88d2c62ff462f88e4a121f17a82c1af05693a2f192b5c38d14de73c19f6" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.3.23" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ce6fffb678c9b80a70b6b6de0aad31df727623a70fd9a842c30cd573e2fa98" +checksum = "2bb9faaa7c2ef94b2743a21f5a29e6f0010dff4caa69ac8e9d6cf8b6fa74da08" dependencies = [ "anstream", "anstyle", @@ -350,9 +376,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" dependencies = [ "heck", "proc-macro2", @@ -362,9 +388,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" [[package]] name = "cmac" @@ -583,9 +609,9 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b30f669a7961ef1631673d2766cc92f52d64f7ef354d4fe0ddfd30ed52f0f4f" +checksum = "136526188508e25c6fef639d7927dfb3e0e3084488bf202267829cf7fc23dbdd" dependencies = [ "errno-dragonfly", "libc", @@ -969,6 +995,20 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "infisto" +version = "0.1.0" +dependencies = [ + "bincode 2.0.0-rc.3", + "chacha20", + "criterion", + "pbkdf2", + "rand", + "serde", + "sha2", + "uuid", +] + [[package]] name = "inout" version = "0.1.3" @@ -1125,9 +1165,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "5486aed0026218e61b8a01d5fbd5a0a134649abb71a0e53b7bc088529dced86e" [[package]] name = "memoffset" @@ -1162,6 +1202,7 @@ dependencies = [ name = "models" version = "0.1.0" dependencies = [ + "bincode 2.0.0-rc.3", "serde", "serde_json", ] @@ -1425,6 +1466,7 @@ dependencies = [ "async-trait", "async-traits", "base64", + "bincode 1.3.3", "chacha20", "clap", "feed", @@ -1433,6 +1475,7 @@ dependencies = [ "generic-array", "hyper", "hyper-rustls", + "infisto", "models", "nasl-interpreter", "osp", @@ -1537,9 +1580,9 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "pin-project-lite" -version = "0.2.12" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12cc1b0bf1727a77a54b6654e7b5f1af8604923edc8b81885f8ec92f9e3f0a05" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -1803,14 +1846,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.3.7", + "regex-syntax 0.7.5", ] [[package]] @@ -1824,13 +1867,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.7.5", ] [[package]] @@ -1841,9 +1884,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "ring" @@ -1877,12 +1920,12 @@ checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "ed6248e1caa625eb708e266e06159f135e8c26f2bb7ceb72dc4b2766d0340964" dependencies = [ "bitflags 2.4.0", - "errno 0.3.2", + "errno 0.3.3", "libc", "linux-raw-sys", "windows-sys 0.48.0", @@ -1890,9 +1933,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", @@ -2009,18 +2052,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.185" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be9b6f69f1dfd54c3b568ffa45c310d6973a5e5148fd40cf515acaf38cf5bc31" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.185" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc59dfdcbad1437773485e0367fea4b090a2e0a16d9ffc46af47764536a298ec" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", @@ -2209,9 +2252,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.27" +version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb39ee79a6d8de55f48f2293a830e040392f1c5f16e336bdd1788cd0aadce07" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" dependencies = [ "deranged", "serde", @@ -2227,9 +2270,9 @@ checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" [[package]] name = "time-macros" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "733d258752e9303d392b94b75230d07b0b9c489350c69b851fc6c065fde3e8f9" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" dependencies = [ "time-core", ] @@ -2466,9 +2509,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -2516,6 +2559,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "virtue" +version = "0.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dcc60c0624df774c82a0ef104151231d37da4962957d691c011c852b2473314" + [[package]] name = "walkdir" version = "2.3.3" @@ -2756,9 +2805,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" dependencies = [ "memchr", ] diff --git a/rust/Cargo.toml b/rust/Cargo.toml index d76034b8c..95789f473 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,4 +24,11 @@ members = [ "osp", "openvasd", "scanconfig", + "infisto", ] + +[workspace.package] +version = "0.1.0" +edition = "2021" +license = "GPL-2.0-or-later" + diff --git a/rust/examples/openvasd/config.example.toml b/rust/examples/openvasd/config.example.toml index ac0545483..4a8df6d76 100644 --- a/rust/examples/openvasd/config.example.toml +++ b/rust/examples/openvasd/config.example.toml @@ -46,3 +46,16 @@ address = "127.0.0.1:3000" [log] # level of the log messages: TRACE > DEBUG > INFO > WARN > ERROR level = "INFO" + +[storage] +# can be either fs (file system) or inmemory (in memory). +# If it is set to fs is highly recommended to set `STORAGE_KEY` in the env variable. +# WARNING: if the type is set to fs and no encryption key is set then the data is stored unencrypted. +#type = "fs" +type = "inmemory" + +[storage.fs] +# Sets the storage root directory if the storage.type is set to `fs`. +path = "/var/lib/openvasd/storage" +# Sets the key used to ecrypt the storage data. It is recommended to set it via the `STORAGE_KEY` environment variable. +#key = "changeme" diff --git a/rust/infisto/Cargo.toml b/rust/infisto/Cargo.toml new file mode 100644 index 000000000..1a8422f25 --- /dev/null +++ b/rust/infisto/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "infisto" +description = "A library to store and retrieve serializeable vec data in a file that can be accessed by an index." +version.workspace = true +license.workspace = true +edition.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +bincode = { version = "2.0.0-rc.3", features = ["serde"] } +serde = { version = "1", features = ["derive"] } +rand = "0" +chacha20 = "0" +pbkdf2 = { version = "0", features = ["password-hash"] } +sha2 = "0" + +[dev-dependencies] +criterion = "0" +uuid = "1.4.1" + +[[bench]] +name = "comparison" +harness = false diff --git a/rust/infisto/README.md b/rust/infisto/README.md new file mode 100644 index 000000000..c29b9888d --- /dev/null +++ b/rust/infisto/README.md @@ -0,0 +1,60 @@ +# INdexed FIle STOrage + +Is a library to store data on to disk and fetch elements from that rather than loading the whole file. + +## CachedIndexFileStorer + +Caches the last files idx files into memory. + +``` +use infisto::base::IndexedByteStorage; +let base = "/tmp/openvasd/storage"; +let name = "readme_cached"; +let mut store = infisto::base::CachedIndexFileStorer::init(base).unwrap(); +store.put(name, "Hello World".as_bytes()).unwrap(); +store.append_all(name, &["a".as_bytes(), "b".as_bytes()]).unwrap(); +let data: Vec> = store.by_range(name, infisto::base::Range::Between(1, 3)).unwrap(); +assert_eq!(data.len(), 2); +assert_eq!(&data[0], "a".as_bytes()); +assert_eq!(&data[1], "b".as_bytes()); +store.remove(name).unwrap(); +``` + +## ChaCha20IndexFileStorer + +Encryptes the given data with chacha20 before storing it. + +``` +use infisto::base::IndexedByteStorage; +let base = "/tmp/openvasd/storage"; +let name = "readme_crypt"; +let key = "changeme"; +let store = infisto::base::CachedIndexFileStorer::init(base).unwrap(); +let mut store = infisto::crypto::ChaCha20IndexFileStorer::new(store, key); +store.put(name, "Hello World".as_bytes()).unwrap(); +store.append_all(name, &["a".as_bytes(), "b".as_bytes()]).unwrap(); +let data: Vec> = store.by_range(name, infisto::base::Range::Between(1, 3)).unwrap(); +assert_eq!(data.len(), 2); +assert_eq!(&data[0], "a".as_bytes()); +assert_eq!(&data[1], "b".as_bytes()); +store.remove(name).unwrap(); +``` + +## IndexedByteStorageIterator + +Instead of loading all elements at once it allows to fetch single elements when required. + +``` +use infisto::base::IndexedByteStorage; +let base = "/tmp/openvasd/storage"; +let name = "readme_iter"; +let key = "changeme"; +let mut store = infisto::base::CachedIndexFileStorer::init(base).unwrap(); +store.put(name, "Hello World".as_bytes()).unwrap(); +let mut iter: infisto::base::IndexedByteStorageIterator<_, Vec> = + infisto::base::IndexedByteStorageIterator::new(name, store.clone()).unwrap(); +assert_eq!(iter.next(), Some(Ok("Hello World".as_bytes().to_vec()))); +assert_eq!(iter.next(), None); +store.remove(name).unwrap(); +``` + diff --git a/rust/infisto/benches/comparison.rs b/rust/infisto/benches/comparison.rs new file mode 100644 index 000000000..c8b3e071e --- /dev/null +++ b/rust/infisto/benches/comparison.rs @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: 2023 Greenbone AG +// +// SPDX-License-Identifier: GPL-2.0-or-later + +use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion}; +use infisto::base::{CachedIndexFileStorer, IndexedByteStorage}; +use rand::distributions::Alphanumeric; +use rand::Rng; + +const BASE: &str = "/tmp/openvasd"; + +pub fn reading(c: &mut Criterion) { + let amount = 1000000; + fn random_data() -> Vec { + use rand::RngCore; + let mut rng = rand::thread_rng(); + let mut data = vec![0; 1024]; + rng.fill_bytes(&mut data); + data + } + let mut data = Vec::with_capacity(amount); + for _ in 0..amount { + data.push(random_data()); + } + + let fname = |pre: &str| { + format!( + "{}{}", + pre, + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(7) + .map(char::from) + .collect::() + ) + }; + // prepare data + let cached_name = fname("cached"); + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + store.append_all(&cached_name, &data).unwrap(); + let uncached_name = fname("uncached"); + let mut store = infisto::base::IndexedFileStorer::init(BASE).unwrap(); + store.append_all(&uncached_name, &data).unwrap(); + // to be useable in openvasd we must create Stream interface to allow polling + // on ranges otherwise the use has to wait until the whole file is read + let crypto_name = fname("crypto"); + let mut store = infisto::crypto::ChaCha20IndexFileStorer::new( + CachedIndexFileStorer::init(BASE).unwrap(), + infisto::crypto::Key::default(), + ); + store.append_all(&crypto_name, &data).unwrap(); + let mut group = c.benchmark_group("reading"); + group.sample_size(10); + let store = CachedIndexFileStorer::init(BASE).unwrap(); + group.bench_with_input("cached", &cached_name, move |b, key| { + b.iter(|| { + store + .by_range::>(black_box(key), infisto::base::Range::All) + .unwrap(); + }) + }); + let store = infisto::base::IndexedFileStorer::init(BASE).unwrap(); + group.bench_with_input("uncached", &uncached_name, move |b, key| { + b.iter(|| { + store + .by_range::>(black_box(key), infisto::base::Range::All) + .unwrap(); + }) + }); + let store = infisto::crypto::ChaCha20IndexFileStorer::new( + CachedIndexFileStorer::init(BASE).unwrap(), + infisto::crypto::Key::default(), + ); + group.bench_with_input("crypto", &crypto_name, move |b, key| { + b.iter(|| { + store + .by_range::>(black_box(key), infisto::base::Range::All) + .unwrap(); + }) + }); + + group.finish(); + let mut clean_up_store = CachedIndexFileStorer::init(BASE).unwrap(); + clean_up_store.remove(&crypto_name).unwrap(); + clean_up_store.remove(&uncached_name).unwrap(); + clean_up_store.remove(&cached_name).unwrap(); +} +pub fn storing(c: &mut Criterion) { + let amount = 100000; + fn random_data() -> Vec { + use rand::RngCore; + let mut rng = rand::thread_rng(); + let mut data = vec![0; 1024]; + rng.fill_bytes(&mut data); + data + } + let mut data = Vec::with_capacity(amount); + for _ in 0..amount { + data.push(random_data()); + } + + let fname = |pre: &str| { + format!( + "{}{}", + pre, + rand::thread_rng() + .sample_iter(&Alphanumeric) + .take(7) + .map(char::from) + .collect::() + ) + }; + let mut group = c.benchmark_group("storing"); + group.sample_size(10); + let cached_name = fname("cached"); + group.bench_with_input( + BenchmarkId::new("cached", "1million times 1MB"), + &(&cached_name, &data), + move |b, (key, data)| { + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + b.iter(|| { + store.append_all(black_box(key), black_box(data)).unwrap(); + }) + }, + ); + let uncached_name = fname("uncached"); + group.bench_with_input( + "uncached", + &(&uncached_name, &data), + move |b, (key, data)| { + let mut store = infisto::base::IndexedFileStorer::init(BASE).unwrap(); + b.iter(|| { + store.append_all(black_box(key), black_box(data)).unwrap(); + }) + }, + ); + let crypto_name = fname("crypto"); + group.bench_with_input("crypto", &(&crypto_name, &data), move |b, (key, data)| { + let mut store = infisto::crypto::ChaCha20IndexFileStorer::new( + CachedIndexFileStorer::init(BASE).unwrap(), + infisto::crypto::Key::default(), + ); + b.iter(|| { + store.append_all(black_box(key), black_box(data)).unwrap(); + }) + }); + group.finish(); + let mut clean_up_store = CachedIndexFileStorer::init(BASE).unwrap(); + clean_up_store.remove(&crypto_name).unwrap(); + clean_up_store.remove(&uncached_name).unwrap(); + clean_up_store.remove(&cached_name).unwrap(); + reading(c); +} + +criterion_group!(benches, storing); + +criterion_main!(benches); diff --git a/rust/infisto/src/base.rs b/rust/infisto/src/base.rs new file mode 100644 index 000000000..4bb314635 --- /dev/null +++ b/rust/infisto/src/base.rs @@ -0,0 +1,791 @@ +//! The base module contains the basic building blocks for the file store. + +use std::{ + fs, + io::{Read, Seek, Write}, + marker::PhantomData, + path::{Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; +#[derive(Serialize, Deserialize, Clone, Debug)] +/// The index is used to store the start and end position of a data set in the file. +pub struct Index { + /// The start position of the data set in the file. + pub start: usize, + /// The end position of the data set in the file. + pub end: usize, +} + +/// The store is used to store and retrieve data from the file system. +/// +/// It is meant to be a building block for other data stores to store encrypted data and cache the +/// index in memory. +/// +/// Warning: When working with the same file from multiple threads, the store is not thread safe. +#[derive(Clone, Debug)] +pub struct IndexedFileStorer { + /// The base path where the idx and dat files are stored. + base: PathBuf, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +/// The error type for the store. +pub enum Error { + /// The base directory could not be created. + CreateBaseDir(std::io::ErrorKind), + /// The file could not be opened. + FileOpen(std::io::ErrorKind), + /// The file could not be written to. + Write(std::io::ErrorKind), + /// The file could not be read from. + Read(std::io::ErrorKind), + /// The file could not be removed. + Remove(std::io::ErrorKind), + /// The file could not be sought. + Seek(std::io::ErrorKind), + /// The index could not be serialized. + Serialize, +} +/// Is a storage that stores bytes by using a key. +pub trait IndexedByteStorage { + /// Overrides, creates a file with the given data. + /// + /// It creates a idx file for byte ranges of that element and creates data file containing the + /// given data. + fn put(&mut self, key: &str, data: T) -> Result<(), Error> + where + T: AsRef<[u8]>; + /// Appends the given data to the file found via key. + /// + /// It enhances the index with the byte range of the given data and appends the given data to + /// the file found via key. + fn append(&mut self, key: &str, data: T) -> Result<(), Error> + where + T: AsRef<[u8]>, + { + self.append_all(key, &[data]) + } + /// Appends the given data to the file found via key. + /// + /// It enhances the index with the byte range of given data and appends the given data to the + /// file found via key. + fn append_all(&mut self, key: &str, data: &[T]) -> Result<(), Error> + where + T: AsRef<[u8]>; + /// Removes idx and data file of given key. + fn remove(&mut self, key: &str) -> Result<(), Error>; + /// Returns the data for the given key and range. + fn by_range(&self, key: &str, range: Range) -> Result, Error> + where + T: TryFrom>, + { + let indices = self.indices(key)?; + let filtered_indices = range.filter(&indices); + self.by_indices(key, filtered_indices) + } + /// Returns the data for the given key and index. + fn by_index(&self, key: &str, index: &Index) -> Result, Error> + where + T: TryFrom>, + { + let data = self.by_indices(key, &[index.clone()])?; + Ok(data.into_iter().next()) + } + + /// Returns the data for given key and all indices. + fn by_indices(&self, key: &str, indices: &[Index]) -> Result, Error> + where + T: TryFrom>; + + /// Returns all the indices of the data for the given key. + fn indices(&self, key: &str) -> Result, Error>; +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let msg = match self { + Error::CreateBaseDir(e) => format!("Could not create base directory: {}", e), + Error::FileOpen(e) => format!("Could not open file: {}", e), + Error::Write(e) => format!("Could not write to file: {}", e), + Error::Read(e) => format!("Could not read from file: {}", e), + Error::Remove(e) => format!("Could not remove file: {}", e), + Error::Seek(e) => format!("Could not seek in file: {}", e), + Error::Serialize => "Could not serialize index".to_string(), + }; + write!(f, "{}", msg) + } +} + +impl std::error::Error for Error {} + +impl IndexedFileStorer { + /// Verifies if the base path exists and creates it if not before returning the store. + pub fn init

(path: P) -> Result + where + P: AsRef, + { + let path = path.as_ref(); + if !path.exists() { + std::fs::create_dir_all(path) + .map_err(|e| e.kind()) + .map_err(Error::CreateBaseDir)?; + } + Ok(IndexedFileStorer { + base: PathBuf::from(path), + }) + } + + /// Creates a new index element and stored the element in the file. + pub fn create(&self, id: &str, element: T) -> Result, Error> + where + T: AsRef<[u8]>, + { + let element = element.as_ref(); + let index = vec![Index { + start: 0, + end: element.len(), + }]; + self.store_index(&index, id)?; + let fn_name = format!("{}.dat", id); + let path = Path::new(&self.base).join(fn_name); + let mut file = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(path) + .map_err(|e| e.kind()) + .map_err(Error::FileOpen)?; + file.write_all(element) + .map_err(|e| e.kind()) + .map_err(Error::Write)?; + + Ok(index) + } + + fn store_index(&self, index: &[Index], id: &str) -> Result<(), Error> { + let fn_name = format!("{}.idx", id); + let config = bincode::config::standard(); + let to_store = + bincode::serde::encode_to_vec(index, config).map_err(|_| Error::Serialize)?; + + let path = Path::new(&self.base).join(fn_name); + let mut file = std::fs::OpenOptions::new() + .truncate(true) + .create(true) + .write(true) + .open(path) + .map_err(|e| e.kind()) + .map_err(Error::FileOpen)?; + file.write_all(&to_store) + .map_err(|e| e.kind()) + .map_err(Error::Write)?; + Ok(()) + } + + /// Gets the data from the file by using the given index. + pub fn data_by_index(&self, key: &str, idx: &Index) -> Result, Error> { + let fn_name = format!("{}.dat", key); + let path = Path::new(&self.base).join(fn_name); + let mut file = std::fs::OpenOptions::new() + .read(true) + .open(path) + .map_err(|e| e.kind()) + .map_err(Error::FileOpen)?; + let mut buffer = vec![0; idx.end - idx.start]; + file.seek(std::io::SeekFrom::Start(idx.start as u64)) + .map_err(|e| e.kind()) + .map_err(Error::Seek)?; + file.read_exact(&mut buffer) + .map_err(|e| e.kind()) + .map_err(Error::Read)?; + Ok(buffer) + } + + /// Load the index from the file. + /// + /// This should be rarely used as the index is usually returned when storing data. + /// The caller should rather cache the index. + pub fn load_index(&self, key: &str) -> Result, Error> { + let fn_name = format!("{}.idx", key); + let path = Path::new(&self.base).join(fn_name); + let mut file = std::fs::OpenOptions::new() + .read(true) + .open(path) + .map_err(|e| e.kind()) + .map_err(Error::FileOpen)?; + let mut buffer = vec![]; + file.read_to_end(&mut buffer) + .map_err(|e| e.kind()) + .map_err(Error::Read)?; + + let config = bincode::config::standard(); + let (index, _) = + bincode::serde::decode_from_slice(&buffer, config).map_err(|_| Error::Serialize)?; + Ok(index) + } + + /// Appends the given data to the file and enlarges the index. + pub fn append(&self, key: &str, index: &[Index], data: T) -> Result, Error> + where + T: AsRef<[u8]>, + { + self.append_all_index(key, index, &[data]) + } + + /// Appends all given data sets to the file and enlarges the index. + pub fn append_all_index( + &self, + key: &str, + index: &[Index], + data: &[T], + ) -> Result, Error> + where + T: AsRef<[u8]>, + { + let fn_name = format!("{}.dat", key); + let path = Path::new(&self.base).join(fn_name); + let mut file = std::fs::OpenOptions::new() + .write(true) + .append(true) + .open(path) + .map_err(|e| e.kind()) + .map_err(Error::FileOpen)?; + let mut index = index.to_vec(); + index.reserve(data.len()); + let mut start = index.last().map(|e| e.end).unwrap_or(0); + for d in data { + let b = d.as_ref(); + file.write_all(b) + .map_err(|e| e.kind()) + .map_err(Error::Write)?; + let end = start + b.len(); + let idx = Index { start, end }; + index.push(idx); + start = end; + } + self.store_index(&index, key)?; + Ok(index) + } + + /// Removes dat and idx files from the file system. + pub fn clean(&self, key: &str) -> Result<(), Error> { + let dat_fn = format!("{}.dat", key); + let dat_path = Path::new(&self.base).join(dat_fn); + fs::remove_file(dat_path) + .map_err(|e| e.kind()) + .map_err(Error::Remove)?; + let idx_fn = format!("{}.idx", key); + let idx_path = Path::new(&self.base).join(idx_fn); + fs::remove_file(idx_path) + .map_err(|e| e.kind()) + .map_err(Error::Remove)?; + Ok(()) + } +} + +impl IndexedByteStorage for IndexedFileStorer { + fn put(&mut self, key: &str, data: T) -> Result<(), Error> + where + T: AsRef<[u8]>, + { + self.create(key, data).map(|_| ()) + } + + fn append_all(&mut self, key: &str, data: &[T]) -> Result<(), Error> + where + T: AsRef<[u8]>, + { + match self.load_index(key) { + Ok(i) => self.append_all_index(key, &i, data).map(|_| ()), + Err(Error::FileOpen(ioe)) => match ioe { + std::io::ErrorKind::NotFound if !data.is_empty() => { + let initial_index = self.create(key, &data[0])?; + + self.append_all_index(key, &initial_index, &data[1..]) + .map(|_| ()) + } + _ => Err(Error::FileOpen(ioe)), + }, + Err(e) => Err(e), + } + } + + fn remove(&mut self, key: &str) -> Result<(), Error> { + self.clean(key) + } + + fn indices(&self, key: &str) -> Result, Error> { + self.load_index(key) + } + + fn by_indices(&self, key: &str, indices: &[Index]) -> Result, Error> + where + T: TryFrom>, + { + let start = indices.first().map(|e| e.start).unwrap_or(0); + let end = indices.last().map(|e| e.end).unwrap_or(0); + let index = Index { start, end }; + let data = self.data_by_index(key, &index)?; + let mut result = Vec::new(); + for i in indices { + // on an limiting range (e.g. from or between) the data does not contain + // all previous data, so we need to subtract the start of the first index + // to get the correcte byte range. + match data[(i.start - start)..(i.end - start)].to_vec().try_into() { + Ok(d) => result.push(d), + Err(_) => return Err(Error::Serialize), + } + } + Ok(result) + } +} + +/// Is an indexed file storage that caches the index of the last 5 files. +#[derive(Clone)] +pub struct CachedIndexFileStorer { + base: IndexedFileStorer, + cache: [Option<(String, Vec)>; 5], +} + +#[derive(Debug)] +/// Range to define which indices to load. +pub enum Range { + /// Returns all indices + All, + /// Returns indices from the given start + From(usize), + /// Returns indices until the given end + Until(usize), + /// Returns indices between the given start and end + Between(usize, usize), + /// Returns the bytes definition of the given index + /// + /// Starts at zero. + Single(usize), +} + +impl Range { + /// Filters the given index by the range. + pub fn filter<'a>(&'a self, fi: &'a [Index]) -> &'a [Index] { + match self { + Range::All => fi, + Range::From(i) => { + let i = *i; + if i >= fi.len() { + return &[]; + } + &fi[i..] + } + Range::Until(i) => { + let i = if *i > fi.len() { fi.len() } else { *i }; + &fi[..i] + } + Range::Between(s, e) => { + let s = *s; + let e = *e; + if s >= fi.len() { + return &[]; + } + let e = if e > fi.len() { fi.len() } else { e }; + &fi[s..e] + } + Range::Single(i) => { + let i = *i; + if i >= fi.len() { + return &[]; + } + &fi[i..=i] + } + } + } +} + +impl CachedIndexFileStorer { + /// Initializes the storage. + pub fn init(base: &str) -> Result { + let base = IndexedFileStorer::init(base)?; + let cache = [None, None, None, None, None]; + Ok(Self { base, cache }) + } + fn find_index(&self, key: &str) -> Option<(usize, &Vec)> { + for i in 0..self.cache.len() { + if let Some((k, v)) = &self.cache[i] { + if k == key { + return Some((i, v)); + } + } + } + None + } +} + +impl IndexedByteStorage for CachedIndexFileStorer { + /// Overwrites the data for the given key. + fn put(&mut self, key: &str, data: T) -> Result<(), Error> + where + T: AsRef<[u8]>, + { + // check if key is already in cache + let index = self.find_index(key).map(|(i, _)| i); + if let Some(i) = index { + self.cache.swap(0, i); + } else { + // remove oldest entry by overwriting it later + for i in (1..self.cache.len()).rev() { + self.cache.swap(i - 1, i); + } + } + let result = self.base.create(key, data)?; + self.cache[0] = Some((key.to_string(), result)); + Ok(()) + } + + /// Append the given data to the file and enlarges the index. + fn append(&mut self, key: &str, data: T) -> Result<(), Error> + where + T: AsRef<[u8]>, + { + self.append_all(key, &[data]) + } + + /// Append all given data sets to the file and enlarges the index. + fn append_all(&mut self, key: &str, data: &[T]) -> Result<(), Error> + where + T: AsRef<[u8]>, + { + let (ci, result) = if let Some((ci, fi)) = self.find_index(key) { + (ci, self.base.append_all_index(key, fi, data)?) + } else { + match self.base.load_index(key) { + Ok(i) => ( + self.cache.len() - 1, + self.base.append_all_index(key, &i, data)?, + ), + Err(Error::FileOpen(ioe)) => match ioe { + std::io::ErrorKind::NotFound if !data.is_empty() => { + let initial_index = self.base.create(key, &data[0])?; + + let end_index = + self.base + .append_all_index(key, &initial_index, &data[1..])?; + (self.cache.len() - 1, end_index) + } + _ => return Err(Error::FileOpen(ioe)), + }, + Err(e) => return Err(e), + } + }; + + for i in (1..=ci).rev() { + self.cache.swap(i - 1, i); + } + self.cache[0] = Some((key.to_string(), result)); + Ok(()) + } + + /// Removes dat and idx files from the file system. + fn remove(&mut self, key: &str) -> Result<(), Error> { + self.base.clean(key)?; + for i in 0..self.cache.len() { + if let Some((k, _)) = &self.cache[i] { + if k == key { + self.cache[i] = None; + } + } + } + Ok(()) + } + + fn indices(&self, key: &str) -> Result, Error> { + if let Some((_, fi)) = self.find_index(key) { + Ok(fi.clone()) + } else { + self.base.load_index(key) + } + } + + fn by_indices(&self, key: &str, indices: &[Index]) -> Result, Error> + where + T: TryFrom>, + { + self.base.by_indices(key, indices) + } +} + +/// Iterator over the data of a file. +pub struct IndexedByteStorageIterator { + current: usize, + indices: Vec, + storage: S, + key: String, + _phantom: PhantomData, +} + +impl IndexedByteStorageIterator +where + S: IndexedByteStorage, +{ + fn load_indices(key: &str, storage: &S) -> Result, Error> { + match storage.indices(key) { + Ok(i) => Ok(i), + Err(Error::FileOpen(ioe)) => match ioe { + std::io::ErrorKind::NotFound => Ok(vec![]), + _ => Err(Error::FileOpen(ioe)), + }, + Err(e) => Err(e), + } + } + + /// Creates a new instance for all indices of given key. + pub fn new(key: &str, storage: S) -> Result { + let indices = Self::load_indices(key, &storage)?; + Ok(Self { + current: 0, + indices, + storage, + key: key.to_string(), + _phantom: PhantomData, + }) + } + + /// Creates a new instance for indices found by range of given key. + pub fn by_range(key: &str, storage: S, range: Range) -> Result { + let indices = Self::load_indices(key, &storage)?; + let indices: Vec = range.filter(&indices).to_vec(); + Ok(Self { + current: 0, + indices, + storage, + key: key.to_string(), + _phantom: PhantomData, + }) + } +} + +impl Iterator for IndexedByteStorageIterator +where + S: IndexedByteStorage, + T: TryFrom>, +{ + type Item = Result; + + fn next(&mut self) -> Option { + if self.current >= self.indices.len() { + return None; + } + let result = self + .storage + .by_index(&self.key, &self.indices[self.current]); + self.current += 1; + match result { + Ok(None) => None, + Ok(Some(result)) => Some(Ok(result)), + Err(e) => Some(Err(e)), + } + } +} + +#[cfg(test)] +mod iter { + use super::*; + const BASE: &str = "/tmp/openvasd/unittest"; + + #[test] + fn empty() { + let key = "indexed_empty"; + let store = CachedIndexFileStorer::init(BASE).unwrap(); + let mut iter: IndexedByteStorageIterator<_, Vec> = + IndexedByteStorageIterator::new(key, store).unwrap(); + assert_eq!(iter.next(), None); + } + + #[test] + fn single() { + let key = "indexed_single"; + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + store.put(key, "Hello World".as_bytes()).unwrap(); + let mut iter = IndexedByteStorageIterator::new(key, store.clone()).unwrap(); + assert_eq!(iter.next(), Some(Ok("Hello World".as_bytes().to_vec()))); + assert_eq!(iter.next(), None); + store.remove(key).unwrap(); + } +} + +#[cfg(test)] +mod cached { + use super::*; + const BASE: &str = "/tmp/openvasd/unittest"; + + #[test] + fn invalid_ranges() { + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + store.put("a", "Hello World".as_bytes()).unwrap(); + let result = store + .by_range::>("a", Range::Between(1, 1000)) + .unwrap(); + assert_eq!(result.len(), 0); + } + + #[test] + fn index_order() { + let mut keys = ["a", "b", "c", "d", "e", "f"]; + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + for k in &keys { + store.put(k, "Hello World".as_bytes()).unwrap(); + } + let ordered_keys = store + .cache + .iter() + .map(|e| e.as_ref().unwrap().0.as_str()) + .collect::>(); + assert_eq!(ordered_keys, ["f", "e", "d", "c", "b"]); + for k in &keys { + store.base.clean(k).unwrap(); + } + + keys.reverse(); + for k in &keys { + store.put(k, "Hello World".as_bytes()).unwrap(); + } + let ordered_keys = store + .cache + .iter() + .map(|e| e.as_ref().unwrap().0.as_str()) + .collect::>(); + assert_eq!(ordered_keys, ["a", "b", "f", "e", "d"]); + keys.reverse(); + for k in &keys { + store.append(k, "Hello World".as_bytes()).unwrap(); + } + let ordered_keys = store + .cache + .iter() + .map(|e| e.as_ref().unwrap().0.as_str()) + .collect::>(); + assert_eq!(ordered_keys, ["f", "e", "d", "c", "b"]); + } + + #[test] + fn append_all_and_ranges() { + let key = "test_cached_append_all"; + let amount = 1000; + fn random_data() -> Vec { + use rand::RngCore; + let mut rng = rand::thread_rng(); + let mut data = vec![0; 1024]; + rng.fill_bytes(&mut data); + data + } + let mut data = Vec::with_capacity(amount); + for _ in 0..amount { + data.push(random_data()); + } + + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + store.put(key, "Hello World".as_bytes()).unwrap(); + store.append_all(key, &data).unwrap(); + let results_all: Vec> = store.by_range(key, Range::All).unwrap(); + assert_eq!(results_all.len(), amount + 1); + assert_eq!(results_all[0], "Hello World".as_bytes()); + let results: Vec> = store.by_range(key, Range::Between(1, amount + 1)).unwrap(); + let results_from: Vec> = store.by_range(key, Range::From(1)).unwrap(); + let results_until: Vec> = store.by_range(key, Range::Until(amount + 1)).unwrap(); + assert_eq!(results_until[0], results_all[0]); + + for i in 0..amount { + assert_eq!(results[i], data[i]); + assert_eq!(results[i], results_from[i]); + // include the first element + assert_eq!(results[i], results_until[i + 1]); + assert_eq!(results[i], results_all[i + 1]); + } + store.remove(key).unwrap(); + } + + #[test] + fn create_on_append() { + let key = "create_on_append"; + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + store.append(key, "Hello World".as_bytes()).unwrap(); + let results: Vec> = store.by_range(key, Range::All).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0], "Hello World".as_bytes()); + store.remove(key).unwrap(); + } +} + +#[cfg(test)] +mod indexed { + + use super::*; + const BASE: &str = "/tmp/openvasd/unittest"; + #[test] + fn storage_single_file() { + let key = "test"; + let store = IndexedFileStorer::init(BASE).unwrap(); + let result = store.create(key, "Hello World".as_bytes()).unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].start, 0); + assert_eq!(result[0].end, 11); + let data = store.data_by_index(key, &result[0]).unwrap(); + assert_eq!(data, "Hello World".as_bytes()); + let index = store.load_index(key).unwrap(); + assert_eq!(index.len(), 1); + assert_eq!(index[0].start, 0); + assert_eq!(index[0].end, 11); + store.clean(key).unwrap(); + } + + #[test] + fn append_to_file() { + let key = "test_append"; + let store = IndexedFileStorer::init(BASE).unwrap(); + let idx = store.create(key, "Hello World".as_bytes()).unwrap(); + store + .append(key, &idx, "The world does not care.".as_bytes()) + .unwrap(); + let index = store.load_index(key).unwrap(); + assert_eq!(index.len(), 2); + assert_eq!(index[0].start, 0); + assert_eq!(index[0].end, 11); + assert_eq!(index[1].start, 11); + assert_eq!(index[1].end, 35); + let data = store.data_by_index(key, &index[0]).unwrap(); + assert_eq!(data, "Hello World".as_bytes()); + let data = store.data_by_index(key, &index[1]).unwrap(); + assert_eq!(data, "The world does not care.".as_bytes()); + store.clean(key).unwrap(); + } + + #[test] + fn append_all() { + let key = "test_append_all"; + let amount = 1000; + fn random_data() -> Vec { + use rand::RngCore; + let mut rng = rand::thread_rng(); + let mut data = vec![0; 1024]; + rng.fill_bytes(&mut data); + data + } + let mut data = Vec::with_capacity(amount); + for _ in 0..amount { + data.push(random_data()); + } + + let store = IndexedFileStorer::init(BASE).unwrap(); + let idx = store.create(key, "Hello World".as_bytes()).unwrap(); + let index = store.append_all_index(key, &idx, &data).unwrap(); + assert_eq!(index.len(), amount + 1); + assert_eq!(index[0].start, 0); + assert_eq!(index[0].end, 11); + for i in 1..amount + 1 { + assert_eq!(index[i].start, index[i - 1].end); + assert_eq!(index[i].end, index[i - 1].end + 1024); + let dr = store.data_by_index(key, &index[i]).unwrap(); + assert_eq!(dr, data[i - 1]); + } + store.clean(key).unwrap(); + } +} diff --git a/rust/infisto/src/bincode.rs b/rust/infisto/src/bincode.rs new file mode 100644 index 000000000..0f4dc6563 --- /dev/null +++ b/rust/infisto/src/bincode.rs @@ -0,0 +1,112 @@ +//! Contains helper for serializing and deserializing structs. +use crate::base; +use bincode::{Decode, Encode}; + +#[derive(Debug)] +/// Serializes and deserializes data +pub enum Serialization { + /// Wrapper for Deserialized T + Deserialized(T), + /// Wrapper for Serialized T + Serialized(Vec), +} + +impl Serialization +where + T: Encode, +{ + /// Serializes given data to Vec + pub fn serialize(t: T) -> Result { + let config = bincode::config::standard(); + + match bincode::encode_to_vec(&t, config) { + Ok(v) => Ok(Serialization::Serialized(v)), + Err(_) => Err(base::Error::Serialize), + } + } + + /// Deserializes given Serialization to T + pub fn deserialize(self) -> Result { + match self { + Serialization::Deserialized(s) => Ok(s), + Serialization::Serialized(_) => Err(base::Error::Serialize), + } + } +} + +impl TryFrom> for Serialization +where + T: Decode, +{ + type Error = base::Error; + + fn try_from(value: Vec) -> Result { + let config = bincode::config::standard(); + match bincode::decode_from_slice(&value, config) { + Ok((t, _)) => Ok(Serialization::Deserialized(t)), + Err(_) => Err(base::Error::Serialize), + } + } +} + +impl AsRef<[u8]> for Serialization { + fn as_ref(&self) -> &[u8] { + match self { + Serialization::Deserialized(_) => &[0u8], + Serialization::Serialized(v) => v.as_ref(), + } + } +} + +impl From> for Vec { + fn from(s: Serialization) -> Self { + match s { + Serialization::Deserialized(_) => vec![0u8], + Serialization::Serialized(v) => v, + } + } +} + +#[cfg(test)] +mod test { + + use crate::base::{CachedIndexFileStorer, IndexedByteStorage, Range}; + + const BASE: &str = "/tmp/openvasd/unittest"; + #[derive(bincode::Decode, bincode::Encode, Clone, Debug, PartialEq)] + struct Test { + a: u32, + b: u32, + } + + #[test] + fn serialization() { + let t = Test { a: 1, b: 2 }; + let s = super::Serialization::serialize(t).unwrap(); + let v = Vec::::from(s); + let s = super::Serialization::::try_from(v).unwrap(); + let t = match s { + super::Serialization::Deserialized(t) => t, + _ => panic!("Serialization::try_from failed"), + }; + assert_eq!(t, Test { a: 1, b: 2 }); + } + + #[test] + fn create_on_append() { + let key = "create_serde_on_append"; + let test = Test { a: 1, b: 2 }; + let serialized = super::Serialization::serialize(&test).unwrap(); + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + store.append(key, serialized).unwrap(); + let results: Vec> = store.by_range(key, Range::All).unwrap(); + assert_eq!(results.len(), 1); + let test2 = match results.get(0).unwrap() { + super::Serialization::Deserialized(t) => t.clone(), + _ => panic!("Serialization::try_from failed"), + }; + + assert_eq!(test, test2); + store.remove(key).unwrap(); + } +} diff --git a/rust/infisto/src/crypto.rs b/rust/infisto/src/crypto.rs new file mode 100644 index 000000000..0359c4701 --- /dev/null +++ b/rust/infisto/src/crypto.rs @@ -0,0 +1,226 @@ +//! Contains helper for encryption. +use chacha20::cipher::generic_array::GenericArray; +use chacha20::cipher::typenum::{U12, U32}; +use chacha20::cipher::{KeyIvInit, StreamCipher}; +use chacha20::ChaCha20; +use pbkdf2::pbkdf2_hmac; +use rand::RngCore; +use sha2::Sha256; + +use crate::base::IndexedByteStorage; + +#[derive(Clone, Debug)] +struct Encrypted { + /// The first 12 bytes of the encrypted data are the nonce. + /// + /// They are combined to implement AsRef<[u8]> for Encrypted. + data_and_nonce: Vec, +} + +impl Encrypted { + fn new(nonce: [u8; 12], mut data: Vec) -> Self { + data.splice(..0, nonce.iter().cloned()); + Self { + data_and_nonce: data, + } + } + + fn data(&self) -> &[u8] { + &self.data_and_nonce[12..] + } + + fn nonce(&self) -> &GenericArray { + self.data_and_nonce[..12].into() + } +} + +impl From> for Encrypted { + fn from(data_and_nonce: Vec) -> Self { + Self { data_and_nonce } + } +} + +impl AsRef<[u8]> for Encrypted { + fn as_ref(&self) -> &[u8] { + self.data_and_nonce.as_ref() + } +} + +/// A ChaCha20 index file storer. +/// +/// Encrypts and decrypts the index file using ChaCha20 and a given password. +#[derive(Clone, Debug)] +pub struct ChaCha20IndexFileStorer { + store: T, + key: Key, +} + +#[derive(Clone, Debug)] +/// Key to used for encryption +pub struct Key(GenericArray); + +impl Default for Key { + fn default() -> Self { + let mut key = [0u8; 32]; + let mut rng = rand::thread_rng(); + rng.fill_bytes(&mut key); + Key(key.into()) + } +} + +impl From<&str> for Key { + fn from(s: &str) -> Self { + let mut key = [0u8; 32]; + // we currently don't need a salt as we only have one key + let salt = [0u8; 8]; + pbkdf2_hmac::(s.as_bytes(), &salt, 8000, &mut key); + Key(key.into()) + } +} + +impl From<&String> for Key { + fn from(s: &String) -> Self { + Key::from(s.as_str()) + } +} +impl From for Key { + fn from(s: String) -> Self { + Key::from(s.as_str()) + } +} +impl ChaCha20IndexFileStorer { + /// Creates a new instance. + pub fn new(store: T, key: K) -> Self + where + K: Into, + { + Self { + store, + key: key.into(), + } + } + + fn encrypt(key: &Key, mut data: Vec) -> Encrypted { + let mut nonce = [0u8; 12]; + let mut rng = rand::thread_rng(); + rng.fill_bytes(&mut nonce); + let Key(key) = key; + let mut cipher = ChaCha20::new(key, &nonce.into()); + cipher.apply_keystream(&mut data); + Encrypted::new(nonce, data) + } + + fn decrypt(key: &Key, encrypted: &Encrypted) -> Vec { + let mut data = encrypted.data().to_vec(); + let Key(key) = key; + let mut cipher = ChaCha20::new(key, encrypted.nonce()); + cipher.apply_keystream(&mut data); + data.to_vec() + } +} + +impl IndexedByteStorage for ChaCha20IndexFileStorer +where + S: IndexedByteStorage, +{ + fn put(&mut self, key: &str, data: T) -> Result<(), crate::base::Error> + where + T: AsRef<[u8]>, + { + let encrypted = Self::encrypt(&self.key, data.as_ref().to_vec()); + self.store.put(key, encrypted) + } + + fn append_all(&mut self, key: &str, data: &[T]) -> Result<(), crate::base::Error> + where + T: AsRef<[u8]>, + { + let data = data + .iter() + .map(|d| Self::encrypt(&self.key, d.as_ref().to_vec())) + .collect::>(); + self.store.append_all(key, &data) + } + + fn remove(&mut self, key: &str) -> Result<(), crate::base::Error> { + self.store.remove(key) + } + + fn indices(&self, key: &str) -> Result, crate::base::Error> { + self.store.indices(key) + } + + fn by_indices( + &self, + key: &str, + indices: &[crate::base::Index], + ) -> Result, crate::base::Error> + where + T: TryFrom>, + { + let encrypted = self.store.by_indices::(key, indices)?; + Ok(encrypted + .into_iter() + .map(|e| Self::decrypt(&self.key, &e).try_into()) + .filter_map(Result::ok) + .collect()) + } +} + +#[cfg(test)] +mod tests { + use crate::base::{CachedIndexFileStorer, Range}; + + use super::*; + const BASE: &str = "/tmp/openvasd/unittest"; + + #[test] + fn append_all() { + let key = "test_crypto_append_all"; + let amount = 1000; + fn random_data() -> Vec { + use rand::RngCore; + let mut rng = rand::thread_rng(); + let mut data = vec![0; 1024]; + rng.fill_bytes(&mut data); + data + } + let mut data = Vec::with_capacity(amount); + for _ in 0..amount { + data.push(random_data()); + } + + let store = CachedIndexFileStorer::init(BASE).unwrap(); + let mut store = ChaCha20IndexFileStorer::new(store, Key::default()); + store.put(key, "Hello World".as_bytes()).unwrap(); + store.append_all(key, &data).unwrap(); + let results_all: Vec> = store.by_range(key, Range::All).unwrap(); + assert_eq!(results_all.len(), amount + 1); + assert_eq!(results_all[0], "Hello World".as_bytes()); + let results: Vec> = store.by_range(key, Range::Between(1, amount + 1)).unwrap(); + let results_from: Vec> = store.by_range(key, Range::From(1)).unwrap(); + let results_until: Vec> = store.by_range(key, Range::Until(amount + 1)).unwrap(); + assert_eq!(results_until[0], results_all[0]); + + for i in 0..amount { + assert_eq!(results[i], data[i]); + assert_eq!(results[i], results_from[i]); + // include the first element + assert_eq!(results[i], results_until[i + 1]); + assert_eq!(results[i], results_all[i + 1]); + } + store.remove(key).unwrap(); + } + + #[test] + fn create_on_append() { + let key = "create_crypto_on_append"; + let store = CachedIndexFileStorer::init(BASE).unwrap(); + let mut store = ChaCha20IndexFileStorer::new(store, Key::default()); + store.append(key, "Hello World".as_bytes()).unwrap(); + let results: Vec> = store.by_range(key, Range::All).unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0], "Hello World".as_bytes()); + store.remove(key).unwrap(); + } +} diff --git a/rust/infisto/src/lib.rs b/rust/infisto/src/lib.rs new file mode 100644 index 000000000..780e86081 --- /dev/null +++ b/rust/infisto/src/lib.rs @@ -0,0 +1,7 @@ +#![doc = include_str!("../README.md")] +#![warn(missing_docs)] + +pub mod base; +pub mod bincode; +pub mod crypto; +pub mod serde; diff --git a/rust/infisto/src/serde.rs b/rust/infisto/src/serde.rs new file mode 100644 index 000000000..d1e971d2d --- /dev/null +++ b/rust/infisto/src/serde.rs @@ -0,0 +1,114 @@ +//! Contains helper for serializing and deserializing structs. +use serde::{Deserialize, Serialize}; + +use crate::base; + +#[derive(Debug)] +/// Serializes and deserializes data +pub enum Serialization { + /// Wrapper for Deserialized T + Deserialized(T), + /// Wrapper for Serialized T + Serialized(Vec), +} + +impl Serialization +where + T: Serialize, +{ + /// Serializes given data to Vec + pub fn serialize(t: T) -> Result { + let config = bincode::config::standard(); + + match bincode::serde::encode_to_vec(&t, config) { + Ok(v) => Ok(Serialization::Serialized(v)), + Err(_) => Err(base::Error::Serialize), + } + } + + /// Deserializes given Serialization to T + pub fn deserialize(self) -> Result { + match self { + Serialization::Deserialized(s) => Ok(s), + Serialization::Serialized(_) => Err(base::Error::Serialize), + } + } +} + +impl TryFrom> for Serialization +where + T: for<'de> Deserialize<'de>, +{ + type Error = base::Error; + + fn try_from(value: Vec) -> Result { + let config = bincode::config::standard(); + match bincode::serde::decode_from_slice(&value, config) { + Ok((t, _)) => Ok(Serialization::Deserialized(t)), + Err(_) => Err(base::Error::Serialize), + } + } +} + +impl AsRef<[u8]> for Serialization { + fn as_ref(&self) -> &[u8] { + match self { + Serialization::Deserialized(_) => &[0u8], + Serialization::Serialized(v) => v.as_ref(), + } + } +} + +impl From> for Vec { + fn from(s: Serialization) -> Self { + match s { + Serialization::Deserialized(_) => vec![0u8], + Serialization::Serialized(v) => v, + } + } +} + +#[cfg(test)] +mod test { + use serde::{Deserialize, Serialize}; + + use crate::base::{CachedIndexFileStorer, IndexedByteStorage, Range}; + + const BASE: &str = "/tmp/openvasd/unittest"; + #[derive(Serialize, Deserialize, Clone, Debug, PartialEq)] + struct Test { + a: u32, + b: u32, + } + + #[test] + fn serialization() { + let t = Test { a: 1, b: 2 }; + let s = super::Serialization::serialize(t).unwrap(); + let v = Vec::::from(s); + let s = super::Serialization::::try_from(v).unwrap(); + let t = match s { + super::Serialization::Deserialized(t) => t, + _ => panic!("Serialization::try_from failed"), + }; + assert_eq!(t, Test { a: 1, b: 2 }); + } + + #[test] + fn create_on_append() { + let key = "create_serde_on_append"; + let test = Test { a: 1, b: 2 }; + let serialized = super::Serialization::serialize(&test).unwrap(); + let mut store = CachedIndexFileStorer::init(BASE).unwrap(); + store.append(key, serialized).unwrap(); + let results: Vec> = store.by_range(key, Range::All).unwrap(); + assert_eq!(results.len(), 1); + let test2 = match results.get(0).unwrap() { + super::Serialization::Deserialized(t) => t.clone(), + _ => panic!("Serialization::try_from failed"), + }; + + assert_eq!(test, test2); + store.remove(key).unwrap(); + } +} diff --git a/rust/models/Cargo.toml b/rust/models/Cargo.toml index 2ce7b2510..c8f38cfd4 100644 --- a/rust/models/Cargo.toml +++ b/rust/models/Cargo.toml @@ -8,11 +8,15 @@ license = "GPL-2.0-or-later" [dependencies] serde = {version = "1", features = ["derive"], optional = true} +bincode = {version = "2.0.0-rc.3", optional = true } [features] -default = ["serde_support"] +default = ["serde_support", "bincode_support"] serde_support = ["serde"] +bincode_support = ["bincode"] [dev-dependencies] serde_json = "1" +# required for credentials +bincode = "2.0.0-rc.3" diff --git a/rust/models/src/credential.rs b/rust/models/src/credential.rs index 81ede0ed4..692aac47c 100644 --- a/rust/models/src/credential.rs +++ b/rust/models/src/credential.rs @@ -8,6 +8,7 @@ feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct Credential { /// Service to use for accessing a host pub service: Service, @@ -60,6 +61,7 @@ impl Default for Credential { feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub enum Service { #[cfg_attr(feature = "serde_support", serde(rename = "ssh"))] /// SSH, supports [UP](CredentialType::UP) and [USK](CredentialType::USK) as credential types @@ -91,6 +93,7 @@ impl AsRef for Service { feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] /// Enum representing the type of credentials. pub enum CredentialType { #[cfg_attr(feature = "serde_support", serde(rename = "up"))] diff --git a/rust/models/src/host_info.rs b/rust/models/src/host_info.rs index 073f9245a..c0c5b6164 100644 --- a/rust/models/src/host_info.rs +++ b/rust/models/src/host_info.rs @@ -3,11 +3,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later /// Information about hosts of a running scan -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] #[cfg_attr( feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct HostInfo { /// Number of all hosts, that are contained in a target pub all: u32, diff --git a/rust/models/src/lib.rs b/rust/models/src/lib.rs index 4d37a80a8..01f8a4529 100644 --- a/rust/models/src/lib.rs +++ b/rust/models/src/lib.rs @@ -38,6 +38,8 @@ where #[cfg(test)] //#[cfg(feature = "serde_support")] mod tests { + use crate::Target; + use super::scan::Scan; #[test] @@ -173,120 +175,9 @@ mod tests { } "#; // tests that it doesn't panic when parsing the json - let _: Scan = serde_json::from_str(json_str).unwrap(); - } -} + let s: Scan = serde_json::from_str(json_str).unwrap(); -#[test] -#[should_panic] -fn parses_complex_example_fails() { - let json_str = r#"{ - "scan_id": "6c591f83-8f7b-452a-8c78-ba35779e682f", - "target": { - "hosts": [ - "127.0.0.1", - "192.168.0.1-15", - "10.0.5.0/24", - "::1", - "2001:db8:0000:0000:0000:0000:0000:0001-00ff", - "2002::1234:abcd:ffff:c0a8:101/64", - "examplehost" - ], - "ports": [ - { - "protocol": "udp", - "range": [{"start": 22}, {"start": 1024, "end": 1030}] - }, - { - "protocol": "tcp", - "range": [{"start": 24, "end": 30}] - }, - { - "range": [{"start": 100, "end": 1000}] - } - ], - "credentials": [ - { - "service": "ssh", - "port": 22, - "usk": { - "username": "user", - "password": "pw", - "private": "ssh-key..." - } - }, - { - "service": "smb", - "up": { - "username": "user", - "password": "pw" - } - }, - { - "service": "snmp", - "snmp": { - "username": "user", - "password": "pw", - "community": "my_community", - "auth_algorithm": "md5", - "privacy_password": "priv_pw", - "privacy_algorithm": "aes" - } - } - ], - "alive_test_ports": [ - { - "protocol": "tcp", - "range": [{"start": 1, "end": 100}] - }, - { - "range": [{ "start": 443 }] - } - ], - "alive_test_methods": [ - "icmp", - "tcp_syn", - "tcp_ack", - "arp", - "consider_alive" - ], - "reverse_lookup_unify": true, - "reverse_lookup_only": false - }, - "scanner_preferences": [ - { - "id": "target_port", - "value": "443" - }, - { - "id": "use_https", - "value": "1" - }, - { - "id": "profile", - "value": "fast_scan" + let b = bincode::encode_to_vec(&s, bincode::config::standard()).unwrap(); + //let _: Target = bincode::deserialize(&b).unwrap(); } - ], - "vts": [ - { - "oid": "1.3.6.1.4.1.25623.1.0.10662", - "parameters": [ - { - "id": 1, - "value": "200" - }, - { - "id": 2, - "value": "yes" - } - ] - }, - { - "oid": "1.3.6.1.4.1.25623.1.0.10330" - } - ] -} -"#; - // tests that it doesn't panic when parsing the json - let _: Scan = serde_json::from_str(json_str).unwrap(); } diff --git a/rust/models/src/parameter.rs b/rust/models/src/parameter.rs index 0cd77e271..a675bdb04 100644 --- a/rust/models/src/parameter.rs +++ b/rust/models/src/parameter.rs @@ -7,6 +7,7 @@ feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] /// Represents a parameter for a VTS configuration. pub struct Parameter { /// The ID of the parameter. diff --git a/rust/models/src/port.rs b/rust/models/src/port.rs index 6864528ac..086cb94a3 100644 --- a/rust/models/src/port.rs +++ b/rust/models/src/port.rs @@ -9,6 +9,7 @@ use std::fmt::Display; feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct Port { #[cfg_attr( feature = "serde_support", @@ -26,6 +27,7 @@ pub struct Port { feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct PortRange { /// The required start port. /// @@ -57,6 +59,7 @@ impl Display for PortRange { feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] #[cfg_attr(feature = "serde_support", serde(rename_all = "lowercase"))] pub enum Protocol { UDP, diff --git a/rust/models/src/scan.rs b/rust/models/src/scan.rs index 9ed216119..aac965aa6 100644 --- a/rust/models/src/scan.rs +++ b/rust/models/src/scan.rs @@ -11,19 +11,14 @@ use super::{scanner_preference::ScannerPreference, target::Target, vt::VT}; derive(serde::Serialize, serde::Deserialize), serde(deny_unknown_fields) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct Scan { - #[cfg_attr( - feature = "serde_support", - serde(skip_serializing_if = "Option::is_none", skip_deserializing) - )] + #[cfg_attr(feature = "serde_support", serde(default))] /// Unique ID of a scan pub scan_id: Option, /// Information about the target to scan pub target: Target, - #[cfg_attr( - feature = "serde_support", - serde(default, skip_serializing_if = "Vec::is_empty") - )] + #[cfg_attr(feature = "serde_support", serde(default))] /// Configuration options for the scanner pub scanner_preferences: Vec, /// List of VTs to execute for the target diff --git a/rust/models/src/scanner_preference.rs b/rust/models/src/scanner_preference.rs index 0b9d31d94..b00eebd5d 100644 --- a/rust/models/src/scanner_preference.rs +++ b/rust/models/src/scanner_preference.rs @@ -3,11 +3,12 @@ // SPDX-License-Identifier: GPL-2.0-or-later /// Configuration preference for the scanner -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Default, Debug, Clone, PartialEq, Eq)] #[cfg_attr( feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct ScannerPreference { /// The ID of the scanner preference. pub id: String, diff --git a/rust/models/src/status.rs b/rust/models/src/status.rs index 2486a703f..2d7a11851 100644 --- a/rust/models/src/status.rs +++ b/rust/models/src/status.rs @@ -7,30 +7,19 @@ use std::fmt::Display; use super::host_info::HostInfo; /// Status information about a scan -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] #[cfg_attr( feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct Status { - #[cfg_attr( - feature = "serde_support", - serde(skip_serializing_if = "Option::is_none") - )] /// Timestamp for the start of a scan pub start_time: Option, - #[cfg_attr( - feature = "serde_support", - serde(skip_serializing_if = "Option::is_none") - )] /// Timestamp for the end of a scan pub end_time: Option, /// The phase, a scan is currently in pub status: Phase, - #[cfg_attr( - feature = "serde_support", - serde(skip_serializing_if = "Option::is_none") - )] /// Information about the hosts of a running scan pub host_info: Option, } @@ -52,6 +41,7 @@ impl Status { derive(serde::Serialize, serde::Deserialize) )] #[cfg_attr(feature = "serde_support", serde(rename_all = "snake_case"))] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub enum Phase { /// A scan has been stored but not started yet #[default] diff --git a/rust/models/src/target.rs b/rust/models/src/target.rs index 424bd9550..c255c3dce 100644 --- a/rust/models/src/target.rs +++ b/rust/models/src/target.rs @@ -10,40 +10,26 @@ use super::{credential::Credential, port::Port}; feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct Target { /// List of hosts to scan pub hosts: Vec, /// List of ports used for scanning pub ports: Vec, - #[cfg_attr( - feature = "serde_support", - serde(default, skip_serializing_if = "Vec::is_empty") - )] + #[cfg_attr(feature = "serde_support", serde(default))] /// List of credentials used to get access to a system pub credentials: Vec, - #[cfg_attr( - feature = "serde_support", - serde(default, skip_serializing_if = "Vec::is_empty") - )] + #[cfg_attr(feature = "serde_support", serde(default))] /// List of ports used for alive testing pub alive_test_ports: Vec, - #[cfg_attr( - feature = "serde_support", - serde(default, skip_serializing_if = "Vec::is_empty") - )] + #[cfg_attr(feature = "serde_support", serde(default))] /// Methods used for alive testing pub alive_test_methods: Vec, - #[cfg_attr( - feature = "serde_support", - serde(skip_serializing_if = "Option::is_none") - )] + #[cfg_attr(feature = "serde_support", serde(default))] /// If multiple IP addresses resolve to the same DNS name the DNS name will only get scanned /// once. pub reverse_lookup_unify: Option, - #[cfg_attr( - feature = "serde_support", - serde(skip_serializing_if = "Option::is_none") - )] + #[cfg_attr(feature = "serde_support", serde(default))] /// Only scan IP addresses that can be resolved into a DNS name. pub reverse_lookup_only: Option, } @@ -54,6 +40,7 @@ pub struct Target { feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] #[cfg_attr(feature = "serde_support", serde(rename_all = "snake_case"))] pub enum AliveTestMethods { Icmp, diff --git a/rust/models/src/vt.rs b/rust/models/src/vt.rs index a0b35c414..7000d9f19 100644 --- a/rust/models/src/vt.rs +++ b/rust/models/src/vt.rs @@ -5,18 +5,16 @@ use super::parameter::Parameter; /// A VT to execute during a scan, including its parameters -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Default)] #[cfg_attr( feature = "serde_support", derive(serde::Serialize, serde::Deserialize) )] +#[cfg_attr(feature = "bincode_support", derive(bincode::Encode, bincode::Decode))] pub struct VT { /// The ID of the VT to execute pub oid: String, - #[cfg_attr( - feature = "serde_support", - serde(default, skip_serializing_if = "Vec::is_empty") - )] + #[cfg_attr(feature = "serde_support", serde(default,))] /// The list of parameters for the VT pub parameters: Vec, } diff --git a/rust/nasl-builtin-misc/src/lib.rs b/rust/nasl-builtin-misc/src/lib.rs index 51cc2d2e3..f280fda22 100644 --- a/rust/nasl-builtin-misc/src/lib.rs +++ b/rust/nasl-builtin-misc/src/lib.rs @@ -265,7 +265,7 @@ fn localtime(register: &Register, _: &Context) -> Result match NaiveDateTime::from_timestamp_opt(secs, 0) { Some(dt) => { let offset = chrono::Local::now().offset().fix(); - let dt: DateTime = DateTime::from_utc(dt, offset); + let dt: DateTime = DateTime::from_naive_utc_and_offset(dt, offset); create_localtime_map(dt) } _ => create_localtime_map(Local::now()), diff --git a/rust/nasl-builtin-ssh/src/lib.rs b/rust/nasl-builtin-ssh/src/lib.rs index d8991571a..1dece97a2 100644 --- a/rust/nasl-builtin-ssh/src/lib.rs +++ b/rust/nasl-builtin-ssh/src/lib.rs @@ -963,18 +963,16 @@ impl Ssh { loop { match session.session.userauth_keyboard_interactive(None, None) { Ok(AuthStatus::Info) => { - let info = match session - .session - .userauth_keyboard_interactive_info() - { - Ok(i) => i, - Err(_) => { - return Err(FunctionErrorKind::Dirty(format!( + let info = + match session.session.userauth_keyboard_interactive_info() { + Ok(i) => i, + Err(_) => { + return Err(FunctionErrorKind::Dirty(format!( "Failed setting user authentication for SessionID {}", session_id ))); - } - }; + } + }; if verbose { ctx.logger().info(&format!("SSH kbdint name={}", info.name)); ctx.logger().info(&format!( @@ -1533,18 +1531,16 @@ impl Ssh { loop { match session.session.userauth_keyboard_interactive(None, None) { Ok(AuthStatus::Info) => { - let info = match session - .session - .userauth_keyboard_interactive_info() - { - Ok(i) => i, - Err(_) => { - return Err(FunctionErrorKind::Dirty(format!( + let info = + match session.session.userauth_keyboard_interactive_info() { + Ok(i) => i, + Err(_) => { + return Err(FunctionErrorKind::Dirty(format!( "Failed setting user authentication for SessionID {}", session_id ))); - } - }; + } + }; if verbose { ctx.logger().info(&format!("SSH kbdint name={}", info.name)); ctx.logger().info(&format!( diff --git a/rust/nasl-interpreter/src/operator.rs b/rust/nasl-interpreter/src/operator.rs index 8f2a074aa..e5ee6f0d2 100644 --- a/rust/nasl-interpreter/src/operator.rs +++ b/rust/nasl-interpreter/src/operator.rs @@ -140,17 +140,12 @@ where TokenCategory::Ampersand => self.execute(stmts, |a, b| num_expr!(& a b)), TokenCategory::Pipe => self.execute(stmts, |a, b| num_expr!(| a b)), TokenCategory::Caret => self.execute(stmts, |a, b| num_expr!(^ a b)), - TokenCategory::StarStar => self.execute( - stmts, - |a, b| { - let (a, b) = as_i64(a, b); - let result = (a as u32).pow(b as u32); - Ok(NaslValue::Number(result as i64)) - } - ), - TokenCategory::Tilde => { - self.execute(stmts, |a, _| Ok((!i64::from(&a)).into())) - } + TokenCategory::StarStar => self.execute(stmts, |a, b| { + let (a, b) = as_i64(a, b); + let result = (a as u32).pow(b as u32); + Ok(NaslValue::Number(result as i64)) + }), + TokenCategory::Tilde => self.execute(stmts, |a, _| Ok((!i64::from(&a)).into())), // string TokenCategory::EqualTilde => self.execute(stmts, match_regex), TokenCategory::BangTilde => self.execute(stmts, not_match_regex), diff --git a/rust/nasl-syntax/Cargo.toml b/rust/nasl-syntax/Cargo.toml index b1d1e14a0..fd64a48b6 100644 --- a/rust/nasl-syntax/Cargo.toml +++ b/rust/nasl-syntax/Cargo.toml @@ -10,7 +10,7 @@ license = "GPL-2.0-or-later" storage = { path = "../storage" } [dev-dependencies] -criterion = "0.5" +criterion = "0" [[bench]] name = "parse" diff --git a/rust/nasl-syntax/src/token.rs b/rust/nasl-syntax/src/token.rs index 381aba33c..d7f6b28e6 100644 --- a/rust/nasl-syntax/src/token.rs +++ b/rust/nasl-syntax/src/token.rs @@ -873,10 +873,7 @@ mod tests { [(Category::Data("Hello \\'you\\'!".as_bytes().to_vec()), 1, 1)] ); let code = "'Hello \\'you\\'!\\'"; - verify_tokens!( - code, - [(Category::Unclosed(UnclosedCategory::Data), 1, 1)] - ); + verify_tokens!(code, [(Category::Unclosed(UnclosedCategory::Data), 1, 1)]); } #[test] @@ -887,17 +884,11 @@ mod tests { verify_tokens!("0b01", [(Number(1), 1, 1)]); verify_tokens!("1234567890", [(Number(1234567890), 1, 1)]); verify_tokens!("012345670", [(Number(2739128), 1, 1)]); - verify_tokens!( - "0x1234567890ABCDEF", - [(Number(1311768467294899695), 1, 1)] - ); + verify_tokens!("0x1234567890ABCDEF", [(Number(1311768467294899695), 1, 1)]); // That would be later illegal because a number if followed by a number // but within tokenizing I think it is the best to ignore that and let it be handled by AST verify_tokens!("0b02", [(Number(0), 1, 1), (Number(2), 1, 4)]); - verify_tokens!( - "0b2", - [(IllegalNumber(Binary), 1, 1), (Number(2), 1, 3)] - ); + verify_tokens!("0b2", [(IllegalNumber(Binary), 1, 1), (Number(2), 1, 3)]); } #[test] @@ -927,8 +918,10 @@ mod tests { ); verify_tokens!( "4_h4llo", - [(Number(4), 1, 1), - (Identifier(Undefined("_h4llo".to_owned())), 1, 2)] + [ + (Number(4), 1, 1), + (Identifier(Undefined("_h4llo".to_owned())), 1, 2) + ] ); } diff --git a/rust/openvasd/Cargo.toml b/rust/openvasd/Cargo.toml index 0a620326a..40fdd9cce 100644 --- a/rust/openvasd/Cargo.toml +++ b/rust/openvasd/Cargo.toml @@ -11,6 +11,7 @@ osp = {path = "../osp"} nasl-interpreter = { path = "../nasl-interpreter" } feed = {path = "../feed"} storage = { path = "../storage" } +infisto = { path = "../infisto" } hyper = { version = "0.14.26", features = ["full", "stream"] } tokio = { version = "1.28.1", features = ["full"] } tracing = "0.1.37" @@ -36,3 +37,4 @@ async-traits = "0.0.0" base64 = "0.21.2" [dev-dependencies] +bincode = "1" diff --git a/rust/openvasd/src/config.rs b/rust/openvasd/src/config.rs index cf2f0e38a..4d15a1837 100644 --- a/rust/openvasd/src/config.rs +++ b/rust/openvasd/src/config.rs @@ -9,7 +9,7 @@ use std::{ time::Duration, }; -use clap::ArgAction; +use clap::{builder::TypedValueParser, ArgAction}; use serde::{Deserialize, Serialize}; #[derive(Deserialize, Serialize, Debug, Clone)] @@ -85,6 +85,62 @@ impl Default for Logging { } } +#[derive(Deserialize, Serialize, Default, Debug, Clone, PartialEq, Eq)] +pub enum StorageType { + #[default] + #[serde(rename = "inmemory")] + InMemory, + #[serde(rename = "fs")] + FileSystem, +} + +impl TypedValueParser for StorageType { + type Value = StorageType; + + fn parse_ref( + &self, + cmd: &clap::Command, + _: Option<&clap::Arg>, + value: &std::ffi::OsStr, + ) -> Result { + Ok(match value.to_str().unwrap_or_default() { + "fs" => StorageType::FileSystem, + "inmemory" => StorageType::InMemory, + _ => { + let mut cmd = cmd.clone(); + let err = cmd.error( + clap::error::ErrorKind::InvalidValue, + "`{}` is not an storage type.", + ); + return Err(err); + } + }) + } +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub struct FileStorage { + pub path: PathBuf, + pub key: Option, +} + +impl Default for FileStorage { + fn default() -> Self { + Self { + path: PathBuf::from("/var/lib/openvasd/storage"), + key: None, + } + } +} + +#[derive(Deserialize, Serialize, Default, Debug, Clone)] +pub struct Storage { + #[serde(default, rename = "type")] + pub storage_type: StorageType, + #[serde(default)] + pub fs: FileStorage, +} + #[derive(Deserialize, Serialize, Debug, Clone, Default)] pub struct Config { #[serde(default)] @@ -99,6 +155,8 @@ pub struct Config { pub listener: Listener, #[serde(default)] pub log: Logging, + #[serde(default)] + pub storage: Storage, } impl Display for Config { @@ -209,7 +267,6 @@ impl Config { .long("read-timeout") .value_parser(clap::value_parser!(u64)) .value_name("SECONDS") - // .default_value("1") .help("read timeout in seconds on the ospd-openvas socket"), ) .arg( @@ -218,7 +275,6 @@ impl Config { .long("result-check-interval") .value_parser(clap::value_parser!(u64)) .value_name("SECONDS") - // .default_value("1") .help("interval to check for new results in seconds"), ) .arg( @@ -230,12 +286,34 @@ impl Config { .value_parser(clap::value_parser!(SocketAddr)) .help("the address to listen to (e.g. 127.0.0.1:3000 or 0.0.0.0:3000)."), ) + .arg( + clap::Arg::new("storage_type") + .env("STORAGE_TYPE") + .long("storage-type") + .value_name("fs,inmemory") + .value_parser(StorageType::InMemory) + .help("either be stored in memory or on the filesystem."), + ) + .arg( + clap::Arg::new("storage_path") + .env("STORAGE_PATH") + .long("storage-path") + .value_name("PATH") + .value_parser(clap::builder::PathBufValueParser::new()) + .help("the path that contains the files when type is set to fs."), + ) + .arg( + clap::Arg::new("storage_key") + .env("STORAGE_KEY") + .long("storage-key") + .value_name("KEY") + .help("the password to use for encryption when type is set to fs. If not set the files are not encrypted."), + ) .arg( clap::Arg::new("log-level") .env("OPENVASD_LOG") .long("log-level") .short('L') - // .default_value("INFO") .help("Level of log messages to be shown. TRACE > DEBUG > INFO > WARN > ERROR"), ) .get_matches(); @@ -286,6 +364,17 @@ impl Config { if let Some(log_level) = cmds.get_one::("log-level") { config.log.level = log_level.clone(); } + if let Some(stype) = cmds.get_one::("storage_type") { + config.storage.storage_type = stype.clone(); + } + if let Some(path) = cmds.get_one::("storage_path") { + config.storage.fs.path = path.clone(); + } + if let Some(key) = cmds.get_one::("storage_key") { + if !key.is_empty() { + config.storage.fs.key = Some(key.clone()); + } + } config } } @@ -294,6 +383,8 @@ impl Config { mod tests { use std::{path::PathBuf, time::Duration}; + use crate::config::StorageType; + #[test] fn defaults() { let config = super::Config::default(); @@ -324,8 +415,19 @@ mod tests { fn parse_toml() { let cfg = r#"[log] level = "DEBUG" + [storage] + type = "fs" + [storage.fs] + path = "/var/lib/openvasd/storage/test" + key = "changeme" "#; let config: super::Config = toml::from_str(cfg).unwrap(); assert_eq!(config.log.level, "DEBUG"); + assert_eq!( + config.storage.fs.path, + PathBuf::from("/var/lib/openvasd/storage/test") + ); + assert_eq!(config.storage.fs.key, Some("changeme".to_string())); + assert_eq!(config.storage.storage_type, StorageType::FileSystem); } } diff --git a/rust/openvasd/src/controller/context.rs b/rust/openvasd/src/controller/context.rs index 36e5df858..62ba73dca 100644 --- a/rust/openvasd/src/controller/context.rs +++ b/rust/openvasd/src/controller/context.rs @@ -63,12 +63,15 @@ pub struct ContextBuilder { marker: std::marker::PhantomData, response: response::Response, } -impl ContextBuilder, NoScanner> { + +impl + ContextBuilder, NoScanner> +{ /// Creates a new context builder. pub fn new() -> Self { Self { scanner: NoScanner, - storage: crate::storage::InMemoryStorage::default(), + storage: crate::storage::inmemory::Storage::default(), result_config: None, feed_config: None, api_key: None, @@ -115,9 +118,27 @@ impl ContextBuilder { /// Sets the storage. #[allow(dead_code)] - pub fn storage(mut self, storage: DB) -> Self { - self.storage = storage; - self + pub fn storage(self, storage: NDB) -> ContextBuilder { + let ContextBuilder { + scanner, + storage: _, + result_config, + feed_config, + api_key, + enable_get_scans, + marker, + response, + } = self; + ContextBuilder { + scanner, + storage, + result_config, + feed_config, + api_key, + enable_get_scans, + marker, + response, + } } } @@ -159,7 +180,6 @@ impl ContextBuilder> { scanner: self.scanner.0, response: self.response, db: self.storage, - oids: Default::default(), result_config: self.result_config, feed_config: self.feed_config, abort: Default::default(), @@ -181,8 +201,6 @@ pub struct Context { /// It is locked to allow concurrent access, usually the results are updated /// with a background task and appended to the progress of the scan. pub db: DB, - /// The OIDs thate can be handled by this sensor. - pub oids: RwLock<(String, Vec)>, /// Configuration for result fetching pub result_config: Option, /// Configuration for feed handling. @@ -239,7 +257,7 @@ impl ScanResultFetcher for NoOpScanner { } impl Default - for Context> + for Context> { fn default() -> Self { ContextBuilder::new().scanner(Default::default()).build() diff --git a/rust/openvasd/src/controller/entry.rs b/rust/openvasd/src/controller/entry.rs index ce6959d1c..53c4299a0 100644 --- a/rust/openvasd/src/controller/entry.rs +++ b/rust/openvasd/src/controller/entry.rs @@ -116,6 +116,11 @@ where (&Method::POST, Scans(None)) => { match crate::request::json_request::(&ctx.response, req).await { Ok(mut scan) => { + if scan.scan_id.is_some() { + return Ok(ctx + .response + .bad_request("field scan_id is not allowed to be set.")); + } let id = uuid::Uuid::new_v4().to_string(); let resp = ctx.response.created(&id); scan.scan_id = Some(id.clone()); @@ -166,11 +171,8 @@ where } (&Method::GET, Scans(None)) => { if ctx.enable_get_scans { - match ctx.db.get_scans().await { - Ok(scans) => Ok(ctx.response.ok(&scans - .into_iter() - .map(|s| s.0.scan_id.unwrap_or_default()) - .collect::>())), + match ctx.db.get_scan_ids().await { + Ok(scans) => Ok(ctx.response.ok(&scans)), Err(e) => Ok(ctx.response.internal_server_error(&e)), } } else { @@ -187,15 +189,17 @@ where Err(crate::storage::Error::NotFound) => Ok(ctx.response.not_found("scans/status", &id)), Err(e) => Ok(ctx.response.internal_server_error(&e)), }, - (&Method::DELETE, Scans(Some(id))) => match ctx.db.remove_scan(&id).await? { - Some((_, status)) => { + (&Method::DELETE, Scans(Some(id))) => match ctx.db.get_status(&id).await { + Ok(status) => { if status.is_running() { ctx.scanner.stop_scan(id.clone()).await?; } + ctx.db.remove_scan(&id).await?; ctx.scanner.delete_scan(id).await?; Ok(ctx.response.no_content()) } - None => Ok(ctx.response.not_found("scans", &id)), + Err(crate::storage::Error::NotFound) => Ok(ctx.response.not_found("scans", &id)), + Err(e) => Err(e.into()), }, (&Method::GET, ScanResults(id, rid)) => { let (begin, end) = { @@ -232,7 +236,8 @@ where } (&Method::GET, Vts) => { - let (_, oids) = ctx.oids.read()?.clone(); + let oids = ctx.db.oids().await?; + Ok(ctx.response.ok_json_stream(oids).await) } _ => Ok(ctx.response.not_found("path", req.uri().path())), diff --git a/rust/openvasd/src/controller/feed.rs b/rust/openvasd/src/controller/feed.rs index 04829f326..8c3a0dfeb 100644 --- a/rust/openvasd/src/controller/feed.rs +++ b/rust/openvasd/src/controller/feed.rs @@ -7,57 +7,55 @@ use std::sync::Arc; use crate::feed::FeedIdentifier; use super::context::Context; -use super::quit_on_poison; pub async fn fetch(ctx: Arc>) where S: super::Scanner + 'static + std::marker::Send + std::marker::Sync, DB: crate::storage::Storage + 'static + std::marker::Send + std::marker::Sync, { + tracing::debug!("Starting VTS synchronization loop"); if let Some(cfg) = &ctx.feed_config { let interval = cfg.verify_interval; - let path = cfg.path.clone(); - tracing::debug!("Starting VTS synchronization loop"); - tokio::task::spawn_blocking(move || loop { + loop { + let path = cfg.path.clone(); if *ctx.abort.read().unwrap() { tracing::trace!("aborting"); break; - } - let last_hash = match ctx.oids.read() { - Ok(vts) => vts.0.clone(), - Err(_) => quit_on_poison(), }; - let hash = match FeedIdentifier::sumfile_hash(&path) { - Ok(h) => h, - Err(e) => { - tracing::warn!("Failed to compute sumfile hash: {e:?}"); - "".to_string() - } - }; - if last_hash != hash { - tracing::debug!("VTS hash {last_hash} changed {hash}, updating"); - match FeedIdentifier::from_feed(&path) { - Ok(o) => { - let mut oids = match ctx.oids.write() { - Ok(oids) => oids, - Err(_) => quit_on_poison(), - }; - tracing::trace!( - "VTS hash changed updated (old: {}, new: {})", - oids.1.len(), - o.len() - ); - *oids = (hash, o); - } + let last_hash = ctx.db.feed_hash().await; + let result = tokio::task::spawn_blocking(move || { + let hash = match FeedIdentifier::sumfile_hash(&path) { + Ok(h) => h, Err(e) => { - tracing::warn!("unable to fetch new oids, leaving the old: {e:?}") + tracing::warn!("Failed to compute sumfile hash: {e:?}"); + "".to_string() } }; - } - std::thread::sleep(interval); - }) - .await - .unwrap(); + if last_hash.is_empty() || last_hash.clone() != hash { + FeedIdentifier::from_feed(&path).map(|x| (hash, x)) + } else { + Ok((String::new(), vec![])) + } + }) + .await + .unwrap(); + match result { + Ok((hash, oids)) => { + if !oids.is_empty() { + match ctx.db.push_oids(hash.clone(), oids).await { + Ok(_) => { + tracing::debug!("updated feed {hash}") + } + Err(e) => { + tracing::warn!("unable to fetch new oids, leaving the old: {e:?}") + } + } + } + } + Err(e) => tracing::warn!("unable to fetch new oids, leaving the old: {e:?}"), + }; + tokio::time::sleep(interval).await; + } } } diff --git a/rust/openvasd/src/controller/mod.rs b/rust/openvasd/src/controller/mod.rs index 5a28c3e42..6e9ddb7fe 100644 --- a/rust/openvasd/src/controller/mod.rs +++ b/rust/openvasd/src/controller/mod.rs @@ -26,6 +26,7 @@ macro_rules! make_svc { ($controller:expr) => {{ // start background service use std::sync::Arc; + tokio::spawn(crate::controller::results::fetch(Arc::clone(&$controller))); tokio::spawn(crate::controller::feed::fetch(Arc::clone(&$controller))); @@ -211,15 +212,12 @@ mod tests { } #[tokio::test] - #[should_panic] async fn add_scan_with_id_fails() { - let scan: models::Scan = models::Scan::default(); - let controller = Arc::new(Context::default()); - let id = post_scan_id(&scan, Arc::clone(&controller)).await; - let resp = get_scan(&id, Arc::clone(&controller)).await; - let resp = hyper::body::to_bytes(resp.into_body()).await.unwrap(); - - let _ = serde_json::from_slice::(&resp).unwrap(); + let mut scan: models::Scan = models::Scan::default(); + scan.scan_id = Some(String::new()); + let ctx = Arc::new(Context::default()); + let resp = post_scan(&scan, Arc::clone(&ctx)).await; + assert_eq!(resp.status(), hyper::http::StatusCode::BAD_REQUEST); } #[tokio::test] @@ -301,8 +299,9 @@ mod tests { break; } } - let resp = get_results(&id, Arc::clone(&controller), None, None).await; + let mut resp = get_results(&id, Arc::clone(&controller), None, None).await; assert_eq!(resp.len(), 4950); + resp.reverse(); resp.iter().enumerate().for_each(|(i, r)| { assert_eq!(r.id, i); }); @@ -312,8 +311,9 @@ mod tests { let resp = get_results(&id, Arc::clone(&controller), Some(4949), None).await; assert_eq!(resp.len(), 1); assert_eq!(resp[0].id, 4949); - let resp = get_results(&id, Arc::clone(&controller), None, Some((4900, 4923))).await; + let mut resp = get_results(&id, Arc::clone(&controller), None, Some((4900, 4923))).await; assert_eq!(resp.len(), 24); + resp.reverse(); for (i, r) in resp.iter().enumerate() { assert_eq!(r.id, i + 4900); } diff --git a/rust/openvasd/src/controller/results.rs b/rust/openvasd/src/controller/results.rs index ba41f7884..078885230 100644 --- a/rust/openvasd/src/controller/results.rs +++ b/rust/openvasd/src/controller/results.rs @@ -26,37 +26,42 @@ where tracing::trace!("aborting"); break; } - let scans = ctx.db.get_scans().await; + let scans = ctx.db.get_scan_ids().await; if let Err(e) = scans { tracing::warn!("Failed to get scans: {e}"); continue; } let scans = scans.unwrap(); - for (scan, status) in scans.iter() { + for id in scans.iter() { // should never be none, probably makes sense to change scan_id // to not be an option and set a uuid on default when it is // missing on json serialization - let id = scan.scan_id.as_ref().unwrap(); - if status.is_done() { - tracing::trace!("{id} skipping status = {}", status.status); - continue; - } - let results = ctx.scanner.fetch_results(id.clone()).await; - match results { - Ok(fr) => { - tracing::trace!("{} fetched results", id); - // we panic when we fetched results but are unable to - // store them in the database. - // When this happens we effectively lost the results - // and need to escalate this. - ctx.db.append_fetch_result(id, fr).await.unwrap(); + match ctx.db.get_status(id).await { + Ok(status) if status.is_done() => { + tracing::trace!("{id} skipping status = {}", status.status); } - Err(crate::scan::Error::Poisoned) => { - quit_on_poison::<()>(); + Ok(_) => { + let results = ctx.scanner.fetch_results(id.clone()).await; + match results { + Ok(fr) => { + tracing::trace!("{} fetched results", id); + // we panic when we fetched results but are unable to + // store them in the database. + // When this happens we effectively lost the results + // and need to escalate this. + ctx.db.append_fetched_result(id, fr).await.unwrap(); + } + Err(crate::scan::Error::Poisoned) => { + quit_on_poison::<()>(); + } + Err(e) => { + tracing::warn!("Failed to fetch results for {}: {e}", &id); + } + } } Err(e) => { - tracing::warn!("Failed to fetch results for {}: {e}", &id); + tracing::warn!("Unable to get status for {}: {}", id, e); } } } diff --git a/rust/openvasd/src/main.rs b/rust/openvasd/src/main.rs index 7c056c63d..aec6ea0ef 100644 --- a/rust/openvasd/src/main.rs +++ b/rust/openvasd/src/main.rs @@ -12,25 +12,23 @@ mod scan; mod storage; mod tls; -#[tokio::main] -async fn main() -> Result<(), Box> { - let config = config::Config::load(); - let filter = tracing_subscriber::EnvFilter::builder() - .with_default_directive(tracing::metadata::LevelFilter::INFO.into()) - .parse_lossy(config.log.level.clone()); - tracing_subscriber::fmt().with_env_filter(filter).init(); +pub async fn run<'a, DB>( + db: DB, + config: config::Config, +) -> Result<(), Box> +where + DB: crate::storage::Storage + std::marker::Send + 'static + std::marker::Sync, +{ + let scanner = scan::OSPDWrapper::new(config.ospd.socket.clone(), config.ospd.read_timeout); let rc = config.ospd.result_check_interval; let fc = (config.feed.path.clone(), config.feed.check_interval); - if !config.ospd.socket.exists() { - tracing::warn!("OSPD socket {} does not exist. Some commands will not work until the socket is created!", config.ospd.socket.display()); - } - let scanner = scan::OSPDWrapper::new(config.ospd.socket.clone(), config.ospd.read_timeout); let ctx = controller::ContextBuilder::new() .result_config(rc) .feed_config(fc) .scanner(scanner) .api_key(config.endpoints.key.clone()) .enable_get_scans(config.endpoints.enable_get_scans) + .storage(db) .build(); let controller = std::sync::Arc::new(ctx); let addr = config.listener.address; @@ -51,3 +49,38 @@ async fn main() -> Result<(), Box> { } Ok(()) } + +#[tokio::main] +async fn main() -> Result<(), Box> { + let config = config::Config::load(); + let filter = tracing_subscriber::EnvFilter::builder() + .with_default_directive(tracing::metadata::LevelFilter::INFO.into()) + .parse_lossy(config.log.level.clone()); + tracing_subscriber::fmt().with_env_filter(filter).init(); + if !config.ospd.socket.exists() { + tracing::warn!("OSPD socket {} does not exist. Some commands will not work until the socket is created!", config.ospd.socket.display()); + } + match config.storage.storage_type { + config::StorageType::InMemory => { + tracing::info!("using in memory store. No sensitive data will be stored on disk."); + run(storage::inmemory::Storage::default(), config).await + } + config::StorageType::FileSystem => { + if let Some(key) = &config.storage.fs.key { + tracing::info!( + "using in file storage. Sensitive data will be encrypted stored on disk." + ); + run( + storage::file::encrypted(&config.storage.fs.path, key)?, + config, + ) + .await + } else { + tracing::warn!( + "using in file storage. Sensitive data will be stored on disk without any encryption." + ); + run(storage::file::unencrypted(&config.storage.fs.path)?, config).await + } + } + } +} diff --git a/rust/openvasd/src/response.rs b/rust/openvasd/src/response.rs index a560e8e9e..53377fc2a 100644 --- a/rust/openvasd/src/response.rs +++ b/rust/openvasd/src/response.rs @@ -51,11 +51,18 @@ impl Transform> for U8Streamer { } } -#[derive(Debug, Clone)] +#[derive(Default)] +enum ArrayStreamState { + #[default] + First, + Running, + Finished, +} + struct ArrayStreamer { - elements: Vec, + elements: Box + Send>, transform: PhantomData, - first: bool, + state: ArrayStreamState, } impl Unpin for ArrayStreamer {} @@ -64,20 +71,20 @@ impl ArrayStreamer where E: Serialize, { - fn json(elements: Vec) -> Self { + fn json(elements: Box + Send>) -> Self { Self { elements, - first: true, + state: ArrayStreamState::First, transform: PhantomData, } } } impl ArrayStreamer, U8Streamer> { - fn u8(elements: Vec>) -> Self { + fn u8(elements: Box> + Send>) -> Self { Self { elements, - first: true, + state: ArrayStreamState::First, transform: PhantomData, } } @@ -94,24 +101,24 @@ where _: &mut std::task::Context<'_>, ) -> std::task::Poll> { let out = { - if self.first && self.elements.is_empty() { - self.first = false; - Some("[]".to_string()) - } else if self.first { - self.first = false; - Some("[".to_string()) - } else { - match self.elements.pop() { - Some(e) => { - let e = T::transform(e); - if self.elements.is_empty() { - Some(format!("{}]", e)) - } else { - Some(format!("{},", e)) - } + match self.state { + ArrayStreamState::First => { + self.state = ArrayStreamState::Running; + if let Some(e) = self.elements.next() { + Some(format!("[{}", T::transform(e))) + } else { + Some("[".to_string()) } - None => None, } + ArrayStreamState::Running => match self.elements.next() { + Some(e) => Some(format!(",{}", T::transform(e))), + None => { + self.state = ArrayStreamState::Finished; + Some("]".to_string()) + } + }, + + ArrayStreamState::Finished => None, } }; @@ -144,10 +151,9 @@ impl Response { .header("feed-version", &self.feed_version) } - #[tracing::instrument] pub async fn create_stream(&self, code: hyper::StatusCode, value: S) -> Result where - S: Stream> + Send + std::fmt::Debug + 'static, + S: Stream> + Send + 'static, O: Into + 'static, E: Into> + 'static, { @@ -169,14 +175,14 @@ impl Response { } } - pub async fn ok_json_stream(&self, value: Vec) -> Result + pub async fn ok_json_stream(&self, value: Box + Send>) -> Result where T: Serialize + Send + std::fmt::Debug + 'static, { let stream = ArrayStreamer::json(value); self.create_stream(hyper::StatusCode::OK, stream).await } - pub async fn ok_byte_stream(&self, value: Vec>) -> Result { + pub async fn ok_byte_stream(&self, value: Box> + Send>) -> Result { let stream = ArrayStreamer::u8(value); self.create_stream(hyper::StatusCode::OK, stream).await } diff --git a/rust/openvasd/src/scan.rs b/rust/openvasd/src/scan.rs index d0cf1d751..654673287 100644 --- a/rust/openvasd/src/scan.rs +++ b/rust/openvasd/src/scan.rs @@ -58,10 +58,7 @@ impl From> for Error { impl OSPDWrapper { /// Creates a new instance of OSPDWrapper pub fn new(socket: PathBuf, r_timeout: Option) -> Self { - Self { - socket, - r_timeout, - } + Self { socket, r_timeout } } fn check_socket(&self) -> Result { diff --git a/rust/openvasd/src/storage/file.rs b/rust/openvasd/src/storage/file.rs new file mode 100644 index 000000000..1a34767fd --- /dev/null +++ b/rust/openvasd/src/storage/file.rs @@ -0,0 +1,446 @@ +use std::{ops::Deref, path::Path}; + +use super::*; + +pub struct Storage { + storage: Arc>, + // although that will be lost on restart I will be read into immediately on start by parsing + // the feed. + hash: tokio::sync::RwLock, +} +pub fn unencrypted

(path: P) -> Result, Error> +where + P: AsRef, +{ + let ifs = infisto::base::IndexedFileStorer::init(path)?; + Ok(ifs.into()) +} + +pub fn encrypted( + path: P, + key: K, +) -> Result< + Storage>, + Error, +> +where + P: AsRef, + K: Into, +{ + let ifs = infisto::base::IndexedFileStorer::init(path)?; + Ok(infisto::crypto::ChaCha20IndexFileStorer::new(ifs, key).into()) +} + +impl From for Error { + fn from(e: infisto::base::Error) -> Self { + Self::Storage(Box::new(e)) + } +} + +impl From for Storage +where + S: infisto::base::IndexedByteStorage + std::marker::Sync + std::marker::Send + 'static, +{ + fn from(value: S) -> Self { + Storage::new(value) + } +} + +impl Storage { + pub fn new(s: S) -> Self { + Storage { + storage: Arc::new(s.into()), + hash: tokio::sync::RwLock::new(String::new()), + } + } +} + +#[async_trait] +impl ProgressGetter for Storage +where + S: infisto::base::IndexedByteStorage + std::marker::Sync + std::marker::Send + Clone + 'static, +{ + async fn get_decrypted_scan(&self, id: &str) -> Result<(models::Scan, models::Status), Error> { + // the encryption is done in whole, unlike when in memory + self.get_scan(id).await + } + async fn get_results( + &self, + id: &str, + from: Option, + to: Option, + ) -> Result> + Send>, Error> { + let range = { + use infisto::base::Range; + match (from, to) { + (None, None) => Range::All, + (None, Some(to)) => Range::Until(to), + (Some(from), None) => Range::From(from), + (Some(from), Some(to)) => Range::Between(from, to), + } + }; + let key = format!("results_{id}"); + let storage = &self.storage.read().unwrap(); + let storage: S = storage.deref().clone(); + let iter = infisto::base::IndexedByteStorageIterator::<_, Vec>::by_range( + &key, storage, range, + )?; + let parsed = iter.filter_map(|x| x.ok()); + Ok(Box::new(parsed)) + } + async fn get_scan(&self, id: &str) -> Result<(models::Scan, models::Status), Error> { + let key = format!("scan_{id}"); + let status_key = format!("status_{id}"); + + let storage = Arc::clone(&self.storage); + + use infisto::base::Range; + use infisto::bincode::Serialization; + tokio::task::spawn_blocking(move || { + let storage = storage.read().unwrap(); + let scans: Vec> = storage.by_range(&key, Range::All)?; + let status: Vec> = + storage.by_range(&status_key, Range::All)?; + + match scans.first() { + Some(Serialization::Deserialized(scan)) => match status.first() { + Some(Serialization::Deserialized(status)) => Ok((scan.clone(), status.clone())), + Some(_) => Err(Error::Serialization), + None => Err(Error::NotFound), + }, + Some(_) => Err(Error::Serialization), + None => Err(Error::NotFound), + } + }) + .await + .unwrap() + } + async fn get_scan_ids(&self) -> Result, Error> { + let storage = Arc::clone(&self.storage); + use infisto::base::Range; + use infisto::bincode::Serialization; + tokio::task::spawn_blocking(move || { + let storage = &storage.read().unwrap(); + let scans: Vec> = match storage.by_range("scans", Range::All) { + Ok(s) => s, + Err(infisto::base::Error::FileOpen(std::io::ErrorKind::NotFound)) => vec![], + Err(e) => return Err(e.into()), + }; + let mut scans: Vec<_> = scans + .into_iter() + .filter_map(|x| match x { + Serialization::Serialized(_) => None, + Serialization::Deserialized(x) => Some(x), + }) + .collect(); + scans.sort(); + scans.dedup(); + Ok(scans) + }) + .await + .unwrap() + } + async fn get_status(&self, id: &str) -> Result { + let key = format!("status_{id}"); + let storage = Arc::clone(&self.storage); + + use infisto::base::Range; + use infisto::bincode::Serialization; + tokio::task::spawn_blocking(move || { + let storage = &storage.read().unwrap(); + let status: Vec> = storage.by_range(&key, Range::All)?; + match status.first() { + Some(Serialization::Deserialized(status)) => Ok(status.clone()), + Some(_) => Err(Error::Serialization), + None => Err(Error::NotFound), + } + }) + .await + .unwrap() + } +} + +#[async_trait] +impl AppendFetchResult for Storage +where + S: infisto::base::IndexedByteStorage + std::marker::Sync + std::marker::Send + Clone + 'static, +{ + async fn append_fetched_result( + &self, + id: &str, + (status, results): FetchResult, + ) -> Result<(), Error> { + let key = format!("results_{}", id); + self.update_status(id, status).await?; + let scan_key = format!("scan_{id}"); + + let storage = Arc::clone(&self.storage); + tokio::task::spawn_blocking(move || { + let storage = &mut storage.write().unwrap(); + let mut serialized_results = Vec::with_capacity(results.len()); + let ilen = storage.indices(&scan_key)?.len(); + for (i, mut result) in results.into_iter().enumerate() { + result.id = ilen + i; + let bytes = serde_json::to_vec(&result)?; + serialized_results.push(bytes); + } + storage.append_all(&key, &serialized_results)?; + + Ok(()) + }) + .await + .unwrap() + } +} +#[async_trait] +impl ScanStorer for Storage +where + S: infisto::base::IndexedByteStorage + std::marker::Sync + std::marker::Send + Clone + 'static, +{ + async fn insert_scan(&self, scan: models::Scan) -> Result<(), Error> { + let id = scan.scan_id.clone().unwrap_or_default(); + let key = format!("scan_{id}"); + let status_key = format!("status_{id}"); + let storage = Arc::clone(&self.storage); + tokio::task::spawn_blocking(move || { + let scan = infisto::bincode::Serialization::serialize(scan)?; + let status = infisto::bincode::Serialization::serialize(models::Status::default())?; + let mut storage = storage.write().unwrap(); + storage.put(&key, scan)?; + storage.put(&status_key, status)?; + + let stored_key = infisto::bincode::Serialization::serialize(&id)?; + storage.append("scans", stored_key)?; + Ok(()) + }) + .await + .unwrap() + } + async fn remove_scan(&self, id: &str) -> Result<(), Error> { + let key = format!("scan_{}", id); + let status_key = format!("status_{}", id); + let results_key = format!("results_{}", id); + let storage = Arc::clone(&self.storage); + let ids = self.get_scan_ids().await?; + let ids: Vec<_> = ids + .into_iter() + .filter(|x| x != id) + .filter_map(|x| infisto::bincode::Serialization::serialize(x).ok()) + .collect(); + + tokio::task::spawn_blocking(move || { + // we ignore results errors as there may or may not be results + let mut storage = storage.write().unwrap(); + let _ = storage.remove(&results_key); + storage.remove(&key)?; + storage.remove(&status_key)?; + storage.remove("scans")?; + storage.append_all("scans", &ids)?; + Ok(()) + }) + .await + .unwrap() + } + async fn update_status(&self, id: &str, status: models::Status) -> Result<(), Error> { + let key = format!("status_{}", id); + let storage = Arc::clone(&self.storage); + + tokio::task::spawn_blocking(move || { + let status = infisto::bincode::Serialization::serialize(status)?; + let mut storage = storage.write().unwrap(); + storage.put(&key, status)?; + Ok(()) + }) + .await + .unwrap() + } +} + +#[async_trait] +impl OIDStorer for Storage +where + S: infisto::base::IndexedByteStorage + std::marker::Sync + std::marker::Send + Clone + 'static, +{ + async fn push_oids(&self, hash: String, oids: Vec) -> Result<(), Error> { + let key = "oids".to_string(); + let storage = Arc::clone(&self.storage); + let mut h = self.hash.write().await; + *h = hash; + + tokio::task::spawn_blocking(move || { + let oids = oids + .into_iter() + .filter_map(|x| infisto::bincode::Serialization::serialize(x.to_string()).ok()) + .collect::>(); + + let mut storage = storage.write().unwrap(); + match storage.remove(&key) { + Ok(_) => {} + Err(infisto::base::Error::Remove(std::io::ErrorKind::NotFound)) => {} + Err(e) => return Err(e.into()), + }; + storage.append_all(&key, &oids)?; + Ok(()) + }) + .await + .unwrap() + } + + async fn oids(&self) -> Result + Send>, Error> { + let key = "oids".to_string(); + let storage = &self.storage.read().unwrap(); + let storage: S = storage.deref().clone(); + let iter = infisto::base::IndexedByteStorageIterator::< + _, + infisto::bincode::Serialization, + >::new(&key, storage)?; + let parsed = iter + .filter_map(|x| x.ok()) + .filter_map(|x| x.deserialize().ok()); + Ok(Box::new(parsed)) + } + + async fn feed_hash(&self) -> String { + self.hash.read().await.clone() + } +} + +#[cfg(test)] +mod tests { + use models::Scan; + + use super::*; + + #[test] + fn serialize() { + let scan = models::Status::default(); + + let serialized = bincode::serialize(&scan).unwrap(); + let deserialized = bincode::deserialize(&serialized).unwrap(); + assert_eq!(scan, deserialized); + } + + #[tokio::test] + async fn credentials() { + let jraw = r#" +{ + "target": { + "hosts": [ + "192.168.123.52" + ], + "ports": [ + { + "protocol": "tcp", + "range": [ + { + "start": 22, + "end": 22 + } + ] + } + ], + "credentials": [ + { + "service": "ssh", + "port": 22, + "up": { + "username": "msfadmin", + "password": "msfadmin" + } + } + ] + }, + "vts": [ + { + "oid": "1.3.6.1.4.1.25623.1.0.90022" + } + ] +} + "#; + let mut scan: Scan = serde_json::from_str(jraw).unwrap(); + scan.scan_id = Some("aha".to_string()); + let storage = + infisto::base::CachedIndexFileStorer::init("/tmp/openvasd/credential").unwrap(); + let storage = crate::storage::file::Storage::new(storage); + storage.insert_scan(scan.clone()).await.unwrap(); + let (scan2, _) = storage.get_scan("aha").await.unwrap(); + assert_eq!(scan, scan2); + } + + #[tokio::test] + async fn oids() { + let mut oids = Vec::with_capacity(100000); + for i in 0..(oids.capacity()) { + oids.push(i.to_string()); + } + let storage = infisto::base::CachedIndexFileStorer::init("/tmp/openvasd/oids").unwrap(); + let storage = crate::storage::file::Storage::new(storage); + storage + .push_oids(String::new(), oids.clone()) + .await + .unwrap(); + let noids = storage.oids().await.unwrap(); + let mut len = 0; + for (i, s) in noids.enumerate() { + assert_eq!(s, oids[i]); + len += 1; + } + assert_eq!(len, oids.len()); + } + + #[tokio::test] + async fn file_storage_test() { + let mut scans = Vec::with_capacity(100); + for i in 0..100 { + let mut scan = Scan::default(); + scan.scan_id = Some(i.to_string()); + scans.push(scan); + } + + let storage = + infisto::base::CachedIndexFileStorer::init("/tmp/openvasd/file_storage_test").unwrap(); + let storage = crate::storage::file::Storage::new(storage); + for s in scans.clone().into_iter() { + storage.insert_scan(s).await.unwrap() + } + + for s in scans.clone().into_iter() { + storage.get_scan(&s.scan_id.unwrap()).await.unwrap(); + } + storage.remove_scan("5").await.unwrap(); + storage.insert_scan(scans[5].clone()).await.unwrap(); + let ids = storage.get_scan_ids().await.unwrap(); + assert_eq!(scans.len(), ids.len()); + let status = models::Status::default(); + let results = vec![models::Result::default()]; + storage + .append_fetched_result("42", (status, results)) + .await + .unwrap(); + + let mut status = models::Status::default(); + status.status = models::Phase::Running; + + let results = vec![models::Result::default()]; + storage + .append_fetched_result("42", (status.clone(), results)) + .await + .unwrap(); + let stored_status = storage.get_status("42").await.unwrap(); + assert_eq!(status, stored_status); + let range: Vec = storage + .get_results("42", None, None) + .await + .unwrap() + .map(String::from_utf8) + .filter_map(|x| x.ok()) + .collect(); + assert_eq!(2, range.len()); + for s in scans { + let _ = storage.remove_scan(&s.scan_id.unwrap_or_default()).await; + } + + let ids = storage.get_scan_ids().await.unwrap(); + assert_eq!(0, ids.len()); + } +} diff --git a/rust/openvasd/src/storage.rs b/rust/openvasd/src/storage/inmemory.rs similarity index 59% rename from rust/openvasd/src/storage.rs rename to rust/openvasd/src/storage/inmemory.rs index 55cb6a9ce..b206bad15 100644 --- a/rust/openvasd/src/storage.rs +++ b/rust/openvasd/src/storage/inmemory.rs @@ -1,104 +1,6 @@ -use std::collections::HashMap; - -use async_trait::async_trait; +use super::*; use tokio::sync::RwLock; -use crate::{crypt, scan::FetchResult}; - -#[derive(Debug)] -pub enum Error { - SerializationError, - NotFound, -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - use Error::*; - match self { - NotFound => write!(f, "not found"), - SerializationError => write!(f, "serialization error"), - } - } -} - -impl std::error::Error for Error {} - -impl From for Error { - fn from(_: serde_json::Error) -> Self { - Self::SerializationError - } -} - -impl From for Error { - fn from(_: crypt::ParseError) -> Self { - Self::SerializationError - } -} -impl From for Error { - fn from(_: std::string::FromUtf8Error) -> Self { - Self::SerializationError - } -} - -#[async_trait] -/// A trait for getting the progress of a scan, the scan itself with decrypted credentials and -/// encrypted as well as results. -/// -/// The main usage of this trait is in the controller and when transforming a scan to a osp -pub trait ProgressGetter { - /// Returns the scan. - async fn get_scan(&self, id: &str) -> Result<(models::Scan, models::Status), Error>; - /// Returns the scan with dcecrypted passwords. - /// - /// This method should only be used when the password is required. E.g. - /// when transforming a scan to a osp command. - async fn get_decrypted_scan(&self, id: &str) -> Result<(models::Scan, models::Status), Error>; - /// Returns all scans. - async fn get_scans(&self) -> Result, Error>; - /// Returns the status of a scan. - async fn get_status(&self, id: &str) -> Result; - /// Returns the results of a scan as json bytes. - /// - /// OpenVASD just stores to results without processing them therefore we - /// can just return the json bytes. - async fn get_results( - &self, - id: &str, - from: Option, - to: Option, - ) -> Result>, Error>; -} - -#[async_trait] -/// A trait for storing scans. -/// -/// The main usage of this trait is in the controller and when a user inserts or removes a scan. -pub trait ScanStorer { - /// Inserts a scan. - async fn insert_scan(&self, t: models::Scan) -> Result, Error>; - /// Removes a scan. - async fn remove_scan(&self, id: &str) -> Result, Error>; - /// Updates a status of a scan. - /// - /// This is required when a scan is started or stopped. - async fn update_status(&self, id: &str, status: models::Status) -> Result<(), Error>; -} - -#[async_trait] -/// A trait for appending results from a different source. -/// -/// This is used when a scan is started and the results are fetched from ospd. -pub trait AppendFetchResult { - async fn append_fetch_result(&self, id: &str, results: FetchResult) -> Result<(), Error>; -} - -#[async_trait] -/// Combines the traits `ProgressGetter`, `ScanStorer` and `AppendFetchResult`. -pub trait Storage: ProgressGetter + ScanStorer + AppendFetchResult {} - -#[async_trait] -impl Storage for T where T: ProgressGetter + ScanStorer + AppendFetchResult {} - #[derive(Clone, Debug, Default)] struct Progress { /// The scan that is being tracked. The credentials passwords are encrypted. @@ -112,18 +14,23 @@ struct Progress { } #[derive(Debug)] -pub struct InMemoryStorage { +pub struct Storage { scans: RwLock>, + oids: RwLock>, + hash: RwLock, + crypter: E, } -impl InMemoryStorage +impl Storage where E: crate::crypt::Crypt + Send + Sync + 'static, { pub fn new(crypter: E) -> Self { Self { scans: RwLock::new(HashMap::new()), + oids: RwLock::new(vec![]), + hash: RwLock::new(String::new()), crypter, } } @@ -150,34 +57,34 @@ where } } -impl Default for InMemoryStorage { +impl Default for Storage { fn default() -> Self { Self::new(crate::crypt::ChaCha20Crypt::default()) } } #[async_trait] -impl ScanStorer for InMemoryStorage +impl ScanStorer for Storage where E: crate::crypt::Crypt + Send + Sync + 'static, { - async fn insert_scan(&self, sp: models::Scan) -> Result, Error> { + async fn insert_scan(&self, sp: models::Scan) -> Result<(), Error> { let id = sp.scan_id.clone().unwrap_or_default(); let mut scans = self.scans.write().await; if let Some(prgs) = scans.get_mut(&id) { - let old = prgs.scan.clone(); prgs.scan = sp; - Ok(Some(old)) } else { let progress = self.new_progress(sp).await?; - let old = scans.insert(id.clone(), progress); - Ok(old.map(|p| p.scan)) + scans.insert(id.clone(), progress); } + Ok(()) } - async fn remove_scan(&self, id: &str) -> Result, Error> { + async fn remove_scan(&self, id: &str) -> Result<(), Error> { let mut scans = self.scans.write().await; - Ok(scans.remove(id).map(|p| (p.scan, p.status))) + + scans.remove(id); + Ok(()) } async fn update_status(&self, id: &str, status: models::Status) -> Result<(), Error> { @@ -189,11 +96,11 @@ where } #[async_trait] -impl AppendFetchResult for InMemoryStorage +impl AppendFetchResult for Storage where E: crate::crypt::Crypt + Send + Sync + 'static, { - async fn append_fetch_result( + async fn append_fetched_result( &self, id: &str, (status, results): FetchResult, @@ -213,15 +120,17 @@ where } #[async_trait] -impl ProgressGetter for InMemoryStorage +impl ProgressGetter for Storage where E: crate::crypt::Crypt + Send + Sync + 'static, { - async fn get_scans(&self) -> Result, Error> { + async fn get_scan_ids(&self) -> Result, Error> { let scans = self.scans.read().await; let mut result = Vec::with_capacity(scans.len()); for (_, progress) in scans.iter() { - result.push((progress.scan.clone(), progress.status.clone())); + if let Some(id) = progress.scan.scan_id.as_ref() { + result.push(id.clone()); + } } Ok(result) } @@ -256,21 +165,46 @@ where id: &str, from: Option, to: Option, - ) -> Result>, Error> { + ) -> Result> + Send>, Error> { let scans = self.scans.read().await; let progress = scans.get(id).ok_or(Error::NotFound)?; let from = from.unwrap_or(0); let to = to.unwrap_or(progress.results.len()); let to = to.min(progress.results.len()); if from > to || from > progress.results.len() { - return Ok(Vec::new()); + return Ok(Box::new(Vec::new().into_iter())); } let mut results = Vec::with_capacity(to - from); for result in &progress.results[from..to] { let b = self.crypter.decrypt_sync(result); results.push(b); } - Ok(results) + Ok(Box::new(results.into_iter())) + } +} + +#[async_trait] +impl OIDStorer for Storage +where + E: Send + Sync + 'static, +{ + async fn push_oids(&self, hash: String, mut oids: Vec) -> Result<(), Error> { + let mut o = self.oids.write().await; + o.clear(); + o.append(&mut oids); + o.shrink_to_fit(); + let mut f = self.hash.write().await; + *f = hash; + Ok(()) + } + + async fn oids(&self) -> Result + Send>, Error> { + let o = self.oids.read().await.clone(); + Ok(Box::new(o.into_iter())) + } + + async fn feed_hash(&self) -> String { + self.hash.read().await.clone() } } @@ -282,20 +216,18 @@ mod tests { #[tokio::test] async fn store_delete_scan() { - let storage = InMemoryStorage::default(); + let storage = Storage::default(); let scan = Scan::default(); let id = scan.scan_id.clone().unwrap_or_default(); - let inserted = storage.insert_scan(scan).await.unwrap(); - assert!(inserted.is_none()); + storage.insert_scan(scan).await.unwrap(); let (retrieved, _) = storage.get_scan(&id).await.unwrap(); assert_eq!(retrieved.scan_id.unwrap_or_default(), id); - let (removed, _) = storage.remove_scan(&id).await.unwrap().unwrap(); - assert_eq!(removed.scan_id.unwrap_or_default(), id); + storage.remove_scan(&id).await.unwrap(); } #[tokio::test] async fn encrypt_decrypt_passwords() { - let storage = InMemoryStorage::default(); + let storage = Storage::default(); let mut scan = Scan::default(); let mut pw = models::Credential::default(); pw.credential_type = models::CredentialType::UP { @@ -316,54 +248,57 @@ mod tests { assert_eq!(retrieved.target.credentials[0].password(), "test"); } - async fn store_scan(storage: &InMemoryStorage) -> String { + async fn store_scan(storage: &Storage) -> String { let mut scan = Scan::default(); let id = uuid::Uuid::new_v4().to_string(); scan.scan_id = Some(id.clone()); - let inserted = storage.insert_scan(scan).await.unwrap(); - assert!(inserted.is_none()); + storage.insert_scan(scan).await.unwrap(); id } #[tokio::test] async fn get_scans() { - let storage = InMemoryStorage::default(); + let storage = Storage::default(); for _ in 0..10 { store_scan(&storage).await; } - let scans = storage.get_scans().await.unwrap(); + let scans = storage.get_scan_ids().await.unwrap(); assert_eq!(scans.len(), 10); } #[tokio::test] async fn append_results() { - let storage = InMemoryStorage::default(); + let storage = Storage::default(); let scan = Scan::default(); let id = scan.scan_id.clone().unwrap_or_default(); - let inserted = storage.insert_scan(scan).await.unwrap(); - assert!(inserted.is_none()); + storage.insert_scan(scan).await.unwrap(); let fetch_result = (models::Status::default(), vec![models::Result::default()]); storage - .append_fetch_result(&id, fetch_result) + .append_fetched_result(&id, fetch_result) .await .unwrap(); - let results = storage.get_results(&id, None, None).await.unwrap(); + let results: Vec<_> = storage + .get_results(&id, None, None) + .await + .unwrap() + .collect(); assert_eq!(results.len(), 1); + let result: models::Result = serde_json::from_slice(&results[0]).unwrap(); assert_eq!(result, models::Result::default()); let results = storage.get_results(&id, Some(23), Some(1)).await.unwrap(); - assert_eq!(results.len(), 0); + assert_eq!(results.count(), 0); let results = storage.get_results(&id, Some(0), Some(0)).await.unwrap(); - assert_eq!(results.len(), 0); + assert_eq!(results.count(), 0); let results = storage.get_results(&id, Some(0), Some(5)).await.unwrap(); - assert_eq!(results.len(), 1); + assert_eq!(results.count(), 1); let results = storage.get_results(&id, Some(0), None).await.unwrap(); - assert_eq!(results.len(), 1); + assert_eq!(results.count(), 1); } #[tokio::test] async fn update_status() { - let storage = InMemoryStorage::default(); + let storage = Storage::default(); let id = store_scan(&storage).await; let (_, mut status) = storage.get_scan(&id).await.unwrap(); assert_eq!(status.status, models::Phase::Stored); diff --git a/rust/openvasd/src/storage/mod.rs b/rust/openvasd/src/storage/mod.rs new file mode 100644 index 000000000..ec6602650 --- /dev/null +++ b/rust/openvasd/src/storage/mod.rs @@ -0,0 +1,119 @@ +pub mod file; +pub mod inmemory; +use std::{collections::HashMap, sync::Arc}; + +use async_trait::async_trait; + +use crate::{crypt, scan::FetchResult}; + +#[derive(Debug)] +pub enum Error { + Serialization, + NotFound, + Storage(Box), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + use Error::*; + match self { + NotFound => write!(f, "not found"), + Serialization => write!(f, "serialization error"), + Storage(e) => write!(f, "storage error: {e}"), + } + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(_: serde_json::Error) -> Self { + Self::Serialization + } +} + +impl From for Error { + fn from(_: crypt::ParseError) -> Self { + Self::Serialization + } +} +impl From for Error { + fn from(_: std::string::FromUtf8Error) -> Self { + Self::Serialization + } +} + +#[async_trait] +/// A trait for getting the progress of a scan, the scan itself with decrypted credentials and +/// encrypted as well as results. +/// +/// The main usage of this trait is in the controller and when transforming a scan to a osp +pub trait ProgressGetter { + /// Returns the scan. + async fn get_scan(&self, id: &str) -> Result<(models::Scan, models::Status), Error>; + /// Returns the scan with dcecrypted passwords. + /// + /// This method should only be used when the password is required. E.g. + /// when transforming a scan to a osp command. + async fn get_decrypted_scan(&self, id: &str) -> Result<(models::Scan, models::Status), Error>; + /// Returns all scans. + async fn get_scan_ids(&self) -> Result, Error>; + /// Returns the status of a scan. + async fn get_status(&self, id: &str) -> Result; + /// Returns the results of a scan as json bytes. + /// + /// OpenVASD just stores to results without processing them therefore we + /// can just return the json bytes. + async fn get_results( + &self, + id: &str, + from: Option, + to: Option, + ) -> Result> + Send>, Error>; +} + +#[async_trait] +/// A trait for storing and retrieving oids. +/// +/// OIDs are usually retrieved by scanning the feed, although the initial impulse would be to just +/// delete all oids and append new OIDs when finding them. However in a standard scenario the OID +/// list is used to gather capabilities of that particular scanner. To enforce overriding only when +/// all OIDs are gathered it just allows push of all OIDs at once. +pub trait OIDStorer { + /// Overrides oids + async fn push_oids(&self, hash: String, oids: Vec) -> Result<(), Error>; + + async fn oids(&self) -> Result + Send>, Error>; + + async fn feed_hash(&self) -> String; +} + +#[async_trait] +/// A trait for storing scans. +/// +/// The main usage of this trait is in the controller and when a user inserts or removes a scan. +pub trait ScanStorer { + /// Inserts a scan. + async fn insert_scan(&self, t: models::Scan) -> Result<(), Error>; + /// Removes a scan. + async fn remove_scan(&self, id: &str) -> Result<(), Error>; + /// Updates a status of a scan. + /// + /// This is required when a scan is started or stopped. + async fn update_status(&self, id: &str, status: models::Status) -> Result<(), Error>; +} + +#[async_trait] +/// A trait for appending results from a different source. +/// +/// This is used when a scan is started and the results are fetched from ospd. +pub trait AppendFetchResult { + async fn append_fetched_result(&self, id: &str, results: FetchResult) -> Result<(), Error>; +} + +#[async_trait] +/// Combines the traits `ProgressGetter`, `ScanStorer` and `AppendFetchResult`. +pub trait Storage: ProgressGetter + ScanStorer + AppendFetchResult + OIDStorer {} + +#[async_trait] +impl Storage for T where T: ProgressGetter + ScanStorer + AppendFetchResult + OIDStorer {} diff --git a/rust/osp/src/connection.rs b/rust/osp/src/connection.rs index 1d4f716be..9df20a966 100644 --- a/rust/osp/src/connection.rs +++ b/rust/osp/src/connection.rs @@ -5,7 +5,8 @@ use std::{ io::{self, BufReader, Write}, os::unix::net::UnixStream, - path::Path, time::Duration, + path::Path, + time::Duration, }; use crate::{ @@ -15,7 +16,11 @@ use crate::{ }; /// Sends a command to the unix socket and returns the response -pub fn send_command>(address: T, r_timeout: Option, cmd: ScanCommand) -> Result { +pub fn send_command>( + address: T, + r_timeout: Option, + cmd: ScanCommand, +) -> Result { let mut socket = UnixStream::connect(address)?; let cmd = cmd.try_to_xml()?; if let Some(rtimeout) = r_timeout { @@ -64,7 +69,11 @@ pub fn get_delete_scan_results, I: AsRef>( } /// Starts a scan -pub fn start_scan>(address: T, r_timeout: Option, scan: &models::Scan) -> Result { +pub fn start_scan>( + address: T, + r_timeout: Option, + scan: &models::Scan, +) -> Result { let cmd = ScanCommand::Start(scan); let response = send_command(address, r_timeout, cmd)?; match response { @@ -77,7 +86,11 @@ pub fn start_scan>(address: T, r_timeout: Option, scan: } /// Stops a scan -pub fn stop_scan, I: AsRef>(address: T, r_timeout: Option, scan_id: I) -> Result<(), Error> { +pub fn stop_scan, I: AsRef>( + address: T, + r_timeout: Option, + scan_id: I, +) -> Result<(), Error> { let cmd = ScanCommand::Stop(scan_id.as_ref()); let response = send_command(address, r_timeout, cmd)?; match response { @@ -87,7 +100,11 @@ pub fn stop_scan, I: AsRef>(address: T, r_timeout: Option, I: AsRef>(address: T, r_timeout: Option, scan_id: I) -> Result<(), Error> { +pub fn delete_scan, I: AsRef>( + address: T, + r_timeout: Option, + scan_id: I, +) -> Result<(), Error> { let cmd = ScanCommand::Delete(scan_id.as_ref()); let response = send_command(address, r_timeout, cmd)?; match response {