From bd8b941672eff4461f47b67e795cf58cc890245e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Milenkovi=C4=87?= Date: Mon, 29 Apr 2024 13:33:15 +0100 Subject: [PATCH] update testcontainer deps to 0.16 --- .github/dependabot.yml | 4 ++++ .github/workflows/basic.yml | 7 +++---- Cargo.toml | 10 +++++----- README.md | 32 +++++++++++++++++++------------- src/lib.rs | 26 +++++++++++++++++++++----- tests/e2e.rs | 27 +++++++++++---------------- 6 files changed, 63 insertions(+), 43 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4e30578..7d0d545 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -9,3 +9,7 @@ updates: directory: "/" # Location of package manifests schedule: interval: "daily" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file diff --git a/.github/workflows/basic.yml b/.github/workflows/basic.yml index b3fef7a..5866628 100644 --- a/.github/workflows/basic.yml +++ b/.github/workflows/basic.yml @@ -22,12 +22,11 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable with: - profile: minimal toolchain: stable - components: rustfmt, clippy + components: rustfmt - name: Cargo Format run: cargo fmt --all --check -- - uses: Swatinem/rust-cache@v2 diff --git a/Cargo.toml b/Cargo.toml index 0998376..c2e013e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "testcontainers-minidfs-rs" -version = "0.2.0" +version = "0.3.0" edition = "2021" license = "MIT" @@ -13,12 +13,12 @@ keywords = ["testcontainers", "testing", "hdfs", "hsdoop", "integration-testing" [dependencies] log = "0.4" -testcontainers = "0.15" +testcontainers = "^0.16" project-root = "0.2" [dev-dependencies] -env_logger = "0.10" +env_logger = "0.11" tokio = { version = "1", features = ["rt", "net", "io-util", "macros", "sync"] } -serial_test = { version = "2.0" } -hdfs-native = { version = "0.3" } +serial_test = { version = "3.1" } +hdfs-native = { version = "0.9" } ctor = { version = "0.2" } \ No newline at end of file diff --git a/README.md b/README.md index 61917ce..0752e1e 100644 --- a/README.md +++ b/README.md @@ -8,19 +8,23 @@ Opinionated single node HDFS (DFS) cluster captured in a testcontainer. ```toml testcontainers-minidfs-rs = "0.2" +testcontainers = "0.16" ``` Example: ```rust, no_run -use testcontainers::clients; +use testcontainers::runners::AsyncRunner; use testcontainers_minidfs_rs::*; -let docker = clients::Cli::default(); -let container = MiniDFS::runnable(); -let server_node = docker.run(container); +#[tokio::main] +async fn main() { -let hdfs_namenode_url = format!("hdfs://{}:{}/", "localhost", server_node.get_host_port_ipv4(PORT_NAME_NODE)); + let container = MiniDFS::runnable(); + let server_node = container.start().await; + + let hdfs_namenode_url = format!("hdfs://{}:{}/", "localhost", server_node.get_host_port_ipv4(PORT_NAME_NODE).await); +} ``` HDFS name node should be available at `hdfs://localhost:9000` and name node http at `http://localhost:8020`. @@ -30,27 +34,29 @@ HDFS name node should be available at `hdfs://localhost:9000` and name node http MiniHDFS can be configured with kerberos support: ```rust, no_run -use testcontainers::clients; +use testcontainers::runners::AsyncRunner; use testcontainers_minidfs_rs::*; -let docker = clients::Cli::default(); -let container = MiniDFS::builder().with_kerberos(true).build(); -let server_node = docker.run(container); +#[tokio::main] +async fn main() { + let container = MiniDFS::builder().with_kerberos(true).build(); + let server_node = container.start().await; +} ``` MiniDFS will be configured to support kerberos and all required files will be exposed as a docker volume mounted in the target directory. ```rust, no_run -use testcontainers::clients; +use testcontainers::runners::AsyncRunner; use testcontainers_minidfs_rs::*; let container = MiniDFS::builder().with_kerberos(true).build(); // pre-populated kerberos cache file (ccache) -let kerberos_cache = container.inner().kerberos_cache(); +let kerberos_cache = container.image().kerberos_cache(); // kerberos configuration file krb5.conf -let kerberos_config = container.inner().kerberos_config(); +let kerberos_config = container.image().kerberos_config(); // hadoop configuration core-site.xml -let hdfs_config = container.inner().hdfs_config(); +let hdfs_config = container.image().hdfs_config(); ``` All required files needed for hdfs client setup are exposed. (`kinit` will be executed by the container, kerberos cache will be produced). diff --git a/src/lib.rs b/src/lib.rs index 969bb64..7c86556 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,8 +1,11 @@ #![doc = include_str!("../README.md")] -use log::info; +use log::{debug, info}; use std::path::PathBuf; -use testcontainers::{core::WaitFor, Image, ImageArgs, RunnableImage}; +use testcontainers::{ + core::{Mount, WaitFor}, + Image, ImageArgs, RunnableImage, +}; /// Namenode Port pub const PORT_NAME_NODE: u16 = 9000; @@ -86,9 +89,12 @@ impl MiniDFSBuilder { .with_mapped_port((PORT_KERBEROS, PORT_KERBEROS)); if self.config_volume || self.kerberos_enabled { - let volume_kerberos = config_path.unwrap(); - let volume_kerberos = volume_kerberos.to_string_lossy(); + let volume_kerberos_path = config_path.unwrap(); + let volume_kerberos = volume_kerberos_path.to_string_lossy(); if self.kerberos_enabled { + // we need to create dir if does not exist + // ignoring error, as it may already exist + info!( "Kerberos support has been enabled, testcontainer will mount container volume to local directory: [{}]", volume_kerberos @@ -99,8 +105,11 @@ impl MiniDFSBuilder { volume_kerberos ); } + let target_dir_creation = std::fs::create_dir(&volume_kerberos_path); + debug!("local directory creation error: {} ", target_dir_creation.is_err()); - image = image.with_volume((volume_kerberos, "/tmp/HDFS")) + let volume_hdfs = Mount::bind_mount(volume_kerberos, "/tmp/HDFS"); + image = image.with_mount(volume_hdfs) } if self.kerberos_enabled { @@ -196,3 +205,10 @@ impl Image for MiniDFS { }] } } + +#[cfg(test)] +#[ctor::ctor] +fn init() { + // Enable RUST_LOG logging configuration for test + let _ = env_logger::builder().is_test(true).try_init(); +} diff --git a/tests/e2e.rs b/tests/e2e.rs index 044d084..e3e3b36 100644 --- a/tests/e2e.rs +++ b/tests/e2e.rs @@ -1,19 +1,16 @@ use log::info; -use testcontainers::clients; +use testcontainers::runners::AsyncRunner; use testcontainers_minidfs_rs::*; #[tokio::test] #[serial_test::serial] -async fn e2e_nameode_hello() { - let _ = env_logger::builder().is_test(true).try_init(); - - let docker = clients::Cli::default(); +async fn e2e_name_node_hello() { let container = MiniDFS::runnable(); - let server_node = docker.run(container); + let server_node = container.start().await; let hdfs_server_url = format!( "hdfs://{}:{}/", "localhost", - server_node.get_host_port_ipv4(PORT_NAME_NODE) + server_node.get_host_port_ipv4(PORT_NAME_NODE).await ); let client = hdfs_native::Client::new(&hdfs_server_url).expect("HDFS client to be created"); @@ -29,14 +26,12 @@ async fn e2e_nameode_hello() { #[tokio::test] #[serial_test::serial] async fn e2e_kerberos() { - let docker = clients::Cli::default(); - let container = MiniDFS::builder().with_kerberos(true).build(); - let kerberos_cache = container.inner().kerberos_cache(); - let kerberos_config = container.inner().kerberos_config(); - let hdfs_config = container.inner().hdfs_config(); + let kerberos_cache = container.image().kerberos_cache(); + let kerberos_config = container.image().kerberos_config(); + let hdfs_config = container.image().hdfs_config(); - let _server_node = docker.run(container); + let _server_node = container.start().await; assert!(kerberos_cache.is_some()); assert!(kerberos_config.is_some()); @@ -58,17 +53,17 @@ async fn e2e_kerberos() { #[tokio::test] #[serial_test::serial] async fn e2e_config_volume() { - let docker = clients::Cli::default(); let container = MiniDFS::builder().with_config_volume(true).build(); - let hdfs_config = container.inner().hdfs_config(); - let _server_node = docker.run(container); + let hdfs_config = container.image().hdfs_config(); + let _server_node = container.start().await; assert!(hdfs_config.is_some()); let hdfs_config = hdfs_config.unwrap(); info!("hdfs config: {:?}", hdfs_config); assert!(hdfs_config.exists()); } + #[cfg(test)] #[ctor::ctor] fn init() {