openebs · bors · Aug 18, 2023 · Aug 15, 2023 · Aug 15, 2023
diff --git a/io-engine-tests/src/fio.rs b/io-engine-tests/src/fio.rs
@@ -67,7 +67,7 @@ impl FioJob {
         let mut r = vec![
             format!("--name={}", self.name),
             format!("--ioengine={}", self.ioengine),
-            format!("--filename='{}'", self.filename),
+            format!("--filename={}", self.filename),
             format!("--thread=1"),
             format!("--direct={}", if self.direct { "1" } else { "0" }),
             format!("--norandommap=1"),
@@ -112,6 +112,12 @@ impl FioJob {
         self
     }
 
+    /// Read-write FIO mode.
+    pub fn with_rw(mut self, rw: &str) -> Self {
+        self.rw = rw.to_string();
+        self
+    }
+
     /// If true, use non-buffered I/O (usually O_DIRECT). Default: true.
     pub fn with_direct(mut self, v: bool) -> Self {
         self.direct = v;
@@ -171,8 +177,8 @@ impl Fio {
         Default::default()
     }
 
-    pub fn with_jobs(mut self, jobs: Vec<FioJob>) -> Self {
-        self.jobs = jobs;
+    pub fn with_jobs(mut self, jobs: impl Iterator<Item = FioJob>) -> Self {
+        jobs.for_each(|j| self.jobs.push(j));
         self
     }
 
@@ -201,12 +207,12 @@ impl Fio {
             .collect::<Vec<_>>()
             .join(" ");
 
+        let script = format!("{cmd} {args}");
+
         if self.verbose {
-            println!("{cmd} {args}");
+            println!("{script}");
         }
 
-        let script = format!("{cmd} {args}");
-
         let (exit, stdout, stderr) = run_script::run(
             &script,
             &Vec::new(),
@@ -215,11 +221,18 @@ impl Fio {
         .unwrap();
 
         if exit == 0 {
+            if self.verbose {
+                println!("FIO:");
+                println!("{script}");
+                println!("Output:");
+                println!("{stdout}");
+            }
+
             Ok(())
         } else {
             if self.verbose_err {
                 println!("Error running FIO:");
-                println!("{cmd} {args}");
+                println!("{script}");
                 println!("Exit code: {exit}");
                 println!("Output:");
                 println!("{stdout}");

diff --git a/io-engine-tests/src/nexus.rs b/io-engine-tests/src/nexus.rs
@@ -15,6 +15,7 @@ use super::{
             RebuildHistoryRecord,
             RebuildHistoryRequest,
             RemoveChildNexusRequest,
+            ShutdownNexusRequest,
         },
         SharedRpcHandle,
         Status,
@@ -179,6 +180,18 @@ impl NexusBuilder {
             .map(|r| r.into_inner().nexus.unwrap())
     }
 
+    pub async fn shutdown(&mut self) -> Result<(), Status> {
+        self.rpc()
+            .lock()
+            .await
+            .nexus
+            .shutdown_nexus(ShutdownNexusRequest {
+                uuid: self.uuid(),
+            })
+            .await
+            .map(|_| ())
+    }
+
     pub async fn destroy(&mut self) -> Result<(), Status> {
         self.rpc()
             .lock()

diff --git a/io-engine-tests/src/nvmf.rs b/io-engine-tests/src/nvmf.rs
@@ -77,7 +77,7 @@ pub async fn test_fio_to_nvmf(
     nvmf: &NvmfLocation,
     mut fio: Fio,
 ) -> std::io::Result<()> {
-    let tgt = nvmf.as_args().join(" ");
+    let tgt = format!("'{}'", nvmf.as_args().join(" "));
 
     fio.jobs = fio
         .jobs

diff --git a/io-engine/src/core/bdev.rs b/io-engine/src/core/bdev.rs
@@ -240,7 +240,9 @@ where
             Some(Protocol::Nvmf) => {
                 if let Some(ss) = NvmfSubsystem::nqn_lookup(self.name()) {
                     ss.stop().await.context(UnshareNvmf {})?;
-                    ss.destroy();
+                    unsafe {
+                        ss.shutdown_unsafe();
+                    }
                 }
             }
             Some(Protocol::Off) | None => {}

diff --git a/io-engine/src/core/env.rs b/io-engine/src/core/env.rs
@@ -164,6 +164,13 @@ pub struct MayastorCliArgs {
     #[structopt(short = "T", long = "tgt-iface", env = "NVMF_TGT_IFACE")]
     /// NVMF target interface (ip, mac, name or subnet).
     pub nvmf_tgt_interface: Option<String>,
+    /// NVMF target Command Retry Delay.
+    #[structopt(
+        long = "tgt-crdt",
+        env = "NVMF_TGT_CRDT",
+        default_value = "30"
+    )]
+    pub nvmf_tgt_crdt: u16,
     /// api Version
     #[structopt(
         long,
@@ -230,6 +237,7 @@ impl Default for MayastorCliArgs {
             nvme_ctl_io_ctx_pool_size: 65535,
             registration_endpoint: None,
             nvmf_tgt_interface: None,
+            nvmf_tgt_crdt: 30,
             api_versions: vec![ApiVersion::V0, ApiVersion::V1],
             diagnose_stack: None,
             reactor_freeze_detection: false,
@@ -328,6 +336,7 @@ pub struct MayastorEnvironment {
     bdev_io_ctx_pool_size: u64,
     nvme_ctl_io_ctx_pool_size: u64,
     nvmf_tgt_interface: Option<String>,
+    pub nvmf_tgt_crdt: u16,
     api_versions: Vec<ApiVersion>,
 }
 
@@ -372,6 +381,7 @@ impl Default for MayastorEnvironment {
             bdev_io_ctx_pool_size: 65535,
             nvme_ctl_io_ctx_pool_size: 65535,
             nvmf_tgt_interface: None,
+            nvmf_tgt_crdt: 30,
             api_versions: vec![ApiVersion::V0, ApiVersion::V1],
         }
     }
@@ -489,6 +499,7 @@ impl MayastorEnvironment {
             bdev_io_ctx_pool_size: args.bdev_io_ctx_pool_size,
             nvme_ctl_io_ctx_pool_size: args.nvme_ctl_io_ctx_pool_size,
             nvmf_tgt_interface: args.nvmf_tgt_interface,
+            nvmf_tgt_crdt: args.nvmf_tgt_crdt,
             api_versions: args.api_versions,
             ..Default::default()
         }

diff --git a/io-engine/src/subsys/config/opts.rs b/io-engine/src/subsys/config/opts.rs
@@ -27,6 +27,8 @@ use std::{
     str::FromStr,
 };
 
+use crate::core::MayastorEnvironment;
+
 pub trait GetOpts {
     fn get(&self) -> Self;
     fn set(&self) -> bool {
@@ -78,6 +80,8 @@ pub struct NvmfTgtConfig {
     pub name: String,
     /// the max number of namespaces this target should allow for
     pub max_namespaces: u32,
+    /// Command Retry Delay.
+    pub crdt: u16,
     /// TCP transport options
     pub opts: NvmfTcpTransportOpts,
 }
@@ -87,15 +91,18 @@ impl From<NvmfTgtConfig> for Box<spdk_nvmf_target_opts> {
         let mut out = Self::default();
         copy_str_with_null(&o.name, &mut out.name);
         out.max_subsystems = o.max_namespaces;
+        out.crdt[0] = o.crdt;
         out
     }
 }
 
 impl Default for NvmfTgtConfig {
     fn default() -> Self {
+        let args = MayastorEnvironment::global_or_default();
         Self {
             name: "mayastor_target".to_string(),
             max_namespaces: 2048,
+            crdt: args.nvmf_tgt_crdt,
             opts: NvmfTcpTransportOpts::default(),
         }
     }

diff --git a/io-engine/src/subsys/nvmf/subsystem.rs b/io-engine/src/subsys/nvmf/subsystem.rs
@@ -40,6 +40,7 @@ use spdk_rs::libspdk::{
     spdk_nvmf_subsystem_listener_get_trid,
     spdk_nvmf_subsystem_pause,
     spdk_nvmf_subsystem_remove_host,
+    spdk_nvmf_subsystem_remove_ns,
     spdk_nvmf_subsystem_resume,
     spdk_nvmf_subsystem_set_allow_any_host,
     spdk_nvmf_subsystem_set_ana_reporting,
@@ -185,7 +186,9 @@ impl NvmfSubsystem {
         ss.set_ana_reporting(false)?;
         ss.allow_any(false);
         if let Err(e) = ss.add_namespace(bdev, ptpl) {
-            ss.destroy();
+            unsafe {
+                ss.destroy_unsafe();
+            }
             return Err(e);
         }
         Ok(ss)
@@ -535,19 +538,31 @@ impl NvmfSubsystem {
         }
     }
 
-    /// destroy the subsystem
-    pub fn destroy(&self) -> i32 {
-        unsafe {
-            if (*self.0.as_ptr()).destroying {
-                warn!("Subsystem destruction already started");
-                return -libc::EALREADY;
-            }
-            spdk_nvmf_subsystem_destroy(
-                self.0.as_ptr(),
-                None,
-                std::ptr::null_mut(),
-            )
+    /// Removes the namespace and destroys the subsystem.
+    ///
+    /// # Safety
+    ///
+    /// The subsystem must paused or stopped.
+    pub unsafe fn shutdown_unsafe(&self) -> i32 {
+        if spdk_nvmf_subsystem_remove_ns(self.0.as_ptr(), 1) != 0 {
+            error!(?self, "failed to remove namespace while destroying");
         }
+
+        self.destroy_unsafe()
+    }
+
+    /// Destroys the SPDK object for subsystem.
+    ///
+    /// # Safety
+    ///
+    /// The subsystem must paused or stopped.
+    unsafe fn destroy_unsafe(&self) -> i32 {
+        if (*self.0.as_ptr()).destroying {
+            warn!("Subsystem destruction already started");
+            return -libc::EALREADY;
+        }
+
+        spdk_nvmf_subsystem_destroy(self.0.as_ptr(), None, std::ptr::null_mut())
     }
 
     /// Get NVMe subsystem's NQN
@@ -876,7 +891,9 @@ impl NvmfSubsystem {
                 e.to_string(),
             );
 
-            self.destroy();
+            unsafe {
+                self.shutdown_unsafe();
+            }
 
             Err(e)
         } else {

diff --git a/io-engine/src/target/nvmf.rs b/io-engine/src/target/nvmf.rs
@@ -26,7 +26,9 @@ where
 pub async fn unshare(uuid: &str) -> Result<(), NvmfError> {
     if let Some(ss) = NvmfSubsystem::nqn_lookup(uuid) {
         ss.stop().await?;
-        ss.destroy();
+        unsafe {
+            ss.shutdown_unsafe();
+        }
     }
     Ok(())
 }