feat: expose wal config (GreptimeTeam#852)

* feat: wal config * fix: use human-readable string in wal config * feat: copy ReadableSize and humanize size config items in toml files * fix: clippy
v0y4g3r · Jan 10, 2023 · 90fcaa8 · 90fcaa8
1 parent c609b19
commit 90fcaa8
Show file tree

Hide file tree

Showing 18 changed files with 442 additions and 49 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
@@ -1,12 +1,19 @@
 node_id = 42
 mode = 'distributed'
 rpc_addr = '127.0.0.1:3001'
-wal_dir = '/tmp/greptimedb/wal'
 rpc_runtime_size = 8
 mysql_addr = '127.0.0.1:4406'
 mysql_runtime_size = 4
 enable_memory_catalog = false
 
+[wal]
+dir = "/tmp/greptimedb/wal"
+file_size = '1GB'
+purge_interval = '10m'
+purge_threshold = '50GB'
+read_batch_size = 128
+sync_write = false
+
 [storage]
 type = 'File'
 data_dir = '/tmp/greptimedb/data/'

diff --git a/config/standalone.example.toml b/config/standalone.example.toml
@@ -1,12 +1,20 @@
 node_id = 0
 mode = 'standalone'
-wal_dir = '/tmp/greptimedb/wal/'
 enable_memory_catalog = false
 
 [http_options]
 addr = '127.0.0.1:4000'
 timeout = "30s"
 
+[wal]
+dir = "/tmp/greptimedb/wal"
+file_size = '1GB'
+purge_interval = '10m'
+purge_threshold = '50GB'
+read_batch_size = 128
+sync_write = false
+
+
 [storage]
 type = 'File'
 data_dir = '/tmp/greptimedb/data/'

diff --git a/src/cmd/src/datanode.rs b/src/cmd/src/datanode.rs
@@ -125,7 +125,7 @@ impl TryFrom<StartCommand> for DatanodeOptions {
         }
 
         if let Some(wal_dir) = cmd.wal_dir {
-            opts.wal_dir = wal_dir;
+            opts.wal.dir = wal_dir;
         }
         Ok(opts)
     }
@@ -134,6 +134,7 @@ impl TryFrom<StartCommand> for DatanodeOptions {
 #[cfg(test)]
 mod tests {
     use std::assert_matches::assert_matches;
+    use std::time::Duration;
 
     use datanode::datanode::ObjectStoreConfig;
     use servers::Mode;
@@ -151,7 +152,7 @@ mod tests {
         };
         let options: DatanodeOptions = cmd.try_into().unwrap();
         assert_eq!("127.0.0.1:3001".to_string(), options.rpc_addr);
-        assert_eq!("/tmp/greptimedb/wal".to_string(), options.wal_dir);
+        assert_eq!("/tmp/greptimedb/wal".to_string(), options.wal.dir);
         assert_eq!("127.0.0.1:4406".to_string(), options.mysql_addr);
         assert_eq!(4, options.mysql_runtime_size);
         let MetaClientOpts {
@@ -216,6 +217,11 @@ mod tests {
             ..Default::default()
         })
         .unwrap();
+        assert_eq!("/tmp/greptimedb/wal", dn_opts.wal.dir);
+        assert_eq!(Duration::from_secs(600), dn_opts.wal.purge_interval);
+        assert_eq!(1024 * 1024 * 1024, dn_opts.wal.file_size.0);
+        assert_eq!(1024 * 1024 * 1024 * 50, dn_opts.wal.purge_threshold.0);
+        assert!(!dn_opts.wal.sync_write);
         assert_eq!(Some(42), dn_opts.node_id);
         let MetaClientOpts {
             metasrv_addrs: metasrv_addr,

diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
@@ -16,7 +16,7 @@ use std::sync::Arc;
 
 use clap::Parser;
 use common_telemetry::info;
-use datanode::datanode::{Datanode, DatanodeOptions, ObjectStoreConfig};
+use datanode::datanode::{Datanode, DatanodeOptions, ObjectStoreConfig, WalConfig};
 use datanode::instance::InstanceRef;
 use frontend::frontend::{Frontend, FrontendOptions};
 use frontend::grpc::GrpcOptions;
@@ -73,7 +73,7 @@ pub struct StandaloneOptions {
     pub influxdb_options: Option<InfluxdbOptions>,
     pub prometheus_options: Option<PrometheusOptions>,
     pub mode: Mode,
-    pub wal_dir: String,
+    pub wal: WalConfig,
     pub storage: ObjectStoreConfig,
     pub enable_memory_catalog: bool,
 }
@@ -89,7 +89,7 @@ impl Default for StandaloneOptions {
             influxdb_options: Some(InfluxdbOptions::default()),
             prometheus_options: Some(PrometheusOptions::default()),
             mode: Mode::Standalone,
-            wal_dir: "/tmp/greptimedb/wal".to_string(),
+            wal: WalConfig::default(),
             storage: ObjectStoreConfig::default(),
             enable_memory_catalog: false,
         }
@@ -113,7 +113,7 @@ impl StandaloneOptions {
 
     fn datanode_options(self) -> DatanodeOptions {
         DatanodeOptions {
-            wal_dir: self.wal_dir,
+            wal: self.wal,
             storage: self.storage,
             enable_memory_catalog: self.enable_memory_catalog,
             ..Default::default()

diff --git a/src/common/base/Cargo.toml b/src/common/base/Cargo.toml
@@ -11,3 +11,6 @@ common-error = { path = "../error" }
 paste = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 snafu.workspace = true
+
+[dev-dependencies]
+toml = "0.5"
diff --git a/src/common/base/src/lib.rs b/src/common/base/src/lib.rs
@@ -15,5 +15,7 @@
 pub mod bit_vec;
 pub mod buffer;
 pub mod bytes;
+#[allow(clippy::all)]
+pub mod readable_size;
 
 pub use bit_vec::BitVec;