feat: Add a global TTL option for all tables (GreptimeTeam#1679)

* feat: Add a global TTL option for all tables * docs: update config examples * chore: print start command and options when standalone/frontend starts
v0y4g3r · May 31, 2023 · 9d17980 · 9d17980
1 parent 72b6bd1
commit 9d17980
Show file tree

Hide file tree

Showing 7 changed files with 26 additions and 1 deletion.
diff --git a/config/datanode.example.toml b/config/datanode.example.toml
@@ -36,6 +36,8 @@ sync_write = false
 [storage]
 type = "File"
 data_home = "/tmp/greptimedb/"
+# TTL for all tables. Disabled by default.
+# global_ttl = "7d"
 
 # Compaction options, see `standalone.example.toml`.
 [storage.compaction]

diff --git a/config/standalone.example.toml b/config/standalone.example.toml
@@ -97,6 +97,8 @@ sync_write = false
 type = "File"
 # Data directory, "/tmp/greptimedb/data" by default.
 data_home = "/tmp/greptimedb/"
+# TTL for all tables. Disabled by default.
+# global_ttl = "7d"
 
 # Compaction options.
 [storage.compaction]

diff --git a/src/cmd/src/frontend.rs b/src/cmd/src/frontend.rs
@@ -16,6 +16,7 @@ use std::sync::Arc;
 
 use clap::Parser;
 use common_base::Plugins;
+use common_telemetry::logging;
 use frontend::frontend::FrontendOptions;
 use frontend::instance::{FrontendInstance, Instance as FeInstance};
 use frontend::service_config::{InfluxdbOptions, PromOptions};
@@ -202,6 +203,9 @@ impl StartCommand {
     }
 
     async fn build(self, opts: FrontendOptions) -> Result<Instance> {
+        logging::info!("Frontend start command: {:#?}", self);
+        logging::info!("Frontend options: {:#?}", opts);
+
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
 
         let mut instance = FeInstance::try_new_distributed(&opts, plugins.clone())

diff --git a/src/cmd/src/standalone.rs b/src/cmd/src/standalone.rs
@@ -302,6 +302,7 @@ impl StartCommand {
     async fn build(self, fe_opts: FrontendOptions, dn_opts: DatanodeOptions) -> Result<Instance> {
         let plugins = Arc::new(load_frontend_plugins(&self.user_provider)?);
 
+        info!("Standalone start command: {:#?}", self);
         info!(
             "Standalone frontend options: {:#?}, datanode options: {:#?}",
             fe_opts, dn_opts

diff --git a/src/datanode/src/datanode.rs b/src/datanode/src/datanode.rs
@@ -54,6 +54,13 @@ pub enum ObjectStoreConfig {
 #[derive(Debug, Clone, Serialize, Deserialize, Default)]
 #[serde(default)]
 pub struct StorageConfig {
+    /// Retention period for all tables.
+    ///
+    /// Default value is `None`, which means no TTL.
+    ///
+    /// The precedence order is: ttl in table options > global ttl.
+    #[serde(with = "humantime_serde")]
+    pub global_ttl: Option<Duration>,
     #[serde(flatten)]
     pub store: ObjectStoreConfig,
     pub compaction: CompactionConfig,
@@ -300,6 +307,7 @@ impl From<&DatanodeOptions> for StorageEngineConfig {
             picker_schedule_interval: value.storage.flush.picker_schedule_interval,
             auto_flush_interval: value.storage.flush.auto_flush_interval,
             global_write_buffer_size: value.storage.flush.global_write_buffer_size,
+            global_ttl: value.storage.global_ttl,
         }
     }
 }

diff --git a/src/storage/src/config.rs b/src/storage/src/config.rs
@@ -46,6 +46,10 @@ pub struct EngineConfig {
     pub auto_flush_interval: Duration,
     /// Limit for global write buffer size. Disabled by default.
     pub global_write_buffer_size: Option<ReadableSize>,
+    /// Global retention period for all regions.
+    ///
+    /// The precedence order is: region ttl > global ttl.
+    pub global_ttl: Option<Duration>,
 }
 
 impl Default for EngineConfig {
@@ -65,6 +69,7 @@ impl Default for EngineConfig {
             ),
             auto_flush_interval: Duration::from_millis(DEFAULT_AUTO_FLUSH_INTERVAL.into()),
             global_write_buffer_size: None,
+            global_ttl: None,
         }
     }
 }
diff --git a/src/storage/src/engine.rs b/src/storage/src/engine.rs
@@ -465,7 +465,7 @@ impl<S: LogStore> EngineInner<S> {
         write_buffer_size: Option<usize>,
         region_name: &str,
         config: &EngineConfig,
-        ttl: Option<Duration>,
+        region_ttl: Option<Duration>,
         compaction_time_window: Option<i64>,
     ) -> Result<StoreConfig<S>> {
         let parent_dir = util::normalize_dir(parent_dir);
@@ -483,6 +483,9 @@ impl<S: LogStore> EngineInner<S> {
         manifest.start().await?;
         let flush_strategy = self.flush_strategy.clone();
 
+        // If region_ttl is `None`, the global ttl takes effect.
+        let ttl = region_ttl.or(self.config.global_ttl);
+
         Ok(StoreConfig {
             log_store: self.log_store.clone(),
             sst_layer,