Support the way to migrate whole databse with --dump {PATH} argument (

#977) Dump whole schemas and data by generating SQL with --dump {PATH} option gluesql --path ~/gluedata --dump ./dump.sql -- dump.sql CREATE TABLE User (id INT, name TEXT); CREATE INDEX User_id ON User (id); .. INSERT INTO User VALUES (1, 'Foo'), (2, 'Bar') .. .. Import database gluesql --path ~/newdata --execute ./dump.sql
gluesql · Nov 4, 2022 · e023fb2 · e023fb2
1 parent 1eb26c2
commit e023fb2
Show file tree

Hide file tree

Showing 15 changed files with 551 additions and 64 deletions.
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -2,9 +2,9 @@ name: Coverage
 
 on:
   push:
-    branches: [main]
+    branches: [main, release-*]
   pull_request:
-    branches: [main]
+    branches: [main, release-*]
 
 env:
   CARGO_TERM_COLOR: always

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -2,9 +2,9 @@ name: Rust
 
 on:
   push:
-    branches: [main]
+    branches: [main, release-*]
   pull_request:
-    branches: [main]
+    branches: [main, release-*]
 
 env:
   CARGO_TERM_COLOR: always

diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,7 @@
 # GlueSQL project files
 /data/
 /pkg/rust/data/
+/cli/tmp/
 /storages/**/data/
 /storages/**/tmp/
 /reports/

diff --git a/README.md b/README.md
@@ -32,12 +32,41 @@ GlueSQL provides three reference storage options.
 gluesql = "0.13"
 ```
 
-- CLI application
+- Install CLI
 
-```
+```sh
 $ cargo install gluesql
 ```
 
+- Run CLI
+
+```sh
+$ gluesql [--path ~/data_path] [--execute ~/sql_path]
+```
+
+### Migration using CLI
+
+#### Dump whole schemas and data by generating SQL using `--dump {PATH}` option
+
+```sh
+$ gluesql --path ~/glue_data --dump ./dump.sql
+```
+
+```sql
+-- dump.sql
+CREATE TABLE User (id INT, name TEXT);
+CREATE INDEX User_id ON User (id);
+..
+INSERT INTO User VALUES (1, 'Foo'), (2, 'Bar') ..
+..
+```
+
+#### Import database
+
+```sh
+$ gluesql --path ~/new_data --execute ./dump.sql
+```
+
 ### Usage
 
 ```rust

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
@@ -19,3 +19,10 @@ rustyline-derive = "0.6"
 tabled ="0.8"
 thiserror = "1.0"
 edit = "0.1.4"
+futures = "0.3"
+anyhow = "1.0"
+itertools = "0.10"
+
+[dev-dependencies]
+tokio = { version = "1", features = ["rt", "macros"] }
+
diff --git a/cli/src/lib.rs b/cli/src/lib.rs
@@ -7,11 +7,19 @@ mod print;
 
 use {
     crate::cli::Cli,
+    anyhow::{Error, Result},
     clap::Parser,
-    gluesql_core::store::{GStore, GStoreMut},
+    futures::executor::block_on,
+    gluesql_core::{
+        ast::{AstLiteral, Expr, SetExpr, Statement, ToSql, Values},
+        prelude::Row,
+        store::Transaction,
+        store::{GStore, GStoreMut, Store},
+    },
     gluesql_memory_storage::MemoryStorage,
     gluesql_sled_storage::SledStorage,
-    std::{fmt::Debug, path::PathBuf},
+    itertools::Itertools,
+    std::{fmt::Debug, fs::File, io::Write, path::PathBuf},
 };
 
 #[derive(Parser, Debug)]
@@ -24,14 +32,25 @@ struct Args {
     /// SQL file to execute
     #[clap(short, long, value_parser)]
     execute: Option<PathBuf>,
+
+    /// PATH to dump whole database
+    #[clap(short, long, value_parser)]
+    dump: Option<PathBuf>,
 }
 
-pub fn run() {
+pub fn run() -> Result<()> {
     let args = Args::parse();
 
     if let Some(path) = args.path {
         let path = path.as_path().to_str().expect("wrong path");
 
+        if let Some(dump_path) = args.dump {
+            let storage = SledStorage::new(path).expect("failed to load sled-storage");
+            dump_database(storage, dump_path)?;
+
+            return Ok::<_, Error>(());
+        }
+
         println!("[sled-storage] connected to {}", path);
         run(
             SledStorage::new(path).expect("failed to load sled-storage"),
@@ -56,4 +75,55 @@ pub fn run() {
             eprintln!("{}", e);
         }
     }
+
+    Ok(())
+}
+
+pub fn dump_database(storage: SledStorage, dump_path: PathBuf) -> Result<SledStorage> {
+    let file = File::create(dump_path)?;
+
+    block_on(async {
+        let (storage, _) = storage.begin(true).await.map_err(|(_, error)| error)?;
+        let schemas = storage.fetch_all_schemas().await?;
+        for schema in schemas {
+            writeln!(&file, "{}", schema.clone().to_ddl())?;
+
+            let rows_list = storage
+                .scan_data(&schema.table_name)
+                .await?
+                .map_ok(|(_, row)| row)
+                .chunks(100);
+
+            for rows in &rows_list {
+                let exprs_list = rows
+                    .map(|result| {
+                        result.map(|Row(values)| {
+                            values
+                                .into_iter()
+                                .map(|value| Ok(Expr::Literal(AstLiteral::try_from(value)?)))
+                                .collect::<Result<Vec<_>>>()
+                        })?
+                    })
+                    .collect::<Result<Vec<_>, _>>()?;
+
+                let insert_statement = Statement::Insert {
+                    table_name: schema.table_name.clone(),
+                    columns: Vec::new(),
+                    source: gluesql_core::ast::Query {
+                        body: SetExpr::Values(Values(exprs_list)),
+                        order_by: Vec::new(),
+                        limit: None,
+                        offset: None,
+                    },
+                }
+                .to_sql();
+
+                writeln!(&file, "{}", insert_statement)?;
+            }
+
+            writeln!(&file)?;
+        }
+
+        Ok(storage)
+    })
 }
diff --git a/cli/src/main.rs b/cli/src/main.rs
@@ -1,3 +1,3 @@
 fn main() {
-    gluesql_cli::run();
+    gluesql_cli::run().unwrap();
 }
diff --git a/cli/tests/dump.rs b/cli/tests/dump.rs
@@ -0,0 +1,60 @@
+use {
+    gluesql_cli::dump_database,
+    gluesql_core::{
+        prelude::Glue,
+        store::{Store, Transaction},
+    },
+    gluesql_sled_storage::{sled, SledStorage},
+    std::{fs::File, io::Read, path::PathBuf},
+};
+
+#[tokio::test]
+async fn dump_and_import() {
+    let data_path = "tmp/src";
+    let dump_path = PathBuf::from("tmp/dump.sql");
+
+    let config = sled::Config::default().path(data_path).temporary(true);
+    let source_storage = SledStorage::try_from(config).unwrap();
+    let mut source_glue = Glue::new(source_storage);
+
+    let sqls = vec![
+        "CREATE TABLE User (id INT, name TEXT);",
+        "CREATE INDEX User_id ON User (id);",
+        "INSERT INTO User SELECT N, 'a' FROM SERIES(101);",
+    ];
+
+    for sql in sqls {
+        source_glue.execute(sql).unwrap();
+    }
+
+    let sql = "SELECT * FROM User;";
+    let source_data = source_glue.execute(sql).unwrap();
+
+    let source_storage = dump_database(source_glue.storage.unwrap(), dump_path.clone()).unwrap();
+
+    let data_path = "tmp/target";
+    let config = sled::Config::default().path(data_path).temporary(true);
+    let target_storage = SledStorage::try_from(config).unwrap();
+    let mut target_glue = Glue::new(target_storage);
+
+    let mut sqls = String::new();
+    File::open(dump_path)
+        .unwrap()
+        .read_to_string(&mut sqls)
+        .unwrap();
+
+    for sql in sqls.split(';').filter(|sql| !sql.trim().is_empty()) {
+        target_glue.execute(sql).unwrap();
+    }
+
+    let target_data = target_glue.execute(sql).unwrap();
+    assert_eq!(source_data, target_data);
+
+    let (source_storage, _) = source_storage.begin(true).await.unwrap();
+    let source_schemas = source_storage.fetch_all_schemas().await.unwrap();
+
+    let (target_storage, _) = target_glue.storage.unwrap().begin(true).await.unwrap();
+    let target_schemas = target_storage.fetch_all_schemas().await.unwrap();
+
+    assert_eq!(source_schemas, target_schemas);
+}
diff --git a/core/src/ast/ddl.rs b/core/src/ast/ddl.rs
@@ -84,6 +84,7 @@ impl ToSql for ColumnDef {
                 .map(|ColumnOptionDef { option, .. }| option.to_sql())
                 .collect::<Vec<_>>()
                 .join(" ");
+
             format!("{name} {data_type} {options}")
                 .trim_end()
                 .to_owned()