feat!: function code to s3 (#249)

* feat!: function code to s3 * feat: upload function to cloud * feat: zip lambda code * feat!: use Bzip2
flock-lab · Apr 18, 2021 · 70ddbee · 70ddbee
1 parent 6c558f0
commit 70ddbee
Show file tree

Hide file tree

Showing 6 changed files with 156 additions and 40 deletions.
diff --git a/.codecov.yml b/.codecov.yml
@@ -13,4 +13,5 @@ ignore:
   - "src/legacy"
   - "src/function/src/aws"
   - "src/driver/src/build"
+  - "src/bin/cli"
   - "**/*.md"
diff --git a/README.md b/README.md
@@ -10,19 +10,28 @@
 
 ## Nexmark Benchmark
 
-| Query  | Name | Summary | Squirtle |  Function Code (S3) |
-| -------- | -------- | -------- | ------ | ------ |
-| q0 | Pass Through | Measures the monitoring overhead including the source generator. | ✅ ||
-| q1 | Currency Conversion | Convert each bid value from dollars to euros. | ✅ ||
-| q2 | Selection | Find bids with specific auction ids and show their bid price. | ✅ ||
-| q3 | Local Item Suggestion | Who is selling in OR, ID or CA in category 10, and for what auction ids?  | ✅ ||
-| q4 | Average Price for a Category | Select the average of the wining bid prices for all auctions in each category. | ✅ ||
-| q5 | Hot Items | Which auctions have seen the most bids in the last period? | ✅ ||
-| q6 | Average Selling Price by Seller | What is the average selling price per seller for their last 10 closed auctions. | ✅ ||
-| q7 | Highest Bid | Select the bids with the highest bid price in the last period. | ✅ ||
-| q8 | Monitor New Users | Select people who have entered the system and created auctions in the last period. | ✅ ||
-| q9 | Winning Bids | Find the winning bid for each auction. | ✅ ||
-| q13 | Bounded Side Input Join | Joins a stream to a bounded side input, modeling basic stream enrichment. | ✅ ||
+The generic lambda function code is built in advance and uploaded to AWS S3.
+
+| Lambda Function Code    | S3 Bucket      | S3 Key           |    S3 URL                        |
+| ----------------------- | ---------------| ---------------- | -------------------------------- |
+| Nexmark Benchamark      | umd-squirtle   |   nexmark        |  s3://umd-squirtle/nexmark       |
+
+
+All the following Nexmark queries share the same lambda function code.
+
+| Query    | Name     | Summary  | Squirtle |
+| -------- | -------- | -------- | -------- |
+| q0 | Pass Through | Measures the monitoring overhead including the source generator. | ✅ |
+| q1 | Currency Conversion | Convert each bid value from dollars to euros. | ✅ |
+| q2 | Selection | Find bids with specific auction ids and show their bid price. | ✅ |
+| q3 | Local Item Suggestion | Who is selling in OR, ID or CA in category 10, and for what auction ids?  | ✅ |
+| q4 | Average Price for a Category | Select the average of the wining bid prices for all auctions in each category. | ✅ |
+| q5 | Hot Items | Which auctions have seen the most bids in the last period? | ✅ |
+| q6 | Average Selling Price by Seller | What is the average selling price per seller for their last 10 closed auctions. | ✅ |
+| q7 | Highest Bid | Select the bids with the highest bid price in the last period. | ✅ |
+| q8 | Monitor New Users | Select people who have entered the system and created auctions in the last period. | ✅ |
+| q9 | Winning Bids | Find the winning bid for each auction. | ✅ |
+| q13 | Bounded Side Input Join | Joins a stream to a bounded side input, modeling basic stream enrichment. | ✅ |
 
 ## License
 

diff --git a/src/bin/cli/Cargo.toml b/src/bin/cli/Cargo.toml
@@ -17,12 +17,15 @@ cli = [ "rustyline" ]
 [dependencies]
 clap = "2.33.3"
 ctrlc = "3.1.1"
-
+futures = "0.3.12"
 lazy_static = "1.4.0"
 runtime = { path = "../../runtime" }
+rusoto_core = "0.46.0"
+rusoto_s3 = "0.46.0"
 rustyline = { version = "8.0", optional = true }
 sqlparser = { version = "0.9.0", features = [ "json_example" ] }
 tokio = { version = "1.2", features = [ "macros", "io-util", "sync", "rt-multi-thread" ] }
+zip = "0.5.12"
 
 [[bin]]
 name = "squirtle-cli"

diff --git a/src/bin/cli/src/repl.rs b/src/bin/cli/src/repl.rs
@@ -13,9 +13,19 @@
 // limitations under the License.
 
 use clap::{crate_version, App, Arg};
-use runtime::error::Result;
+use futures::executor::block_on;
+use rusoto_core::Region;
+use rusoto_s3::PutObjectRequest;
+use rusoto_s3::{S3Client, S3};
 use rustyline::Editor;
 use std::env;
+use std::f64::consts::PI;
+use std::fs;
+use std::io::Write;
+use std::path::Path;
+
+type Error = Box<dyn std::error::Error + Sync + Send + 'static>;
+pub static S3_BUCKET: &str = "umd-squirtle";
 
 #[tokio::main]
 pub async fn main() {
@@ -24,27 +34,19 @@ pub async fn main() {
         .version(crate_version!())
         .about("Command Line Interactive Contoller for Squirtle")
         .arg(
-            Arg::with_name("config")
-                .short("c")
-                .long("config")
+            Arg::with_name("function_code")
+                .short("u")
+                .long("upload")
                 .value_name("FILE")
-                .help("Specify a custom config file to find AWS credentials configs")
-                .takes_value(true),
-        )
-        .arg(
-            Arg::with_name("exec")
-                .short("e")
-                .long("exec")
-                .value_name("COMMAND")
-                .help("Execute the specified command then exit")
+                .help("Upload lambda execution code to S3.")
                 .takes_value(true),
         )
         .arg(
-            Arg::with_name("filename")
-                .short("f")
-                .long("file")
-                .value_name("FILE")
-                .help("Read commands from the file rather than standard input")
+            Arg::with_name("function_key")
+                .short("k")
+                .long("key")
+                .value_name("STRING")
+                .help("AWS S3 key for this function code.")
                 .takes_value(true),
         )
         .arg(
@@ -55,9 +57,37 @@ pub async fn main() {
         )
         .get_matches();
 
-    println!("{}", include_str!("./squirtle.txt"));
-    let _config = matches.value_of("config").unwrap_or("default.conf");
+    rainbow_println(include_str!("./squirtle.txt"));
+
+    match matches.value_of("function_code") {
+        Some(bin_path) => {
+            rainbow_println("============================================");
+            rainbow_println("         Upload function code to S3         ");
+            rainbow_println("============================================\n\n");
+            if !std::path::Path::new(bin_path).exists() {
+                rainbow_println(&format!(
+                    "[ERROR]: function code '{}' doesn't exist!",
+                    bin_path
+                ));
+                rainbow_println("[EXIT]: ..............");
+                std::process::exit(-1);
+            }
+
+            if let Some(key) = matches.value_of("function_key") {
+                put_object_to_s3(S3_BUCKET, key, bin_path).unwrap();
+            } else {
+                rainbow_println("[ERROR]: AWS S3 key is missing!");
+                rainbow_println("[EXIT]: ..............");
+                std::process::exit(-1);
+            }
+        }
+        None => {
+            cli().await;
+        }
+    }
+}
 
+async fn cli() {
     let mut rl = Editor::<()>::new();
     rl.load_history(".history").ok();
 
@@ -95,7 +125,82 @@ fn is_exit_command(line: &str) -> bool {
     line == "quit" || line == "exit"
 }
 
-#[allow(unused_variables)]
-async fn exec_and_print(sql: String) -> Result<()> {
+async fn exec_and_print(_: String) -> Result<(), Error> {
+    rainbow_println("CLI is under construction. Please try Squirtle API directly.");
+    Ok(())
+}
+
+/// Prints the text in the rainbow fansion.
+pub fn rainbow_println(line: &str) {
+    let frequency: f64 = 0.1;
+    let spread: f64 = 3.0;
+    for (i, c) in line.char_indices() {
+        let (r, g, b) = rgb(frequency, spread, i as f64);
+        print!("\x1b[38;2;{};{};{}m{}\x1b[0m", r, g, b, c);
+    }
+    println!();
+}
+
+/// Generates RGB for rainbow print.
+fn rgb(freq: f64, spread: f64, i: f64) -> (u8, u8, u8) {
+    let j = i / spread;
+    let red = (freq * j + 0.0).sin() * 127.0 + 128.0;
+    let green = (freq * j + 2.0 * PI / 3.0).sin() * 127.0 + 128.0;
+    let blue = (freq * j + 4.0 * PI / 3.0).sin() * 127.0 + 128.0;
+
+    (red as u8, green as u8, blue as u8)
+}
+
+/// Puts a lambda function code to AWS S3.
+pub fn put_object_to_s3(bucket: &str, key: &str, obj_path: &str) -> Result<(), Error> {
+    rainbow_println("[UPLOAD] ▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒... ... ... ...");
+    // compress lambda function code to bootstrap.zip
+    let fname = Path::new(obj_path).parent().unwrap().join("bootstrap.zip");
+    let w = std::fs::File::create(&fname)?;
+    let mut zip = zip::ZipWriter::new(w);
+    let options =
+        zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Bzip2);
+    zip.start_file("bootstrap", options)?;
+    zip.write_all(&fs::read(&obj_path)?)?;
+    zip.finish()?;
+
+    if !fname.exists() {
+        rainbow_println(&format!(
+            "[ERROR]: failed to rename the binary {} to {:?}!",
+            obj_path, fname
+        ));
+        rainbow_println("[EXIT]: ..............");
+        std::process::exit(-1);
+    }
+
+    // uploads bootstrap.zip to AWS S3
+    if let Ok(bytes) = fs::read(&fname) {
+        let put_obj_req = PutObjectRequest {
+            bucket: String::from(bucket),
+            key: String::from(key),
+            body: Some(bytes.into()),
+            ..Default::default()
+        };
+
+        let client = S3Client::new(Region::default());
+        block_on(client.put_object(put_obj_req))?;
+        rainbow_println("[OK] Upload Succeed.");
+    } else {
+        rainbow_println(&format!("[ERROR]: failed to read {:?}!", fname));
+        rainbow_println("[EXIT]: ..............");
+        std::process::exit(-1);
+    }
+
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_rainbow_print() {
+        let text = include_str!("./squirtle.txt");
+        rainbow_println(text);
+    }
+}
diff --git a/src/bin/cli/src/squirtle.txt b/src/bin/cli/src/squirtle.txt
@@ -37,7 +37,6 @@
 
 Squirtle: Serverless SQL Query Engine for Real-Time and Batch Analytics (https://github.com/DSLAM-UMD/Squirtle)
 
-Copyright (c) 2020-2021, UMD Database Group. All rights reserved.
+Copyright (c) 2020-2021, UMD Data System Group. All rights reserved.
 
-This is the standard command line interactive contoller for Squirtle. This has all the right tools installed
-required to execute a query against local machines or cloud function services.
+▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒
diff --git a/src/function/Cargo.toml b/src/function/Cargo.toml
@@ -17,7 +17,6 @@ datafusion = { git = "https://github.com/DSLAM-UMD/arrow", branch = "scq", featu
 futures = "0.3.12"
 lambda_runtime = { git = "https://github.com/awslabs/aws-lambda-rust-runtime/", branch = "master" }
 lazy_static = "1.4"
-
 log = "0.4.14"
 rayon = "1.5"
 runtime = { path = "../runtime" }