From 3e9579114c1c0b6fda8aaf27caf0e91e4eabf208 Mon Sep 17 00:00:00 2001 From: Gang Liao Date: Fri, 7 Jan 2022 23:46:07 -0500 Subject: [PATCH] fix: enable arm64 lambda function (#414) * fix: enable arm64 lambda function * docs: update readme --- README.md | 297 ++++++++++++++++++++------------ benchmarks/src/nexmark/main.rs | 6 - benchmarks/src/ysb/main.rs | 3 - flock_bench.sh => configure | 56 +++--- flock-function/src/aws/main.rs | 3 + flock/src/configs/aws_lambda.rs | 17 +- flock/src/configs/flock.toml | 3 +- flock/src/services.rs | 25 ++- scripts/build_arm64.sh | 1 + 9 files changed, 264 insertions(+), 147 deletions(-) rename flock_bench.sh => configure (80%) diff --git a/README.md b/README.md index 8afc2115..e3ba1d65 100644 --- a/README.md +++ b/README.md @@ -9,26 +9,18 @@ The generic lambda function code is built in advance and uploaded to AWS S3. -| Service | Cloud Platform | S3 Bucket | S3 Key | Hardware | [YSB Bench](https://github.com/yahoo/streaming-benchmarks) | [NEXMark Bench](https://beam.apache.org/documentation/sdks/java/testing/nexmark/) | -| :-------: | :------------: | :-------: | :----: | :------: | :--------------------------------------------------------: | :-------------------------------------------------------------------------------: | -| **Flock** | AWS Lambda | flock-lab | flock | Arm, x86 | ✅ | ✅ | +| FaaS Service | AWS Lambda | GCP Functions | Azure Functions | Architectures | SIMD | [YSB](https://github.com/yahoo/streaming-benchmarks) | [NEXMark](https://beam.apache.org/documentation/sdks/java/testing/nexmark/) | +| :----------: | :--------: | :-----------: | :-------------: | :-----------: | :--: | :--------------------------------------------------: | :-------------------------------------------------------------------------: | +| **Flock** | 🏅🏅🏅🏅 | 👉 TBD | 👉 TBD | **Arm**, x86 | ✅ | ✅ | ✅ | ## Build From Source Code You can enable the features `simd` (to use SIMD instructions) and/or `mimalloc` or `snmalloc` (to use either the mimalloc or snmalloc allocator) as features by passing them in as --features: -```shell -# yum install -y openssl-devel gcc -$ cargo +nightly build --target x86_64-unknown-linux-gnu --release --features "simd snmalloc" -``` - -## Upgrade Cloud Function Services +To build and deploy Flock to AWS Lambda in one step, you can use the following command: -flock-cli is an interactive command-line query tool for Flock. - -```shell -$ cd target/x86_64-unknown-linux-gnu/release/ -$ ./flock-cli --help +```ignore +$ ./configure -c -a x86_64 ```
@@ -36,34 +28,64 @@ $ ./flock-cli --help Output -```shell -Flock 0.2.0 -UMD Database Group -Command Line Interactive Contoller for Flock +```ignore +============================================================ + Compiling and Deploying Benchmarks +============================================================ -USAGE: - flock-cli [OPTIONS] [SUBCOMMAND] +Building x86_64-unknown-linux-gnu -FLAGS: - -h, --help Prints help information - -V, --version Prints version information +[1/3] Compiling Flock Lambda Function... +[2/3] Compiling Flock CLI... +[3/3] Deploying Flock Lambda Function... -OPTIONS: - -c, --config Sets a custom config file +============================================================ + Upload function code to S3 +============================================================ + +Packaging code and uploading to S3... +[OK] Upload Succeed. -SUBCOMMANDS: - fsql The terminal-based front-end to Flock - help Prints this message or the help of the given subcommand(s) - nexmark The NEXMark Benchmark Tool - upload Uploads a function code to AWS S3 +============================================================ ```

+If you prefer to use the `cargo` command to build and deploy Flock, you can use the following commands: + +
+ +Commands + + +1. Build Flock for x86_64 + + ```ignore + $ cargo +nightly build --target x86_64-unknown-linux-gnu --release --features "simd mimalloc" + ``` + +2. Deploy Flock binary to AWS S3 + + ```ignore + $ cd ./target/x86_64-unknown-linux-gnu/release + $ ./flock-cli upload --path ./flock -key flock_x86_64 + ``` + +
+
+ ## Nexmark Benchmark -All the following Nexmark queries share the same lambda function code. +### Schemas + +These are multiple queries over a three entities model representing on online auction system: + +- **Person** represents a person submitting an item for auction and/or making a bid on an auction. +- **Auction** represents an item under auction. +- **Bid** represents a bid for an item under auction. + +### Queries | Query | Name | Summary | Flock | | -------------------------------------------------------------------------------------------------- | ------------------------------- | ------------------------------------------------------------------------------------------------------------------- | ----- | @@ -82,36 +104,14 @@ All the following Nexmark queries share the same lambda function code. | [q12](https://github.com/flock-lab/flock/blob/master/flock/src/datasource/nexmark/queries/q12.sql) | Processing Time Windows | How many bids does a user make within a fixed processing time limit? Illustrates working in processing time window. | ✅ | | [q13](https://github.com/flock-lab/flock/blob/master/flock/src/datasource/nexmark/queries/q13.sql) | Bounded Side Input Join | Joins a stream to a bounded side input, modeling basic stream enrichment. | ✅ | -We provide a script (`flock_bench.sh`) to build, deploy and run the benchmark. - -```shell -$ ./flock_bench.sh -help +> Note: q1 ~ q8 are from original NEXMark queries, q0 and q9 ~ q13 are from Apache Beam. -A Benchmark Script for Flock - -Syntax: flock_bench [-g|-h|-c|-r [-b ] [-q ] [-s ] [-e ] [-p ]] -options: -g Print the GPL license notification. -h Print this Help. -c Compile and deploy the benchmark. -r Run the benchmark. Default: false -b The type of the benchmark [nexmark, ysb]. Default: 'nexmark' -q NexMark Query Number [0-9]. Ignored if '-b' is not 'nexmark'. Default: 5 -p Number of Data Generators. Default: 1 -s Seconds to run the benchmark. Default: 10 -e Number of events per second. Default: 1000 -``` - -To build and deploy the benchmark, run: - -```shell -$ ./flock_bench.sh -c -``` +### Run the benchmark -For example, to run the query 5 for 10 seconds with 1,000 events per second, and 1 generator, you can run: +`flock-cli` is a command line interface to Flock. It can be used to run queries or benchmarks against Flock. For example, to run the query 5 on x86_64 for 10 seconds with 1,000 events per second, and 1 generator, you can run: -```shell -$ ./flock_bench.sh -r -b nexmark -q 5 -s 10 -e 1000 -p 1 +```ignore +$ ./flock-cli nexmark run -q 5 -s 10 -e 1000 -g 1 --arch x86_64 ```
@@ -119,56 +119,80 @@ $ ./flock_bench.sh -r -b nexmark -q 5 -s 10 -e 1000 -p 1 Client Output -```bash -============================================================ - Running the benchmark -============================================================ -Benchmark Type: NEXMARK -Query Number: 5 (ignored for YSB) -Generators: 1 -Events Per Second: 1000 -Seconds to Run: 10 -============================================================ - -[OK] Benchmark Starting - -[1] Warming up the lambda functions - -[2021-12-16T19:09:13Z INFO nexmark_bench] Running the NEXMark benchmark with the following options: NexmarkBenchmarkOpt { query_number: 5, debug: true, generators: 1, seconds: 10, events_per_second: 1000 } -[2021-12-16T19:09:13Z INFO nexmark_bench] Creating lambda function: flock_datasource -[2021-12-16T19:09:13Z INFO nexmark_bench] Creating lambda function group: Group(("q5-00", 8)) -[2021-12-16T19:09:13Z INFO nexmark_bench] Creating function member: q5-00-00 -[2021-12-16T19:09:14Z INFO nexmark_bench] Creating function member: q5-00-01 -[2021-12-16T19:09:14Z INFO nexmark_bench] Creating function member: q5-00-02 -[2021-12-16T19:09:15Z INFO nexmark_bench] Creating function member: q5-00-03 -[2021-12-16T19:09:15Z INFO nexmark_bench] Creating function member: q5-00-04 -[2021-12-16T19:09:16Z INFO nexmark_bench] Creating function member: q5-00-05 -[2021-12-16T19:09:16Z INFO nexmark_bench] Creating function member: q5-00-06 -[2021-12-16T19:09:17Z INFO nexmark_bench] Creating function member: q5-00-07 -[2021-12-16T19:09:17Z INFO nexmark_bench] [OK] Invoking NEXMark source function: flock_datasource by generator 0 -[2021-12-16T19:09:17Z INFO nexmark_bench] [OK] Received status from function. InvocationResponse { executed_version: None, function_error: None, log_result: None, payload: Some(b""), status_code: Some(202) } -[2021-12-16T19:09:17Z INFO nexmark_bench] Waiting for the current invocations to be logged. -[2021-12-16T19:09:22Z INFO driver::logwatch::tail] Sending log request FilterLogEventsRequest { end_time: None, filter_pattern: None, limit: Some(100), log_group_name: "/aws/lambda/flock_datasource", log_stream_name_prefix: None, log_stream_names: None, next_token: None, start_time: Some(1639681702541) } -[2021-12-16T19:09:24Z INFO driver::logwatch::tail] [OK] Got response from AWS CloudWatch Logs. -[2021-12-16T19:09:24Z INFO driver::deploy::common] Got a Token response -------------------------------------------------------------- - -[2] Running the benchmark - -[2021-12-16T19:09:26Z INFO nexmark_bench] Running the NEXMark benchmark with the following options: NexmarkBenchmarkOpt { query_number: 5, debug: true, generators: 1, seconds: 10, events_per_second: 1000 } -[2021-12-16T19:09:26Z INFO nexmark_bench] Creating lambda function: flock_datasource -[2021-12-16T19:09:26Z INFO nexmark_bench] Creating lambda function group: Group(("q5-00", 8)) -[2021-12-16T19:09:26Z INFO nexmark_bench] Creating function member: q5-00-00 -[2021-12-16T19:09:27Z INFO nexmark_bench] Creating function member: q5-00-01 -[2021-12-16T19:09:27Z INFO nexmark_bench] Creating function member: q5-00-02 -[2021-12-16T19:09:28Z INFO nexmark_bench] Creating function member: q5-00-03 -[2021-12-16T19:09:28Z INFO nexmark_bench] Creating function member: q5-00-04 -[2021-12-16T19:09:28Z INFO nexmark_bench] Creating function member: q5-00-05 -[2021-12-16T19:09:29Z INFO nexmark_bench] Creating function member: q5-00-06 -[2021-12-16T19:09:29Z INFO nexmark_bench] Creating function member: q5-00-07 -[2021-12-16T19:09:30Z INFO nexmark_bench] [OK] Invoking NEXMark source function: flock_datasource by generator 0 -[2021-12-16T19:09:30Z INFO nexmark_bench] [OK] Received status from function. InvocationResponse { executed_version: None, function_error: None, log_result: None, payload: Some(b""), status_code: Some(202) } -[2021-12-16T19:09:30Z INFO nexmark_bench] Waiting for the current invocations to be logged. +``````bash + /` +/:y/` ` +`shdhso. + -yhddh+. + .yhhhy+- + .syyhs+/. + `+shhs++:. + `:syyyo++/. + .+ssys+++/-` `.----.` + ./oyyyo+++/:.` `-/+++/-..` + -/osyso++++/:.` -/++/-` + .-/osssoo++++/:++++` + `.-/++osooo++++++++- + `-:/+oooo++++++o/ + `-:/+o++++++oo- ````` ` + `.-//++++++o/ `:++::::// .:++:` .:///////. .://///+- ./++:` .++/. + ``..:+++++o+` os` -+ ss `/+-`//. `-+/` `+s:` `o: `so `:+- + :+++++/` os` -- ss /o` `+o` `++ +s: `` `so .+:` + `:+++++: os:::o/ ss o/ /+++` :s ss. `ss/so` + .:++++:.` os` -- ss ` /o``+/``o/:`++ +s: ` `so .oo. + `.:/++++/. os` ss :+ /+:-` .-:+/` `+s:` o/ `so `+s: + .-----:/++- `:++:- .:++::::+/ .:++//++:. ./++///+- .:o+:` :o/: + `.-:::-/:` `` + `--.``-/:` + .:-` + + +Flock: A Practical Serverless Streaming SQL Query Engine (https://github.com/flock-lab/flock) + +Copyright (c) 2020-present, UMD Data System Group. + +▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ + +This program is free software: you can use, redistribute, and/or modify it under the terms of the GNU Affero +General Public License, version 3 or later ("AGPL"), as published by the Free Software Foundation. + +This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the +implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. + +You should have received a copy of the GNU Affero General Public License along with this program. If not, see +. + +▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒▒ + +================================================================ + Running the benchmark +================================================================ +[info] Running the NEXMark benchmark with the following options: + +NexmarkBenchmarkOpt { + query_number: 5, + generators: 1, + seconds: 10, + events_per_second: 1000, + data_sink_type: "blackhole", + async_type: false, + memory_size: 128, + architecture: "x86_64", +} + +[info] Creating lambda function: flock_datasource +[info] Creating lambda function group: Group(("q5-00", 8)) +[info] Creating function member: q5-00-00 +[info] Creating function member: q5-00-01 +[info] Creating function member: q5-00-02 +[info] Creating function member: q5-00-03 +[info] Creating function member: q5-00-04 +[info] Creating function member: q5-00-05 +[info] Creating function member: q5-00-06 +[info] Creating function member: q5-00-07 +[info] [OK] Invoking NEXMark source function: flock_datasource by generator 0 +[info] [OK] Received status from function. InvocationResponse { executed_version: None, function_error: None, log_result: None, payload: Some(b""), status_code: Some(202) } +[info] Waiting for the current invocations to be logged. [2021-12-16T19:09:35Z INFO driver::logwatch::tail] Sending log request FilterLogEventsRequest { end_time: None, filter_pattern: None, limit: Some(100), log_group_name: "/aws/lambda/flock_datasource", log_stream_name_prefix: None, log_stream_names: None, next_token: None, start_time: Some(1639681715344) } [2021-12-16T19:09:38Z INFO driver::logwatch::tail] [OK] Got response from AWS CloudWatch Logs. 2021-12-16 14:09:18 START RequestId: fd8ae2ad-b64a-4e02-88c8-c43f00974022 Version: $LATEST @@ -213,7 +237,7 @@ Seconds to Run: 10 ------------------------------------------------------------- [OK] Nexmark Benchmark Complete -``` +``````

@@ -283,6 +307,61 @@ START RequestId: 78c64a1a-b312-4099-b596-541c078b04b7 Version: $LATEST
+## Advanced Usage + +`flock-cli` has a number of advanced features that can be used to control and customize the behavior of Flock. + +For example, to delete all functions, you can use the `flock-cli lambda -D` command. Or use the `flock-cli lambda -d ` command to delete specific functions. To list all functions, use the `flock-cli lambda -L` command. + +To see the help for the `nexmark run` command, issue the command: `flock-cli nexmark run -h` + +```ignore +Runs the NEXMark Benchmark + +USAGE: + flock-cli nexmark run [OPTIONS] + +OPTIONS: + -a, --async-type + Runs the NEXMark benchmark with async function invocations + + -e, --events-per-second + Runs the NEXMark benchmark with a number of events per second [default: 1000] + + -g, --generators + Runs the NEXMark benchmark with a number of data generators [default: 1] + + -h, --help + Print help information + + --log-level + Log level [default: info] [possible values: error, warn, info, debug, trace, off] + + -m, --memory-size + Sets the memory size (MB) for the worker function [default: 128] + + -q, --query + Sets the NEXMark benchmark query number [default: 3] [possible values: 0, 1, 2, 3, 4, 5, + 6, 7, 8, 9, 10, 11, 12, 13] + + -r, --arch + Sets the architecture for the worker function [default: x86_64] [possible values: + x86_64, arm64] + + -s, --seconds + Runs the NEXMark benchmark for a number of seconds [default: 20] + + --silent + Suppress all output + + -t, --data-sink-type + Runs the NEXMark benchmark with a data sink type [default: blackhole] [possible values: + sqs, s3, dynamodb, efs, blackhole] + + --trace + Log ultra-verbose (trace level) information +``` + ## License Copyright (c) 2020-present UMD Database Group. diff --git a/benchmarks/src/nexmark/main.rs b/benchmarks/src/nexmark/main.rs index 94d0db57..46da0f4a 100644 --- a/benchmarks/src/nexmark/main.rs +++ b/benchmarks/src/nexmark/main.rs @@ -27,9 +27,7 @@ use log::info; use nexmark::event::{side_input_schema, Auction, Bid, Person}; use nexmark::NEXMarkSource; use rainbow::{rainbow_println, rainbow_string}; -use rusoto_core::Region; use rusoto_lambda::InvocationResponse; -use rusoto_s3::S3Client; use std::collections::HashMap; use std::sync::Arc; use structopt::StructOpt; @@ -43,10 +41,6 @@ static SIDE_INPUT_DOWNLOAD_URL: &str = concat!( ); lazy_static! { - pub static ref FLOCK_S3_KEY: String = FLOCK_CONF["s3"]["key"].to_string(); - pub static ref FLOCK_S3_BUCKET: String = FLOCK_CONF["s3"]["bucket"].to_string(); - pub static ref FLOCK_S3_CLIENT: S3Client = S3Client::new(Region::default()); - pub static ref FLOCK_EMPTY_PLAN: Arc = Arc::new(EmptyExec::new(false, Arc::new(Schema::empty()))); pub static ref FLOCK_CONCURRENCY: usize = FLOCK_CONF["lambda"]["concurrency"].parse::().unwrap(); diff --git a/benchmarks/src/ysb/main.rs b/benchmarks/src/ysb/main.rs index 905f5f3c..5e582cf0 100644 --- a/benchmarks/src/ysb/main.rs +++ b/benchmarks/src/ysb/main.rs @@ -31,9 +31,6 @@ use ysb::event::{AdEvent, Campaign}; use ysb::YSBSource; lazy_static! { - static ref FLOCK_S3_KEY: String = FLOCK_CONF["s3"]["key"].to_string(); - static ref FLOCK_S3_BUCKET: String = FLOCK_CONF["s3"]["bucket"].to_string(); - static ref FLOCK_EMPTY_PLAN: Arc = Arc::new(EmptyExec::new(false, Arc::new(Schema::empty()))); static ref FLOCK_CONCURRENCY: usize = FLOCK_CONF["lambda"]["concurrency"].parse::().unwrap(); diff --git a/flock_bench.sh b/configure similarity index 80% rename from flock_bench.sh rename to configure index 7eba5987..6423883f 100755 --- a/flock_bench.sh +++ b/configure @@ -62,21 +62,23 @@ Build_and_Deploy() { echo $(echogreen " Compiling and Deploying Benchmarks ") echo $(echogreen "============================================================") echo - echo $(echogreen "Building $arch") + echo $(echogreen "Building $target_arch") echo echo $(echogreen "[1/3]") $(echoblue "Compiling Flock Lambda Function...") + echo cd flock-function - cargo +nightly build --target $arch --release --features "simd mimalloc" + cargo +nightly build --target $target_arch --release --features "simd mimalloc" + cd .. echo echo $(echogreen "[2/3]") $(echoblue "Compiling Flock CLI...") - cd ../flock-cli - cargo +nightly build --target $arch --release + echo + cd flock-cli + cargo +nightly build --target $host_arch --release + cd .. echo echo $(echogreen "[3/3]") $(echoblue "Deploying Flock Lambda Function...") echo - cd ../target/$arch/release - ./flock-cli upload -p flock -k flock - cd ../../.. + ./target/$host_arch/release/flock-cli upload -p ./target/$target_arch/release/flock -k flock_$target echo echo $(echogreen "============================================================") echo @@ -101,7 +103,7 @@ Run() { ############################################################ # Process the input options. Add options as needed. # ############################################################ -arch="x86_64" +target="unknown" compile=false run=false @@ -123,7 +125,7 @@ while getopts "hgcra:" option; do run=true ;; a) # build the benchmark with specific architechture. - arch=$OPTARG + target=$OPTARG ;; \?) # Invalid option echo $(echored "Error: Invalid option") @@ -134,16 +136,28 @@ while getopts "hgcra:" option; do esac done -if [ "$arch" != "x86_64" ] && [ "$arch" != "arm64" ]; then - echo $(echored "Error: Invalid architechture. Please use \"flock-bench -a x86_64\" or \"flock-bench -a arm64\".") - echo - exit +host=$(uname -m) +if [ "$host" = "x86_64" ]; then + host_arch="x86_64-unknown-linux-gnu" +elif [ "$host" = "aarch64" ]; then + host_arch="aarch64-unknown-linux-gnu" +fi + +if [ "$target" != "x86_64" ] && [ "$target" != "arm64" ]; then + target_arch="$host_arch" + if [ "$host" = "x86_64" ]; then + target="x86_64" + elif [ "$host" = "aarch64" ]; then + target="arm64" + fi +elif [ "$target" == "x86_64" ]; then + target_arch="x86_64-unknown-linux-gnu" +elif [ "$target" == "arm64" ]; then + target_arch="aarch64-unknown-linux-gnu" fi -if [ "$arch" == "x86_64" ]; then - arch="x86_64-unknown-linux-gnu" -elif [ "$arch" == "arm64" ]; then - arch="aarch64-unknown-linux-gnu" +if [ "$target_arch" != "$host_arch" ]; then + export PKG_CONFIG_ALLOW_CROSS=1 fi if [ "$compile" = true ]; then @@ -154,6 +168,8 @@ if [ "$run" = true ]; then Run fi -echo -echo $(echored "Error: incomplete command line arguments, please use '-h' for help.") -echo +if [ "$compile" = false ] && [ "$run" = false ]; then + echo + echo $(echored "Error: incomplete command line arguments, please use '-h' for help.") + echo +fi diff --git a/flock-function/src/aws/main.rs b/flock-function/src/aws/main.rs index 23b5b498..017182ed 100644 --- a/flock-function/src/aws/main.rs +++ b/flock-function/src/aws/main.rs @@ -25,6 +25,7 @@ use flock::prelude::*; use hashring::HashRing; use lambda_runtime::{handler_fn, Context}; use lazy_static::lazy_static; +use log::info; use serde_json::Value; use std::cell::Cell; use std::sync::Once; @@ -118,6 +119,8 @@ macro_rules! init_exec_context { async fn handler(event: Payload, _: Context) -> Result { let (ctx, arena) = init_exec_context!(); + info!("Lambda function architecture: {}", std::env::consts::ARCH); + match &event.datasource { DataSource::Payload(_) => actor::handler(ctx, arena, event).await, DataSource::NEXMarkEvent(_) => nexmark::handler(ctx, event).await, diff --git a/flock/src/configs/aws_lambda.rs b/flock/src/configs/aws_lambda.rs index 6088586b..d14980e1 100644 --- a/flock/src/configs/aws_lambda.rs +++ b/flock/src/configs/aws_lambda.rs @@ -187,7 +187,7 @@ impl AwsLambdaConfig { // S3 bucket for the pre-compiled deployment package. s3_bucket: Some(FLOCK_CONF["s3"]["bucket"].to_string()), // S3 key for the pre-compiled deployment package. - s3_key: Some(FLOCK_CONF["s3"]["key"].to_string()), + s3_key: Some(FLOCK_CONF["s3"]["x86_64_key"].to_string()), s3_object_version: None, zip_file: None, image_uri: None, @@ -210,6 +210,21 @@ impl AwsLambdaConfig { }) } + /// Creates a new AWS Lambda function with the specified function code's s3 + /// key. + pub fn set_code(&mut self, key: &str) -> &mut Self { + self.code = FunctionCode { + // S3 bucket for the pre-compiled deployment package. + s3_bucket: Some(FLOCK_CONF["s3"]["bucket"].to_string()), + // S3 key for the pre-compiled deployment package. + s3_key: Some(key.to_string()), + s3_object_version: None, + zip_file: None, + image_uri: None, + }; + self + } + /// Creates a new AWS Lambda function with the specified function name. pub fn set_runtime(&mut self, runtime: &str) -> &mut Self { self.runtime = Some(runtime.to_string()); diff --git a/flock/src/configs/flock.toml b/flock/src/configs/flock.toml index a936c056..6a6e52af 100644 --- a/flock/src/configs/flock.toml +++ b/flock/src/configs/flock.toml @@ -12,7 +12,8 @@ production = false bucket = "flock-lab" # S3 key prefix to store the generic function code -key = "flock" +x86_64_key = "flock_x86_64" +arm_64_key = "flock_arm64" # AWS configuration [aws] diff --git a/flock/src/services.rs b/flock/src/services.rs index 53996ddf..4572fd66 100644 --- a/flock/src/services.rs +++ b/flock/src/services.rs @@ -53,8 +53,10 @@ lazy_static! { /// Flock async invocation granularity. pub static ref FLOCK_ASYNC_GRANULE_SIZE: usize = FLOCK_CONF["lambda"]["async_granule"].parse::().unwrap(); - /// Flock S3 key prefix. - pub static ref FLOCK_S3_KEY: String = FLOCK_CONF["s3"]["key"].to_string(); + /// Flock x86_64 binary S3 key prefix. + pub static ref FLOCK_S3_X86_64_KEY: String = FLOCK_CONF["s3"]["x86_64_key"].to_string(); + /// Flock Arm_64 binary S3 key prefix. + pub static ref FLOCK_S3_ARM_64_KEY: String = FLOCK_CONF["s3"]["arm_64_key"].to_string(); /// Flock S3 bucket name. pub static ref FLOCK_S3_BUCKET: String = FLOCK_CONF["s3"]["bucket"].to_string(); /// Flock availablity zone. @@ -214,6 +216,18 @@ pub async fn create_lambda_function( architecture: &str, ) -> Result { let func_name = ctx.name.clone(); + let flock_s3_key = if architecture == "x86_64" { + FLOCK_S3_X86_64_KEY.clone() + } else { + FLOCK_S3_ARM_64_KEY.clone() + }; + + let mut conf = AwsLambdaConfig::try_new().await?; + conf.set_memory_size(memory_size); + conf.set_function_spec(ctx); + conf.set_architectures(vec![architecture.to_string()]); + conf.set_code(&flock_s3_key); + if FLOCK_LAMBDA_CLIENT .get_function(GetFunctionRequest { function_name: ctx.name.clone(), @@ -224,9 +238,10 @@ pub async fn create_lambda_function( { let conf = FLOCK_LAMBDA_CLIENT .update_function_code(UpdateFunctionCodeRequest { + architectures: conf.architectures, function_name: func_name.clone(), s3_bucket: Some(FLOCK_S3_BUCKET.clone()), - s3_key: Some(FLOCK_S3_KEY.clone()), + s3_key: Some(flock_s3_key), ..Default::default() }) .await @@ -234,10 +249,6 @@ pub async fn create_lambda_function( conf.function_name .ok_or_else(|| FlockError::AWS("No function name!".to_string())) } else { - let mut conf = AwsLambdaConfig::try_new().await?; - conf.set_memory_size(memory_size); - conf.set_function_spec(ctx); - conf.set_architectures(vec![architecture.to_string()]); let resp = FLOCK_LAMBDA_CLIENT .create_function(CreateFunctionRequest { architectures: conf.architectures, diff --git a/scripts/build_arm64.sh b/scripts/build_arm64.sh index f465b9fd..beee1f14 100644 --- a/scripts/build_arm64.sh +++ b/scripts/build_arm64.sh @@ -26,6 +26,7 @@ sudo yum install -y openssl-devel llvm-devel curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y source $HOME/.cargo/env rustup target add --toolchain nightly aarch64-unknown-linux-gnu +rustup target add --toolchain nightly x86_64-unknown-linux-gnu # rustup target add --toolchain nightly aarch64-unknown-linux-musl # git clone git://git.musl-libc.org/musl