From 44dfa9b08048897a9de5f025b672bbda3eee6479 Mon Sep 17 00:00:00 2001 From: Iulian Barbu <14218860+iulianbarbu@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:39:38 +0200 Subject: [PATCH 01/23] misc(auth): added service healthcheck (#1394) * auth: added service healthcheck * address feedback, shorten interval * auth: healthcheck runs forever, and shouldn't be... ...used necessarily for startup --- docker-compose.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index c3729dc7f..f6bc7e10c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,13 @@ services: - "start" - "--address=0.0.0.0:8000" - "--stripe-secret-key=${STRIPE_SECRET_KEY}" + healthcheck: + test: curl --fail http://localhost:8000/ || exit 1 + interval: 1m + timeout: 10s + retries: 5 + start_period: 10s + start_interval: 2s builder: image: "${CONTAINER_REGISTRY}/builder:${BUILDER_TAG}" depends_on: From 8000d1eb2fb59cb96bd1c408e2fc3356eaa8a77c Mon Sep 17 00:00:00 2001 From: HardikBandhiya Date: Sun, 19 Nov 2023 03:33:05 +0530 Subject: [PATCH 02/23] docs(readme): update the alt text for Twitter page (#1404) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b65eb3817..168270053 100644 --- a/README.md +++ b/README.md @@ -168,7 +168,7 @@ Sign up to the [Algora Console](https://console.algora.io/org/shuttle/bounties?s ## Community and Support - [GitHub Issues](https://github.com/shuttle-hq/shuttle/issues). Best for: bugs and errors you encounter using Shuttle. -- [Twitter](https://twitter.com/shuttle_dev). Best for: keeping up with announcements, releases, collaborations and other events. +- [X (Twitter)](https://twitter.com/shuttle_dev). Best for: keeping up with announcements, releases, collaborations and other events. - [Discord](https://discord.gg/shuttle). Best for: *ALL OF THE ABOVE* + help, support, sharing your applications and hanging out with the community. ## Project Status From 5f5501af3fb4042f6c12a5c25a6e37329a942aab Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:50:15 +0100 Subject: [PATCH 03/23] fix: merge new&old secrets in deployer (#1407) --- deployer/src/deployment/run.rs | 17 +++++++++++++++-- runtime/src/alpha/mod.rs | 2 ++ 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/deployer/src/deployment/run.rs b/deployer/src/deployment/run.rs index d0b7bbccb..3322fc214 100644 --- a/deployer/src/deployment/run.rs +++ b/deployer/src/deployment/run.rs @@ -15,7 +15,7 @@ use shuttle_common::{ DEPLOYER_END_MSG_COMPLETED, DEPLOYER_END_MSG_CRASHED, DEPLOYER_END_MSG_STARTUP_ERR, DEPLOYER_END_MSG_STOPPED, DEPLOYER_RUNTIME_START_RESPONSE, }, - resource, + resource, SecretStore, }; use shuttle_proto::{ resource_recorder::record_request, @@ -315,7 +315,7 @@ async fn load( mut resource_manager: impl ResourceManager, mut runtime_client: RuntimeClient>>, claim: Claim, - secrets: HashMap, + mut secrets: HashMap, ) -> Result<()> { info!("Loading resources"); @@ -336,6 +336,19 @@ async fn load( }) .ok() }) + // inject old secrets into the secrets added in this deployment + .inspect(|r| { + if r.r#type == shuttle_common::resource::Type::Secrets { + match serde_json::from_value::(r.data.clone()) { + Ok(ss) => { + secrets.extend(ss.into_iter()); + } + Err(err) => { + error!(error = ?err, "failed to parse old secrets data"); + } + } + } + }) .map(resource::Response::into_bytes) .collect(); diff --git a/runtime/src/alpha/mod.rs b/runtime/src/alpha/mod.rs index 8c55b9989..0b6d294d4 100644 --- a/runtime/src/alpha/mod.rs +++ b/runtime/src/alpha/mod.rs @@ -218,6 +218,8 @@ where let provisioner_client = ProvisionerClient::new(channel); + // TODO: merge new & old secrets + let past_resources = resources .into_iter() .map(resource::Response::from_bytes) From d9c015c9e89032ab3b8ad1fe6529b32468596a57 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Mon, 20 Nov 2023 17:24:00 +0100 Subject: [PATCH 04/23] chore: cargo update (#1391) * chore: cargo update * cargo update agane * again * fix: lock rustrict * refactor: better feature scope for rustrict --- Cargo.lock | 310 +++++++++++++++++++++++----------------------- common/Cargo.toml | 5 +- 2 files changed, 161 insertions(+), 154 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 27ea2aefb..f0f12429b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,7 +24,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if 1.0.0", - "getrandom 0.2.10", + "getrandom 0.2.11", "once_cell", "version_check", "zerocopy", @@ -264,7 +264,7 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -295,7 +295,7 @@ checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -968,9 +968,9 @@ dependencies = [ [[package]] name = "bstr" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79ad7fb2dd38f3dabd76b09c6a5a20c038fc0213ef1e9afd30eb777f120f019" +checksum = "542f33a8835a0884b006a0c3df3dadd99c0c3f296ed26c2fdc8028e01ad6230c" dependencies = [ "memchr", "regex-automata 0.4.3", @@ -1006,9 +1006,9 @@ checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "bytes-utils" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e47d3a8076e283f3acd27400535992edb3ba4b5bb72f8891ad8fbe7932a7d4b9" +checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" dependencies = [ "bytes", "either", @@ -1112,9 +1112,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12024c4645c97566567129c204f65d5815a8c9aecf30fcbe682b2fe034996d36" +checksum = "e34637b3140142bdf929fb439e8aa4ebad7651ebf7b1080b3930aa16ac1459ff" dependencies = [ "serde", ] @@ -1166,7 +1166,7 @@ dependencies = [ "tokio", "tokio-tungstenite", "tokiotest-httpserver", - "toml 0.8.6", + "toml 0.8.8", "toml_edit 0.20.7", "tonic 0.10.2", "tracing", @@ -1258,9 +1258,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" +checksum = "2275f18819641850fa26c89acc84d465c1bf91ce57bc2748b28c420473352f64" dependencies = [ "clap_builder", "clap_derive", @@ -1268,9 +1268,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.7" +version = "4.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" +checksum = "07cdf1b148b25c1e1f7a42225e30a0d99a615cd4637eae7365548dd4529b95bc" dependencies = [ "anstream", "anstyle", @@ -1296,7 +1296,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -1566,9 +1566,9 @@ dependencies = [ [[package]] name = "crc-catalog" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4939f9ed1444bd8c896d37f3090012fa6e7834fe84ef8c9daa166109515732f9" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32fast" @@ -1710,7 +1710,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37e366bff8cd32dd8754b0991fb66b279dc48f598c3a18914852a6673deef583" dependencies = [ "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -1939,7 +1939,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2005,7 +2005,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2016,9 +2016,9 @@ checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" [[package]] name = "errno" -version = "0.3.5" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" +checksum = "f258a7194e7f7c2a7837a8913aeab7fd8c383457034fa20ce4dd3dcb813e8eb8" dependencies = [ "libc", "windows-sys 0.48.0", @@ -2241,7 +2241,7 @@ checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -2328,9 +2328,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +checksum = "fe9006bed769170c11f845cf00c7c1e9092aeb3f268e007c3e760ac68008070f" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -2713,7 +2713,7 @@ checksum = "9d8acb5ee668d55f0f2d19a320a3f9ef67a6999ad483e11135abcc2464ed18b6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -3127,9 +3127,9 @@ checksum = "92620684d99f750bae383ecb3be3748142d6095760afd5cbcf2261e9a279d780" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "4d6250322ef6e60f93f9a2162799302cd6f68f79f6e5d85c8c16f14d1d958178" dependencies = [ "bytes", "fnv", @@ -3137,7 +3137,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -3273,9 +3273,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" dependencies = [ "bytes", "fnv", @@ -3744,9 +3744,9 @@ checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" [[package]] name = "libgit2-sys" @@ -3766,6 +3766,17 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "libredox" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +dependencies = [ + "bitflags 2.4.1", + "libc", + "redox_syscall 0.4.1", +] + [[package]] name = "libsqlite3-sys" version = "0.26.0" @@ -3797,9 +3808,9 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" [[package]] name = "lock_api" @@ -4462,7 +4473,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -4610,9 +4621,9 @@ checksum = "794b5bf8e2d19b53dcdcec3e4bba628e20f5b6062503ba89281fa7037dd7bbcf" [[package]] name = "proptest" -version = "1.3.1" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c003ac8c77cb07bb74f5f198bce836a689bcd5a42574612bf14d17bfd08c20e" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" dependencies = [ "bit-set", "bit-vec", @@ -4622,7 +4633,7 @@ dependencies = [ "rand 0.8.5", "rand_chacha 0.3.1", "rand_xorshift", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", "rusty-fork", "tempfile", "unarray", @@ -4640,12 +4651,12 @@ dependencies = [ [[package]] name = "prost" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fdd22f3b9c31b53c060df4a0613a1c7f062d4115a2b984dd15b1858f7e340d" +checksum = "5a5a410fc7882af66deb8d01d01737353cf3ad6204c408177ba494291a626312" dependencies = [ "bytes", - "prost-derive 0.12.1", + "prost-derive 0.12.2", ] [[package]] @@ -4663,24 +4674,24 @@ dependencies = [ [[package]] name = "prost-derive" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" +checksum = "065717a5dfaca4a83d2fe57db3487b311365200000551d7a364e715dbf4346bc" dependencies = [ "anyhow", "itertools 0.11.0", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] name = "prost-types" -version = "0.12.1" +version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e081b29f63d83a4bc75cfc9f3fe424f9156cf92d8a4f0c9407cce9a1b67327cf" +checksum = "8339f32236f590281e2f6368276441394fcd1b2133b549cc895d0ae80f2f9a52" dependencies = [ - "prost 0.12.1", + "prost 0.12.2", ] [[package]] @@ -4789,7 +4800,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", ] [[package]] @@ -4877,12 +4888,12 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b" +checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" dependencies = [ - "getrandom 0.2.10", - "redox_syscall 0.2.16", + "getrandom 0.2.11", + "libredox", "thiserror", ] @@ -4937,12 +4948,6 @@ version = "0.6.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" -[[package]] -name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - [[package]] name = "regex-syntax" version = "0.8.2" @@ -5016,7 +5021,7 @@ dependencies = [ "async-trait", "chrono", "futures", - "getrandom 0.2.10", + "getrandom 0.2.11", "http", "hyper", "parking_lot 0.11.2", @@ -5085,7 +5090,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb0205304757e5d899b9c2e448b867ffd03ae7f988002e47cd24954391394d0b" dependencies = [ "cc", - "getrandom 0.2.10", + "getrandom 0.2.11", "libc", "spin 0.9.8", "untrusted 0.9.0", @@ -5155,7 +5160,7 @@ dependencies = [ "quote", "rust-embed-utils", "shellexpand", - "syn 2.0.38", + "syn 2.0.39", "walkdir", ] @@ -5220,9 +5225,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.21" +version = "0.38.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" +checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" dependencies = [ "bitflags 2.4.1", "errno", @@ -5235,9 +5240,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.8" +version = "0.21.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "446e14c5cda4f3f30fe71863c34ec70f5ac79d6087097ad0bb433e1be5edf04c" +checksum = "629648aced5775d558af50b2b4c7b02983a04b312126d45eeead26e7caa498b9" dependencies = [ "log", "ring 0.17.5", @@ -5259,9 +5264,9 @@ dependencies = [ [[package]] name = "rustls-pemfile" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ "base64 0.21.5", ] @@ -5400,9 +5405,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.190" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" +checksum = "bca2a08484b285dcb282d0f67b26cadc0df8b19f8c12502c13d966bf9482f001" dependencies = [ "serde_derive", ] @@ -5418,13 +5423,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.190" +version = "1.0.192" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" +checksum = "d6c7207fbec9faa48073f3e3074cbe553af6ea512d7c21ba46e434e70ea9fbc1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -5479,7 +5484,7 @@ checksum = "3081f5ffbb02284dda55132aa26daecedd7372a42417bbbab6f14ab7d6bb9145" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -5629,7 +5634,7 @@ dependencies = [ "serde_json", "shuttle-common", "tokio", - "toml 0.8.6", + "toml 0.8.8", "tracing", "tracing-subscriber", ] @@ -5684,7 +5689,7 @@ dependencies = [ "tempfile", "thiserror", "tokio", - "toml 0.8.6", + "toml 0.8.8", "tonic 0.10.2", "tracing", "tracing-subscriber", @@ -5703,7 +5708,7 @@ dependencies = [ "serde", "serde_json", "shuttle-common-tests", - "syn 2.0.38", + "syn 2.0.39", "tokio", "trybuild", ] @@ -5814,7 +5819,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", - "toml 0.8.6", + "toml 0.8.8", "tonic 0.10.2", "tower", "tower-http 0.4.4", @@ -5924,7 +5929,7 @@ dependencies = [ "dunce", "futures-core", "home", - "prost 0.12.1", + "prost 0.12.2", "prost-types", "serde_json", "shuttle-common", @@ -5946,7 +5951,7 @@ dependencies = [ "mongodb", "once_cell", "portpicker", - "prost 0.12.1", + "prost 0.12.2", "rand 0.8.5", "serde_json", "shuttle-common", @@ -6028,7 +6033,7 @@ dependencies = [ "strfmt", "thiserror", "tokio", - "toml 0.8.6", + "toml 0.8.8", "tracing", ] @@ -6064,9 +6069,9 @@ dependencies = [ [[package]] name = "signature" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e1788eed21689f9cf370582dfc467ef36ed9c707f073528ddafa8d83e3b8500" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ "digest 0.10.7", "rand_core 0.6.4", @@ -6101,9 +6106,9 @@ checksum = "826167069c09b99d56f31e9ae5c99049e932a98c9dc2dac47645b08dbbf76ba7" [[package]] name = "smallvec" -version = "1.11.1" +version = "1.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "4dccd0940a2dcdf68d092b8cbab7dc0ad8fa938bf95787e1b916b0e3d0e8e970" [[package]] name = "smart-default" @@ -6476,7 +6481,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -6498,9 +6503,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.38" +version = "2.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" +checksum = "23e78b90f2fcf45d3e842032ce32e3f2d1545ba6636271dcbf24fa306d87be7a" dependencies = [ "proc-macro2", "quote", @@ -6626,9 +6631,9 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6093bad37da69aab9d123a8091e4be0aa4a03e4d601ec641c327398315f62b64" +checksum = "ff1bc3d3f05aff0403e8ac0d92ced918ec05b666a43f83297ccef5bea8a3d449" dependencies = [ "winapi-util", ] @@ -6677,7 +6682,7 @@ checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -6738,9 +6743,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.33.0" +version = "1.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" +checksum = "d0c014766411e834f7af5b8f4cf46257aab4036ca95e9d2c144a10f59ad6f5b9" dependencies = [ "backtrace", "bytes", @@ -6767,13 +6772,13 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -6880,14 +6885,14 @@ dependencies = [ [[package]] name = "toml" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ff9e3abce27ee2c9a37f9ad37238c1bdd4e789c84ba37df76aa4d528f5072cc" +checksum = "a1a195ec8c9da26928f773888e0742ca3ca1040c6cd859c919c9f59c1954ab35" dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit 0.20.7", + "toml_edit 0.21.0", ] [[package]] @@ -6917,6 +6922,17 @@ name = "toml_edit" version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" +dependencies = [ + "indexmap 2.1.0", + "toml_datetime", + "winnow", +] + +[[package]] +name = "toml_edit" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d34d383cd00a163b4a5b85053df514d45bc330f6de7737edfe0a93311d1eaa03" dependencies = [ "indexmap 2.1.0", "serde", @@ -6971,7 +6987,7 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost 0.12.1", + "prost 0.12.2", "tokio", "tokio-stream", "tower", @@ -7083,7 +7099,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -7107,17 +7123,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "tracing-log" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - [[package]] name = "tracing-log" version = "0.2.0" @@ -7142,16 +7147,16 @@ dependencies = [ "smallvec", "tracing", "tracing-core", - "tracing-log 0.2.0", + "tracing-log", "tracing-subscriber", "web-time", ] [[package]] name = "tracing-subscriber" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30a651bc37f915e81f087d86e62a18eec5f79550c7faff886f7090b4ea757c77" +checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ "matchers", "nu-ansi-term", @@ -7162,7 +7167,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log 0.1.4", + "tracing-log", ] [[package]] @@ -7356,9 +7361,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-bom" -version = "2.0.2" +version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98e90c70c9f0d4d1ee6d0a7d04aa06cb9bbd53d8cfbdd62a0269a7c2eb640552" +checksum = "7eec5d1121208364f6793f7d2e222bf75a915c19557537745b195b253dd64217" [[package]] name = "unicode-ident" @@ -7452,9 +7457,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "utoipa" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b208a50ff438dcdc887ea3f2db59530bd2f4bc3d2c70630e4d7ee7a281a1d1b" +checksum = "0ff05e3bac2c9428f57ade702667753ca3f5cf085e2011fe697de5bfd49aa72d" dependencies = [ "indexmap 2.1.0", "serde", @@ -7464,14 +7469,14 @@ dependencies = [ [[package]] name = "utoipa-gen" -version = "4.0.0" +version = "4.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bd516d8879043e081537690bc96c8f17b5a4602c336aecb8f1de89d9d9c7e72" +checksum = "5f0b6f4667edd64be0e820d6631a60433a269710b6ee89ac39525b872b76d61d" dependencies = [ "proc-macro-error", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "uuid", ] @@ -7497,7 +7502,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88ad59a7560b41a70d191093a945f0b87bc1deeda46fb237479708a1d6b6cdfc" dependencies = [ - "getrandom 0.2.10", + "getrandom 0.2.11", "serde", ] @@ -7636,7 +7641,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-shared", ] @@ -7670,7 +7675,7 @@ checksum = "c5353b8dab669f5e10f5bd76df26a9360c748f054f862ff5f3f8aae0c7fb3907" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -7692,9 +7697,9 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.36.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53ae0be20bf87918df4fa831bfbbd0b491d24aee407ed86360eae4c2c5608d38" +checksum = "7d135e8940b69dbee0f5b0a0be9c1cd6fa8b71d774904c13a3fcfc5dc265e43d" dependencies = [ "leb128", ] @@ -7726,22 +7731,23 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.116.0" +version = "0.117.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53290b1276c5c2d47d694fb1a920538c01f51690e7e261acbe1d10c5fc306ea1" +checksum = "9b206de0c992af9f0b51ef2fb9455623e0a19eb68f172cd8ba9cd0e46637f5ab" dependencies = [ + "hashbrown 0.14.2", "indexmap 2.1.0", "semver 1.0.20", ] [[package]] name = "wasmprinter" -version = "0.2.71" +version = "0.2.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f98260aa20f939518bcec1fac32c78898d5c68872e7363a4651f21f791b6c7e" +checksum = "7a4fdb34710b461c868c3f79a10a48b404f23b46fd471ab02bcaa60fd96c5c4b" dependencies = [ "anyhow", - "wasmparser 0.116.0", + "wasmparser 0.117.0", ] [[package]] @@ -7822,7 +7828,7 @@ dependencies = [ "anyhow", "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wasmtime-component-util", "wasmtime-wit-bindgen", "wit-parser", @@ -8013,7 +8019,7 @@ checksum = "6362c557c36d8ad4aaab735f14ed9e4f78d6b40ec85a02a88fd859af87682e52" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] @@ -8096,23 +8102,23 @@ dependencies = [ [[package]] name = "wast" -version = "67.0.0" +version = "68.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36c2933efd77ff2398b83817a98984ffe4b67aefd9aa1d2c8e68e19b553f1c38" +checksum = "7bf3081ac6bcb3a5b72a401693b3566feb529dc2b7e7b62ea544c8a30d0f4d05" dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder 0.36.1", + "wasm-encoder 0.37.0", ] [[package]] name = "wat" -version = "1.0.78" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c02905d13751dcb18f4e19f489d37a1bf139f519feaeef28d072a41a78e69a74" +checksum = "6fabe07d22a837b3bd5662ba9e980d73de115c040923659a1801934c7ccebe49" dependencies = [ - "wast 67.0.0", + "wast 68.0.0", ] [[package]] @@ -8205,7 +8211,7 @@ dependencies = [ "proc-macro2", "quote", "shellexpand", - "syn 2.0.38", + "syn 2.0.39", "witx", ] @@ -8217,7 +8223,7 @@ checksum = "476e3e09bc68e82624b70a322265515523754cb9e05fcacceabd216e276bc2ed" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", "wiggle-generate", ] @@ -8420,9 +8426,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.18" +version = "0.5.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176b6138793677221d420fd2f0aeeced263f197688b36484660da767bca2fa32" +checksum = "829846f3e3db426d4cee4510841b71a8e58aa2a76b1132579487ae430ccd9c7b" dependencies = [ "memchr", ] @@ -8535,29 +8541,29 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.7.24" +version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "092cd76b01a033a9965b9097da258689d9e17c69ded5dcf41bca001dd20ebc6d" +checksum = "e97e415490559a91254a2979b4829267a57d2fcd741a98eee8b722fb57289aa0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.24" +version = "0.7.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13a20a7c6a90e2034bcc65495799da92efcec6a8dd4f3fcb6f7a48988637ead" +checksum = "dd7e48ccf166952882ca8bd778a43502c64f33bf94c12ebe2a7f08e5a0f6689f" dependencies = [ "proc-macro2", "quote", - "syn 2.0.38", + "syn 2.0.39", ] [[package]] name = "zeroize" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" +checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" [[package]] name = "zip" diff --git a/common/Cargo.toml b/common/Cargo.toml index 22ed4c4f2..512f0b637 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -29,7 +29,8 @@ pin-project = { workspace = true, optional = true } rand = { workspace = true, optional = true } reqwest = { workspace = true, optional = true } rmp-serde = { workspace = true, optional = true } -rustrict = { version = "0.7.4", optional = true } +# keep locked to not accidentally invalidate someone's project name +rustrict = { version = "=0.7.12", optional = true } semver = { workspace = true } serde = { workspace = true, features = ["derive", "std"] } serde_json = { workspace = true } @@ -58,6 +59,7 @@ backend = [ "hyper/client", "opentelemetry_sdk", "opentelemetry-otlp", + "rustrict", # only ProjectName model uses it "thiserror", "tokio", "tonic", @@ -90,7 +92,6 @@ models = [ "display", "http", "reqwest", - "rustrict", "service", "thiserror", ] From e9ec21b99a9043d78d559a3844e89ca6d1fbbe7c Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Mon, 20 Nov 2023 17:24:57 +0100 Subject: [PATCH 05/23] fix(gateway): handle invalid project names in ScopedUser (#1396) * fix(gateway): handle invalid project names in ScopedUser * nit: warn event is excessive * clippy moment * nit: better tracing error --- common/src/models/error.rs | 4 ++-- gateway/src/auth.rs | 3 ++- gateway/src/service.rs | 6 +++--- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/common/src/models/error.rs b/common/src/models/error.rs index 8be5d9d09..65a3e0439 100644 --- a/common/src/models/error.rs +++ b/common/src/models/error.rs @@ -1,6 +1,6 @@ use std::fmt::{Display, Formatter}; -use crossterm::style::{Color, Stylize}; +use crossterm::style::Stylize; use http::StatusCode; use serde::{Deserialize, Serialize}; use tracing::{error, warn}; @@ -23,7 +23,7 @@ impl Display for ApiError { f, "{}\nMessage: {}", self.status().to_string().bold(), - self.message.to_string().with(Color::Red) + self.message.to_string().red() ) } } diff --git a/gateway/src/auth.rs b/gateway/src/auth.rs index 6a5a0172e..6365978f8 100644 --- a/gateway/src/auth.rs +++ b/gateway/src/auth.rs @@ -5,6 +5,7 @@ use axum::extract::{FromRef, FromRequestParts, Path}; use axum::http::request::Parts; use serde::{Deserialize, Serialize}; use shuttle_common::claims::{Claim, Scope}; +use shuttle_common::models::error::InvalidProjectName; use shuttle_common::models::project::ProjectName; use tracing::{trace, Span}; @@ -80,7 +81,7 @@ where Err(_) => Path::<(ProjectName, String)>::from_request_parts(parts, state) .await .map(|Path((p, _))| p) - .unwrap(), + .map_err(|_| Error::from(ErrorKind::InvalidProjectName(InvalidProjectName)))?, }; if user.projects.contains(&scope) || user.claim.scopes.contains(&Scope::Admin) { diff --git a/gateway/src/service.rs b/gateway/src/service.rs index 91742d369..2ae1e3991 100644 --- a/gateway/src/service.rs +++ b/gateway/src/service.rs @@ -313,7 +313,7 @@ impl GatewayService { .try_get::, _>("project_state") .map(|p| p.0) .unwrap_or_else(|e| { - error!("Failed to deser `project_state`: {:?}", e); + error!(error = ?e, "Failed to deser `project_state`"); Project::Errored(ProjectError::internal( "Error when trying to deserialize state of project.", )) @@ -364,7 +364,7 @@ impl GatewayService { row.try_get::, _>("project_state") .map(|p| p.0) .unwrap_or_else(|e| { - error!("Failed to deser `project_state`: {:?}", e); + error!(error = ?e, "Failed to deser `project_state`"); Project::Errored(ProjectError::internal( "Error when trying to deserialize state of project.", )) @@ -459,7 +459,7 @@ impl GatewayService { .try_get::, _>("project_state") .map(|p| p.0) .unwrap_or_else(|e| { - error!("Failed to deser `project_state`: {:?}", e); + error!(error = ?e, "Failed to deser `project_state`"); Project::Errored(ProjectError::internal( "Error when trying to deserialize state of project.", )) From 4798777273ae29435a3a8cb05dfe08dfeabf45ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oddbj=C3=B8rn=20Gr=C3=B8dem?= <29732646+oddgrd@users.noreply.github.com> Date: Tue, 21 Nov 2023 08:48:52 +0100 Subject: [PATCH 06/23] feat: remove panamax registry override from deployers (#1399) --- deployer/prepare.sh | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/deployer/prepare.sh b/deployer/prepare.sh index 3b54e87f1..84f924f85 100755 --- a/deployer/prepare.sh +++ b/deployer/prepare.sh @@ -26,19 +26,3 @@ curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo- # Common cargo build tools cargo binstall -y --locked trunk@0.17.5 - -while getopts "p," o; do -case $o in - "p") # if panamax is used, the '-p' parameter is passed - # Make future crates requests to our own mirror - # This is done after shuttle-next install in order to not sabotage it - echo ' -[source.shuttle-crates-io-mirror] -registry = "sparse+http://panamax:8080/index/" -[source.crates-io] -replace-with = "shuttle-crates-io-mirror"' >> $CARGO_HOME/config.toml - ;; - *) - ;; - esac -done From 4a99d4a9351e7e557759a9300c2178f47c68d746 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oddbj=C3=B8rn=20Gr=C3=B8dem?= <29732646+oddgrd@users.noreply.github.com> Date: Tue, 21 Nov 2023 12:28:25 +0100 Subject: [PATCH 07/23] feat(logger): rate limit based on peer address (#1351) --- Cargo.lock | 123 ++++++++++++++++++++++++++++++ cargo-shuttle/src/lib.rs | 19 ++++- common/src/models/error.rs | 7 ++ deployer/src/handlers/error.rs | 3 + deployer/src/handlers/mod.rs | 44 ++++++++--- logger/Cargo.toml | 5 ++ logger/src/lib.rs | 1 + logger/src/main.rs | 31 +++++++- logger/src/rate_limiting.rs | 83 ++++++++++++++++++++ logger/tests/integration_tests.rs | 121 +++++++++++++++++++++++++++++ proto/src/lib.rs | 77 +++++++++++++++++-- 11 files changed, 494 insertions(+), 20 deletions(-) create mode 100644 logger/src/rate_limiting.rs diff --git a/Cargo.lock b/Cargo.lock index f0f12429b..a8ae5d0ab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1748,6 +1748,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dashmap" +version = "5.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "978747c1d849a7d2ee5e8adc0159961c48fb7e5db2f06af6723b80123bb53856" +dependencies = [ + "cfg-if 1.0.0", + "hashbrown 0.14.2", + "lock_api", + "once_cell", + "parking_lot_core 0.9.9", +] + [[package]] name = "data-encoding" version = "2.4.0" @@ -2136,6 +2149,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "forwarded-header-value" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835f84f38484cc86f110a805655697908257fb9a7af005234060891557198e9" +dependencies = [ + "nonempty", + "thiserror", +] + [[package]] name = "fqdn" version = "0.3.2" @@ -2256,6 +2279,12 @@ version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" +[[package]] +name = "futures-timer" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e64b03909df88034c26dc1547e8970b91f98bdb65165d6a4e9110d94263dbb2c" + [[package]] name = "futures-util" version = "0.3.29" @@ -3119,6 +3148,24 @@ dependencies = [ "regex", ] +[[package]] +name = "governor" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "821239e5672ff23e2a7060901fa622950bbd80b649cdaadd78d1c1767ed14eb4" +dependencies = [ + "cfg-if 1.0.0", + "dashmap", + "futures", + "futures-timer", + "no-std-compat", + "nonzero_ext", + "parking_lot 0.12.1", + "quanta", + "rand 0.8.5", + "smallvec", +] + [[package]] name = "guppy-workspace-hack" version = "0.1.0" @@ -3846,6 +3893,15 @@ dependencies = [ "libc", ] +[[package]] +name = "mach2" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d0d1830bcd151a6fc4aea1369af235b36c1528fe976b8ff678683c9995eade8" +dependencies = [ + "libc", +] + [[package]] name = "malloc_buf" version = "0.0.6" @@ -4074,6 +4130,12 @@ dependencies = [ "pin-utils", ] +[[package]] +name = "no-std-compat" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b93853da6d84c2e3c7d730d6473e8817692dd89be387eb01b94d7f108ecb5b8c" + [[package]] name = "nom" version = "7.1.3" @@ -4084,6 +4146,18 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nonempty" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9e591e719385e6ebaeb5ce5d3887f7d5676fceca6411d1925ccc95745f3d6f7" + +[[package]] +name = "nonzero_ext" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38bf9645c8b145698bb0b18a4637dcacbc421ea49bef2317e4fd8065a387cf21" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -4714,6 +4788,22 @@ dependencies = [ "unicase", ] +[[package]] +name = "quanta" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a17e662a7a8291a865152364c20c7abc5e60486ab2001e8ec10b24862de0b9ab" +dependencies = [ + "crossbeam-utils", + "libc", + "mach2", + "once_cell", + "raw-cpuid", + "wasi 0.11.0+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "queues" version = "1.1.0" @@ -4821,6 +4911,15 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "raw-window-handle" version = "0.5.2" @@ -5898,6 +5997,8 @@ dependencies = [ "chrono", "clap", "ctor", + "futures", + "http", "once_cell", "portpicker", "pretty_assertions", @@ -5911,6 +6012,8 @@ dependencies = [ "tokio", "tokio-stream", "tonic 0.10.2", + "tower", + "tower_governor", "tracing", "tracing-subscriber", "uuid", @@ -7079,6 +7182,26 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" +[[package]] +name = "tower_governor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db81d9313372d714152194f3f2b66badda23a783fb6a97462e35f632814f4cff" +dependencies = [ + "axum", + "forwarded-header-value", + "futures", + "futures-core", + "governor", + "http", + "pin-project", + "thiserror", + "tokio", + "tower", + "tower-layer", + "tracing", +] + [[package]] name = "tracing" version = "0.1.40" diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index c94d5d7cc..4aed35ad1 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -698,9 +698,22 @@ impl Shuttle { while let Some(Ok(msg)) = stream.next().await { if let tokio_tungstenite::tungstenite::Message::Text(line) = msg { - let log_item: shuttle_common::LogItem = serde_json::from_str(&line) - .context("Failed parsing logs. Is your cargo-shuttle outdated?")?; - println!("{log_item}") + match serde_json::from_str::(&line) { + Ok(log_item) => { + println!("{log_item}") + } + Err(err) => { + debug!(error = %err, "failed to parse message into log item"); + + let message = if let Ok(err) = serde_json::from_str::(&line) { + err.to_string() + } else { + "failed to parse logs, is your cargo-shuttle outdated?".to_string() + }; + + bail!(message); + } + } } } } else { diff --git a/common/src/models/error.rs b/common/src/models/error.rs index 65a3e0439..f09bc2c38 100644 --- a/common/src/models/error.rs +++ b/common/src/models/error.rs @@ -59,6 +59,7 @@ pub enum ErrorKind { NotReady, ServiceUnavailable, DeleteProjectFailed, + RateLimited(String), } impl From for ApiError { @@ -121,6 +122,12 @@ impl From for ApiError { ErrorKind::Forbidden => (StatusCode::FORBIDDEN, "forbidden"), ErrorKind::NotReady => (StatusCode::INTERNAL_SERVER_ERROR, "service not ready"), ErrorKind::DeleteProjectFailed => (StatusCode::INTERNAL_SERVER_ERROR, "deleting project failed"), + ErrorKind::RateLimited(message) => { + return Self { + message, + status_code: StatusCode::TOO_MANY_REQUESTS.as_u16(), + } + }, }; Self { message: error_message.to_string(), diff --git a/deployer/src/handlers/error.rs b/deployer/src/handlers/error.rs index 292e86728..a58e59de0 100644 --- a/deployer/src/handlers/error.rs +++ b/deployer/src/handlers/error.rs @@ -26,6 +26,8 @@ pub enum Error { Internal(#[from] anyhow::Error), #[error("Missing header: {0}")] MissingHeader(String), + #[error("{0}. Retry the request in a few minutes")] + RateLimited(String), } impl Serialize for Error { @@ -47,6 +49,7 @@ impl IntoResponse for Error { let code = match self { Error::NotFound(_) => StatusCode::NOT_FOUND, + Error::RateLimited(_) => StatusCode::TOO_MANY_REQUESTS, _ => StatusCode::INTERNAL_SERVER_ERROR, }; diff --git a/deployer/src/handlers/mod.rs b/deployer/src/handlers/mod.rs index ccf63fef8..4aa78816a 100644 --- a/deployer/src/handlers/mod.rs +++ b/deployer/src/handlers/mod.rs @@ -33,7 +33,7 @@ use shuttle_common::{ claims::{Claim, Scope}, models::{ deployment::{DeploymentRequest, CREATE_SERVICE_BODY_LIMIT, GIT_STRINGS_MAX_LENGTH}, - error::axum::CustomErrorPath, + error::{axum::CustomErrorPath, ApiError, ErrorKind}, project::ProjectName, }, request_span, LogItem, @@ -653,16 +653,27 @@ pub async fn get_logs( logs_request.extensions_mut().insert(claim); let mut client = deployment_manager.logs_fetcher().clone(); - if let Ok(logs) = client.get_logs(logs_request).await { - Ok(Json( + + match client.get_logs(logs_request).await { + Ok(logs) => Ok(Json( logs.into_inner() .log_items .into_iter() .map(|l| l.to_log_item_with_id(deployment_id)) .collect(), - )) - } else { - Err(Error::NotFound("deployment not found".to_string())) + )), + Err(error) => { + if error.code() == tonic::Code::Unavailable + && error.metadata().get("x-ratelimit-limit").is_some() + { + Err(Error::RateLimited( + "your application is producing too many logs. Interactions with the shuttle logger service will be rate limited" + .to_string(), + )) + } else { + Err(anyhow!("failed to retrieve logs for deployment").into()) + } + } } } @@ -712,9 +723,24 @@ async fn logs_websocket_handler( "failed to get backlog of logs" ); - let _ = s - .send(ws::Message::Text("failed to get logs".to_string())) - .await; + if error.code() == tonic::Code::Unavailable + && error.metadata().get("x-ratelimit-limit").is_some() + { + let message = serde_json::to_string( + &ApiError::from( + ErrorKind::RateLimited( + "your application is producing too many logs. Interactions with the shuttle logger service will be rate limited" + .to_string() + ) + )) + .expect("to convert error to json"); + + let _ = s.send(ws::Message::Text(message)).await; + } else { + let _ = s + .send(ws::Message::Text("failed to get logs".to_string())) + .await; + } let _ = s.close().await; return; } diff --git a/logger/Cargo.toml b/logger/Cargo.toml index 106e67fdc..4772c9c00 100644 --- a/logger/Cargo.toml +++ b/logger/Cargo.toml @@ -12,6 +12,7 @@ shuttle-proto = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } clap = { workspace = true } +http = { workspace = true } prost-types = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true, features = [ @@ -24,6 +25,8 @@ thiserror = { workspace = true } tokio = { workspace = true, features = ["rt-multi-thread"] } tokio-stream = { workspace = true } tonic = { workspace = true } +tower = { workspace = true } +tower_governor = { version= "0.1.0", features = ["tracing"] } tracing = { workspace = true } tracing-subscriber = { workspace = true, features = ["default"] } @@ -35,3 +38,5 @@ serde_json = { workspace = true } shuttle-common-tests = { workspace = true } uuid = { workspace = true } ctor = { workspace = true } +futures = { workspace = true } +tower = { workspace = true, features = ["util"] } diff --git a/logger/src/lib.rs b/logger/src/lib.rs index 98b2418fc..3340f8c8d 100644 --- a/logger/src/lib.rs +++ b/logger/src/lib.rs @@ -15,6 +15,7 @@ use tracing::{debug, error, field, Span}; pub mod args; mod dal; +pub mod rate_limiting; pub use dal::Postgres; diff --git a/logger/src/main.rs b/logger/src/main.rs index 1ae2a876b..8e04193ed 100644 --- a/logger/src/main.rs +++ b/logger/src/main.rs @@ -8,9 +8,15 @@ use shuttle_common::{ }, log::Backend, }; -use shuttle_logger::{args::Args, Postgres, Service}; +use shuttle_logger::{ + args::Args, + rate_limiting::{tonic_error, TonicPeerIpKeyExtractor, BURST_SIZE, REFRESH_INTERVAL}, + Postgres, Service, +}; use shuttle_proto::logger::logger_server::LoggerServer; use tonic::transport::Server; +use tower::ServiceBuilder; +use tower_governor::{governor::GovernorConfigBuilder, GovernorLayer}; use tracing::trace; #[tokio::main] @@ -21,12 +27,33 @@ async fn main() { trace!(args = ?args, "parsed args"); + let governor_config = GovernorConfigBuilder::default() + // Regenerate capacity at a rate of 2 requests per second, meaning the maximum capacity + // for sustained traffic is 2 RPS per peer address. + .per_millisecond(REFRESH_INTERVAL) + // Allow bursts of up to 6 requests, when any burst capacity is used, it will regenerate + // one element at a time at the rate set above. + .burst_size(BURST_SIZE) + .use_headers() + .key_extractor(TonicPeerIpKeyExtractor) + .finish() + .unwrap(); + let mut server_builder = Server::builder() .http2_keepalive_interval(Some(Duration::from_secs(60))) .layer(JwtAuthenticationLayer::new(AuthPublicKey::new( args.auth_uri, ))) - .layer(ExtractPropagationLayer); + .layer(ExtractPropagationLayer) + .layer( + ServiceBuilder::new() + // This middleware goes above `GovernorLayer` because it will receive errors returned by + // `GovernorLayer`. + .map_err(tonic_error) + .layer(GovernorLayer { + config: &governor_config, + }), + ); let postgres = Postgres::new(&args.db_connection_uri).await; diff --git a/logger/src/rate_limiting.rs b/logger/src/rate_limiting.rs new file mode 100644 index 000000000..d14f3ddca --- /dev/null +++ b/logger/src/rate_limiting.rs @@ -0,0 +1,83 @@ +use std::net::IpAddr; + +use tonic::{ + metadata::{KeyAndValueRef, MetadataMap}, + transport::server::TcpConnectInfo, + Status, +}; +use tower::BoxError; +use tower_governor::{key_extractor::KeyExtractor, GovernorError}; + +/// The interval at which the rate limiter refreshes one slot in milliseconds. +pub const REFRESH_INTERVAL: u64 = 500; +/// The quota of requests that can be received before rate limiting is applied. +pub const BURST_SIZE: u32 = 6; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TonicPeerIpKeyExtractor; + +impl KeyExtractor for TonicPeerIpKeyExtractor { + type Key = IpAddr; + + fn name(&self) -> &'static str { + "peer IP" + } + + fn extract(&self, req: &http::Request) -> Result { + req.extensions() + .get::() + .and_then(|info| info.remote_addr()) + .map(|addr| addr.ip()) + .ok_or(GovernorError::UnableToExtractKey) + } + + fn key_name(&self, key: &Self::Key) -> Option { + Some(key.to_string()) + } +} + +/// Convert errors from the Governor rate limiter layer to tonic statuses. +pub fn tonic_error(e: BoxError) -> tonic::Status { + if let Some(error) = e.downcast_ref::() { + match error.to_owned() { + GovernorError::TooManyRequests { wait_time, headers } => { + let mut response = Status::unavailable(format!( + "received too many requests, wait for {wait_time}ms" + )); + + // Add rate limiting headers: x-ratelimit-remaining, x-ratelimit-after, x-ratelimit-limit. + if let Some(headers) = headers { + let metadata = MetadataMap::from_headers(headers); + + for header in metadata.iter() { + if let KeyAndValueRef::Ascii(key, value) = header { + response.metadata_mut().insert(key, value.clone()); + } + } + } + + response + } + GovernorError::UnableToExtractKey => { + Status::unavailable("unable to extract peer address") + } + GovernorError::Other { headers, .. } => { + let mut response = Status::internal("unexpected error in rate limiter"); + + if let Some(headers) = headers { + let metadata = MetadataMap::from_headers(headers); + + for header in metadata.iter() { + if let KeyAndValueRef::Ascii(key, value) = header { + response.metadata_mut().insert(key, value.clone()); + } + } + } + + response + } + } + } else { + Status::internal("unexpected error in rate limiter") + } +} diff --git a/logger/tests/integration_tests.rs b/logger/tests/integration_tests.rs index 81e5b2120..d36dfa2d2 100644 --- a/logger/tests/integration_tests.rs +++ b/logger/tests/integration_tests.rs @@ -36,7 +36,13 @@ fn cleanup() { } mod needs_docker { use super::*; + use futures::future::join_all; use pretty_assertions::assert_eq; + use shuttle_logger::rate_limiting::{ + tonic_error, TonicPeerIpKeyExtractor, BURST_SIZE, REFRESH_INTERVAL, + }; + use tower::ServiceBuilder; + use tower_governor::{governor::GovernorConfigBuilder, GovernorLayer}; #[tokio::test] async fn store_and_get_logs() { @@ -191,6 +197,104 @@ mod needs_docker { } } + #[tokio::test] + async fn store_and_get_logs_rate_limited() { + let logger_port = pick_unused_port().unwrap(); + let deployment_id = "runtime-fetch-logs-deployment-id"; + + // Create a unique database name so we have a new database for each test. + let db_name = Uuid::new_v4().to_string(); + + let server = spawn_server(logger_port, db_name); + + let test_future = tokio::spawn(async move { + // Ensure the DB has been created and server has started. + tokio::time::sleep(Duration::from_millis(300)).await; + + let dst = format!("http://localhost:{logger_port}"); + let mut client = LoggerClient::connect(dst).await.unwrap(); + + let store_logs = || async { + client + .clone() + .store_logs(Request::new(StoreLogsRequest { + logs: vec![LogItem { + deployment_id: deployment_id.to_string(), + log_line: Some(LogLine { + service_name: SHUTTLE_SERVICE.to_string(), + tx_timestamp: Some(Timestamp::from(SystemTime::UNIX_EPOCH)), + data: ("log example").as_bytes().to_vec(), + }), + }], + })) + .await + }; + + // Six concurrent requests succeeds when rate limiter is fresh. + let futures = (0..6).map(|_| store_logs()); + let result = join_all(futures).await; + + assert!(result.iter().all(|response| response.is_ok())); + + // Allow rate limiter time to regenerate two requests. + tokio::time::sleep(Duration::from_millis(1000)).await; + + let futures = (0..2).map(|_| store_logs()); + let result = join_all(futures).await; + + assert!(result.iter().all(|response| response.is_ok())); + + // Allow rate limiter time to regenerate two requests. + tokio::time::sleep(Duration::from_millis(1000)).await; + + // Send three requests when the capacity is two. + let futures = (0..3).map(|_| store_logs()); + let result = join_all(futures).await; + + assert_eq!(result.iter().filter(|response| response.is_ok()).count(), 2); + + // Check that the error has the expected status and rate limiting headers. + result + .iter() + .filter(|response| response.is_err()) + .for_each(|err| { + let err = err.as_ref().unwrap_err(); + + assert_eq!(err.code(), tonic::Code::Unavailable); + assert!(err.message().contains("too many requests")); + + let expected = [ + "x-ratelimit-remaining", + "x-ratelimit-after", + "x-ratelimit-limit", + ]; + + let headers = err.metadata(); + assert!(expected.into_iter().all(|key| headers.contains_key(key))); + }); + + // Allow rate limiter to regenerate a slot for the get_logs request. + tokio::time::sleep(Duration::from_millis(500)).await; + + // Verify that all the logs that weren't rate limited were persisted in the logger. + let logs = client + .get_logs(Request::new(LogsRequest { + deployment_id: deployment_id.into(), + })) + .await + .unwrap() + .into_inner() + .log_items; + + assert_eq!(logs.len(), 10); + }); + + tokio::select! { + _ = server => panic!("server stopped first"), + result = test_future => result.expect("test should succeed") + } + } + fn spawn_server(port: u16, db_name: String) -> JoinHandle<()> { let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port); @@ -199,10 +303,27 @@ mod needs_docker { exec_psql(&format!(r#"CREATE DATABASE "{}";"#, &db_name)); + let governor_config = GovernorConfigBuilder::default() + .per_millisecond(REFRESH_INTERVAL) + .burst_size(BURST_SIZE) + .use_headers() + .key_extractor(TonicPeerIpKeyExtractor) + .finish() + .unwrap(); + tokio::task::spawn(async move { let pg = Postgres::new(&pg_uri).await; Server::builder() .layer(JwtScopesLayer::new(vec![Scope::Logs])) + .layer( + ServiceBuilder::new() + // This middleware goes above `GovernorLayer` because it will receive errors returned by + // `GovernorLayer`. + .map_err(tonic_error) + .layer(GovernorLayer { + config: &governor_config, + }), + ) .add_service(LoggerServer::new(Service::new(pg.get_sender(), pg))) .serve(addr) .await diff --git a/proto/src/lib.rs b/proto/src/lib.rs index e2626d651..336df66ce 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -344,14 +344,79 @@ pub mod logger { type Item = LogItem; async fn receive(&mut self, items: Vec) { + // A log vector should never be received without any items. We clone the first item + // here so we can use IDs to generate a rate limiting logline, which we will send to + // the logger as a warning to the user that they are being rate limited. + let Some(item) = items.first().cloned() else { + error!("received log vector without any items"); + + return; + }; + if let Err(error) = self .store_logs(Request::new(StoreLogsRequest { logs: items })) .await { - error!( - error = &error as &dyn std::error::Error, - "failed to send batch logs to logger" - ); + match error.code() { + tonic::Code::Unavailable => { + if error.metadata().get("x-ratelimit-limit").is_some() { + let LogItem { + deployment_id, + log_line, + } = item; + + let LogLine { service_name, .. } = log_line.unwrap(); + + let timestamp = Utc::now(); + + let new_item = LogItem { + deployment_id, + log_line: Some(LogLine { + tx_timestamp: Some(prost_types::Timestamp { + seconds: timestamp.timestamp(), + nanos: timestamp.timestamp_subsec_nanos() as i32, + }), + service_name: Backend::Runtime(service_name.clone()) + .to_string(), + data: "your application is producing too many logs, log recording is being rate limited".into(), + }), + }; + + // Give the rate limiter time to refresh, the duration we need to sleep + // for here is determined by the refresh rate of the rate limiter in + // the logger server. It is currently set to refresh one slot per 500ms, + // so we could get away with a duration of 500ms here, but we give it + // some extra time in case this rate limiting was caused by a short + // burst of activity. + tokio::time::sleep(Duration::from_millis(1500)).await; + + if let Err(error) = self + // NOTE: the request to send this rate limiting log to the logger will + // also expend a slot in the logger rate limiter. + .store_logs(Request::new(StoreLogsRequest { + logs: vec![new_item], + })) + .await + { + error!( + error = &error as &dyn std::error::Error, + "failed to send rate limiting warning to logger service" + ); + }; + } else { + error!( + error = &error as &dyn std::error::Error, + "failed to send batch logs to logger" + ); + } + } + _ => { + error!( + error = &error as &dyn std::error::Error, + "failed to send batch logs to logger" + ); + } + }; } } } @@ -376,10 +441,10 @@ pub mod logger { Self { tx } } - /// Create a batcher around inner. It will send a batch of items to inner if a capacity of 2048 is reached + /// Create a batcher around inner. It will send a batch of items to inner if a capacity of 256 is reached /// or if an interval of 1 second is reached. pub fn wrap(inner: I) -> Self { - Self::new(inner, 2048, Duration::from_secs(1)) + Self::new(inner, 256, Duration::from_secs(1)) } /// Send a single item into this batcher From c88f0bc9cc7ebcc56e2ecbaea30142e8d6e8ee35 Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Tue, 21 Nov 2023 06:59:42 -0800 Subject: [PATCH 08/23] improvement: rocket-0.5.0 stable (#1401) --- services/shuttle-rocket/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/shuttle-rocket/Cargo.toml b/services/shuttle-rocket/Cargo.toml index 5aa4d006f..5d3b65974 100644 --- a/services/shuttle-rocket/Cargo.toml +++ b/services/shuttle-rocket/Cargo.toml @@ -9,7 +9,7 @@ keywords = ["shuttle-service", "rocket"] [workspace] [dependencies] -rocket = { version = "0.5.0-rc.4" } +rocket = { version = "0.5.0" } shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } [dev-dependencies] From 569e8317d7b51fc2333153a05a356573c4a6a8de Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Tue, 21 Nov 2023 16:09:33 +0100 Subject: [PATCH 09/23] refactor: better feature scoping, fix turso compilation, prune library dependency tree (#1405) * refactor: better feature scoping, fix turso compilation, prune dependency tree for library consumers * fix: features * fix: cargo * nit: use wrapped tonic * fix: scope only ProjectName to backend --- Cargo.lock | 6 +- builder/Cargo.toml | 2 +- cargo-shuttle/Cargo.toml | 4 +- cargo-shuttle/src/args.rs | 3 +- cargo-shuttle/src/lib.rs | 12 +- common-tests/Cargo.toml | 2 +- common/Cargo.toml | 16 +- common/src/claims.rs | 8 +- common/src/constants.rs | 13 +- common/src/models/mod.rs | 1 - common/src/models/project.rs | 259 ++++++++++++++------------- deployer/Cargo.toml | 4 +- deployer/src/runtime_manager.rs | 6 +- gateway/Cargo.toml | 3 +- gateway/src/project.rs | 3 +- logger/Cargo.toml | 2 +- proto/Cargo.toml | 36 ++-- proto/src/lib.rs | 110 ++---------- provisioner/Cargo.toml | 4 +- resource-recorder/Cargo.toml | 2 +- resources/turso/Cargo.toml | 3 +- runtime/Cargo.toml | 7 +- runtime/tests/integration/helpers.rs | 6 +- service/Cargo.toml | 19 +- service/src/lib.rs | 6 +- service/src/runner.rs | 96 ++++++++++ 26 files changed, 333 insertions(+), 300 deletions(-) create mode 100644 service/src/runner.rs diff --git a/Cargo.lock b/Cargo.lock index a8ae5d0ab..7b4c859d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6029,16 +6029,13 @@ version = "0.33.0" dependencies = [ "anyhow", "chrono", - "dunce", "futures-core", - "home", "prost 0.12.2", "prost-types", "serde_json", "shuttle-common", "tokio", "tonic 0.10.2", - "tower", "tracing", ] @@ -6131,12 +6128,15 @@ dependencies = [ "anyhow", "async-trait", "cargo_metadata 0.18.1", + "dunce", "serde", "shuttle-common", + "shuttle-proto", "strfmt", "thiserror", "tokio", "toml 0.8.8", + "tower", "tracing", ] diff --git a/builder/Cargo.toml b/builder/Cargo.toml index 57380702a..2bb30e436 100644 --- a/builder/Cargo.toml +++ b/builder/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] shuttle-common = { workspace = true, features = ["backend", "tonic"] } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["builder"] } async-trait = { workspace = true } clap = { workspace = true } diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index 9e3d8b328..e026ed35e 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -10,8 +10,8 @@ rust-version = "1.70" [dependencies] shuttle-common = { workspace = true, features = ["models"] } -shuttle-proto = { workspace = true } -shuttle-service = { workspace = true, features = ["builder"] } +shuttle-proto = { workspace = true, features = ["provisioner", "runtime"] } +shuttle-service = { workspace = true, features = ["builder", "runner"] } anyhow = { workspace = true } async-trait = { workspace = true } diff --git a/cargo-shuttle/src/args.rs b/cargo-shuttle/src/args.rs index 21830c44f..11803565a 100644 --- a/cargo-shuttle/src/args.rs +++ b/cargo-shuttle/src/args.rs @@ -12,7 +12,8 @@ use clap::{ Parser, ValueEnum, }; use clap_complete::Shell; -use shuttle_common::{models::project::DEFAULT_IDLE_MINUTES, resource}; +use shuttle_common::constants::DEFAULT_IDLE_MINUTES; +use shuttle_common::resource; use uuid::Uuid; #[derive(Parser)] diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 4aed35ad1..dbe1d5c45 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -17,8 +17,9 @@ use std::str::FromStr; use shuttle_common::{ claims::{ClaimService, InjectPropagation}, constants::{ - API_URL_DEFAULT, EXECUTABLE_DIRNAME, SHUTTLE_CLI_DOCS_URL, SHUTTLE_GH_ISSUE_URL, - SHUTTLE_IDLE_DOCS_URL, SHUTTLE_INSTALL_DOCS_URL, SHUTTLE_LOGIN_URL, STORAGE_DIRNAME, + API_URL_DEFAULT, DEFAULT_IDLE_MINUTES, EXECUTABLE_DIRNAME, SHUTTLE_CLI_DOCS_URL, + SHUTTLE_GH_ISSUE_URL, SHUTTLE_IDLE_DOCS_URL, SHUTTLE_INSTALL_DOCS_URL, SHUTTLE_LOGIN_URL, + STORAGE_DIRNAME, }, deployment::{DEPLOYER_END_MESSAGES_BAD, DEPLOYER_END_MESSAGES_GOOD}, models::{ @@ -27,14 +28,15 @@ use shuttle_common::{ GIT_STRINGS_MAX_LENGTH, }, error::ApiError, - project::{self, DEFAULT_IDLE_MINUTES}, + project, resource::get_resource_tables, }, resource, semvers_are_compatible, ApiKey, LogItem, VersionInfo, }; use shuttle_proto::runtime::{ - self, runtime_client::RuntimeClient, LoadRequest, StartRequest, StopRequest, + runtime_client::RuntimeClient, LoadRequest, StartRequest, StopRequest, }; +use shuttle_service::runner; use shuttle_service::{ builder::{build_workspace, BuiltService}, Environment, @@ -935,7 +937,7 @@ impl Shuttle { }; // Child process and gRPC client for sending requests to it - let (mut runtime, mut runtime_client) = runtime::start( + let (mut runtime, mut runtime_client) = runner::start( service.is_wasm, Environment::Local, &format!("http://localhost:{provisioner_port}"), diff --git a/common-tests/Cargo.toml b/common-tests/Cargo.toml index 8f1c6400e..de1f74858 100644 --- a/common-tests/Cargo.toml +++ b/common-tests/Cargo.toml @@ -8,7 +8,7 @@ publish = false [dependencies] cargo-shuttle = { path = "../cargo-shuttle" } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["builder", "logger"] } hyper = { workspace = true } portpicker = { workspace = true } diff --git a/common/Cargo.toml b/common/Cargo.toml index 512f0b637..0678142e2 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -67,9 +67,6 @@ backend = [ "tracing-subscriber/env-filter", "tracing-subscriber/fmt", "ttl_cache", - "sqlx", - "sqlx/postgres", # Fix for derive macro error when different backends enable sqlite & postgres (mainly for grouped compilation in clippy CI and Docker build) - "sqlx/sqlite", ] claims = [ "bytes", @@ -87,16 +84,9 @@ claims = [ ] display = ["chrono/clock", "comfy-table", "crossterm"] openapi = ["utoipa/chrono", "utoipa/uuid"] -models = [ - "async-trait", - "display", - "http", - "reqwest", - "service", - "thiserror", -] -persist = ["sqlx/sqlite", "rand"] -service = ["chrono/serde", "tracing", "tracing-subscriber", "uuid"] +models = ["async-trait", "http", "reqwest", "service", "thiserror"] +persist = ["sqlx", "rand"] +service = ["chrono/serde", "display", "tracing", "tracing-subscriber", "uuid"] tracing = ["dep:tracing"] wasm = [ "chrono/clock", diff --git a/common/src/claims.rs b/common/src/claims.rs index 4bca392cb..e0a5409ee 100644 --- a/common/src/claims.rs +++ b/common/src/claims.rs @@ -171,10 +171,10 @@ impl Default for ScopeBuilder { #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize, Eq, PartialEq)] #[serde(rename_all = "lowercase")] -#[cfg_attr(feature = "backend", derive(strum::Display))] -#[cfg_attr(feature = "backend", derive(sqlx::Type))] -#[cfg_attr(feature = "backend", sqlx(rename_all = "lowercase"))] -#[cfg_attr(feature = "backend", strum(serialize_all = "lowercase"))] +#[cfg_attr(feature = "display", derive(strum::Display))] +#[cfg_attr(feature = "display", strum(serialize_all = "lowercase"))] +#[cfg_attr(feature = "persist", derive(sqlx::Type))] +#[cfg_attr(feature = "persist", sqlx(rename_all = "lowercase"))] pub enum AccountTier { #[default] Basic, diff --git a/common/src/constants.rs b/common/src/constants.rs index af8ad1699..0bdf6276a 100644 --- a/common/src/constants.rs +++ b/common/src/constants.rs @@ -1,6 +1,5 @@ -// -// Constants regarding the deployer environment and conventions -// +//! Shared constants used across Shuttle crates + /// Where executables are moved to in order to persist across deploys, relative to workspace root pub const EXECUTABLE_DIRNAME: &str = ".shuttle-executables"; /// Where general files will persist across deploys, relative to workspace root. Used by plugins. @@ -27,6 +26,14 @@ pub const SHUTTLE_EXAMPLES_README: &str = pub const NEXT_NAME: &str = "shuttle-next"; pub const RUNTIME_NAME: &str = "shuttle-runtime"; +/// Timeframe before a project is considered idle +pub const DEFAULT_IDLE_MINUTES: u64 = 30; + +/// Function to set [DEFAULT_IDLE_MINUTES] as a serde default +pub const fn default_idle_minutes() -> u64 { + DEFAULT_IDLE_MINUTES +} + pub mod limits { pub const MAX_PROJECTS_DEFAULT: u32 = 3; pub const MAX_PROJECTS_EXTRA: u32 = 15; diff --git a/common/src/models/mod.rs b/common/src/models/mod.rs index 3fbc6c1de..1d687f638 100644 --- a/common/src/models/mod.rs +++ b/common/src/models/mod.rs @@ -1,7 +1,6 @@ pub mod admin; pub mod deployment; pub mod error; -#[cfg(feature = "backend")] pub mod project; pub mod resource; pub mod service; diff --git a/common/src/models/project.rs b/common/src/models/project.rs index d4c000a77..688f8d817 100644 --- a/common/src/models/project.rs +++ b/common/src/models/project.rs @@ -1,8 +1,6 @@ -use std::collections::HashSet; use std::fmt::Display; use std::fmt::Formatter; use std::str::FromStr; -use std::sync::OnceLock; use comfy_table::{ modifiers::UTF8_ROUND_CORNERS, @@ -10,9 +8,7 @@ use comfy_table::{ Attribute, Cell, CellAlignment, Color, ContentArrangement, Table, }; use crossterm::style::Stylize; -use rustrict::{Censor, Type}; -use serde::de::Error as DeError; -use serde::{Deserialize, Deserializer, Serialize}; +use serde::{Deserialize, Serialize}; use strum::EnumString; #[cfg(feature = "openapi")] @@ -20,16 +16,6 @@ use crate::ulid_type; #[cfg(feature = "openapi")] use utoipa::ToSchema; -use super::error::InvalidProjectName; - -/// Timeframe before a project is considered idle -pub const DEFAULT_IDLE_MINUTES: u64 = 30; - -/// Function to set [DEFAULT_IDLE_MINUTES] as a serde default -pub const fn default_idle_minutes() -> u64 { - DEFAULT_IDLE_MINUTES -} - #[derive(Deserialize, Serialize, Clone)] #[cfg_attr(feature = "openapi", derive(ToSchema))] #[cfg_attr(feature = "openapi", schema(as = shuttle_common::models::project::Response))] @@ -259,142 +245,159 @@ pub fn get_projects_table( } } -/// Project names must conform to valid Host segments (or labels) -/// as per [IETF RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123). -/// Initially we'll implement a strict subset of the IETF RFC 1123. -/// Additionaly, while host segments are technically case-insensitive, the filesystem isn't, -/// so we restrict project names to be lower case. We also restrict the use of profanity, -/// as well as a list of reserved words. -#[derive(Clone, Serialize, Debug, Eq, Hash, PartialEq, sqlx::Type)] -#[sqlx(transparent)] -pub struct ProjectName(String); - -impl ProjectName { - pub fn new(name: &str) -> Result { - if Self::is_valid(name) { - Ok(Self(name.to_owned())) - } else { - Err(InvalidProjectName) +#[cfg(feature = "backend")] +pub use name::ProjectName; +#[cfg(feature = "backend")] +pub mod name { + use std::collections::HashSet; + use std::fmt::Formatter; + use std::str::FromStr; + use std::sync::OnceLock; + + use rustrict::{Censor, Type}; + use serde::de::Error as DeError; + use serde::{Deserialize, Deserializer, Serialize}; + + use crate::models::error::InvalidProjectName; + + /// Project names must conform to valid Host segments (or labels) + /// as per [IETF RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123). + /// Initially we'll implement a strict subset of the IETF RFC 1123. + /// Additionaly, while host segments are technically case-insensitive, the filesystem isn't, + /// so we restrict project names to be lower case. We also restrict the use of profanity, + /// as well as a list of reserved words. + #[derive(Clone, Serialize, Debug, Eq, Hash, PartialEq)] + #[cfg_attr(feature = "persist", derive(sqlx::Type))] + #[cfg_attr(feature = "persist", sqlx(transparent))] + pub struct ProjectName(String); + + impl ProjectName { + pub fn new(name: &str) -> Result { + if Self::is_valid(name) { + Ok(Self(name.to_owned())) + } else { + Err(InvalidProjectName) + } } - } - pub fn is_valid(name: &str) -> bool { - fn is_valid_char(byte: u8) -> bool { - matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-') - } + pub fn is_valid(name: &str) -> bool { + fn is_valid_char(byte: u8) -> bool { + matches!(byte, b'a'..=b'z' | b'0'..=b'9' | b'-') + } - fn is_profanity_free(name: &str) -> bool { - let (_censored, analysis) = Censor::from_str(name).censor_and_analyze(); - !analysis.is(Type::MODERATE_OR_HIGHER) - } + fn is_profanity_free(name: &str) -> bool { + let (_censored, analysis) = Censor::from_str(name).censor_and_analyze(); + !analysis.is(Type::MODERATE_OR_HIGHER) + } - fn is_reserved(name: &str) -> bool { - static INSTANCE: OnceLock> = OnceLock::new(); - INSTANCE.get_or_init(|| { - HashSet::from(["shuttleapp", "shuttle", "console", "unstable", "staging"]) - }); + fn is_reserved(name: &str) -> bool { + static INSTANCE: OnceLock> = OnceLock::new(); + INSTANCE.get_or_init(|| { + HashSet::from(["shuttleapp", "shuttle", "console", "unstable", "staging"]) + }); - INSTANCE - .get() - .expect("Reserved words not set") - .contains(name) - } + INSTANCE + .get() + .expect("Reserved words not set") + .contains(name) + } - !name.is_empty() - && name.len() < 64 - && !name.starts_with('-') - && !name.ends_with('-') - && !is_reserved(name) - && name.bytes().all(is_valid_char) - && is_profanity_free(name) + !name.is_empty() + && name.len() < 64 + && !name.starts_with('-') + && !name.ends_with('-') + && !is_reserved(name) + && name.bytes().all(is_valid_char) + && is_profanity_free(name) + } } -} -impl std::ops::Deref for ProjectName { - type Target = String; + impl std::ops::Deref for ProjectName { + type Target = String; - fn deref(&self) -> &Self::Target { - &self.0 + fn deref(&self) -> &Self::Target { + &self.0 + } } -} -impl std::fmt::Display for ProjectName { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - self.0.fmt(f) + impl std::fmt::Display for ProjectName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } } -} -impl<'de> Deserialize<'de> for ProjectName { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s: String = String::deserialize(deserializer)?; - s.parse().map_err(DeError::custom) + impl<'de> Deserialize<'de> for ProjectName { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s: String = String::deserialize(deserializer)?; + s.parse().map_err(DeError::custom) + } } -} -impl FromStr for ProjectName { - type Err = InvalidProjectName; + impl FromStr for ProjectName { + type Err = InvalidProjectName; - fn from_str(s: &str) -> Result { - ProjectName::new(s) + fn from_str(s: &str) -> Result { + ProjectName::new(s) + } } -} -/// Test examples taken from a [Pop-OS project](https://github.com/pop-os/hostname-validator/blob/master/src/lib.rs) -/// and modified to our use case -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn valid_labels() { - for name in [ - "50-name", - "235235", - "123", - "kebab-case", - "lowercase", - "myassets", - "dachterrasse", - "another-valid-project-name", - "x", - ] { - assert!(ProjectName::is_valid(name)); + /// Test examples taken from a [Pop-OS project](https://github.com/pop-os/hostname-validator/blob/master/src/lib.rs) + /// and modified to our use case + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn valid_labels() { + for name in [ + "50-name", + "235235", + "123", + "kebab-case", + "lowercase", + "myassets", + "dachterrasse", + "another-valid-project-name", + "x", + ] { + assert!(ProjectName::is_valid(name)); + } } - } - #[test] - fn invalid_labels() { - for name in [ - "UPPERCASE", - "CamelCase", - "pascalCase", - "InVaLid", - "-invalid-name", - "also-invalid-", - "asdf@fasd", - "@asdfl", - "asd f@", - ".invalid", - "invalid.name", - "invalid.name.", - "__dunder_like__", - "__invalid", - "invalid__", - "test-condom-condom", - "s________e", - "snake_case", - "exactly-16-chars\ + #[test] + fn invalid_labels() { + for name in [ + "UPPERCASE", + "CamelCase", + "pascalCase", + "InVaLid", + "-invalid-name", + "also-invalid-", + "asdf@fasd", + "@asdfl", + "asd f@", + ".invalid", + "invalid.name", + "invalid.name.", + "__dunder_like__", + "__invalid", + "invalid__", + "test-condom-condom", + "s________e", + "snake_case", + "exactly-16-chars\ exactly-16-chars\ exactly-16-chars\ exactly-16-chars", - "shuttle", - "shuttleapp", - "", - ] { - assert!(!ProjectName::is_valid(name)); + "shuttle", + "shuttleapp", + "", + ] { + assert!(!ProjectName::is_valid(name)); + } } } } diff --git a/deployer/Cargo.toml b/deployer/Cargo.toml index 40ae6ba08..94b7766d6 100644 --- a/deployer/Cargo.toml +++ b/deployer/Cargo.toml @@ -7,8 +7,8 @@ description = "Service with instances created per project for handling the compi [dependencies] shuttle-common = { workspace = true, features = ["backend", "models", "openapi"] } -shuttle-proto = { workspace = true } -shuttle-service = { workspace = true, features = ["builder"] } +shuttle-proto = { workspace = true, features = ["resource-recorder"] } +shuttle-service = { workspace = true, features = ["builder", "runner"] } anyhow = { workspace = true } async-trait = { workspace = true } diff --git a/deployer/src/runtime_manager.rs b/deployer/src/runtime_manager.rs index 23fdc11a0..8dcd5bb63 100644 --- a/deployer/src/runtime_manager.rs +++ b/deployer/src/runtime_manager.rs @@ -13,9 +13,9 @@ use shuttle_common::{ }; use shuttle_proto::{ logger::{logger_client::LoggerClient, Batcher, LogItem, LogLine}, - runtime::{self, runtime_client::RuntimeClient, StopRequest}, + runtime::{runtime_client::RuntimeClient, StopRequest}, }; -use shuttle_service::Environment; +use shuttle_service::{runner, Environment}; use tokio::{io::AsyncBufReadExt, io::BufReader, process, sync::Mutex}; use tonic::transport::Channel; use tracing::{debug, error, info, trace, warn}; @@ -125,7 +125,7 @@ impl RuntimeManager { .join("bin/shuttle-next") }; - let (mut process, runtime_client) = runtime::start( + let (mut process, runtime_client) = runner::start( is_next, Environment::Deployment, &self.provisioner_address, diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index d463233f7..b21fa22a8 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -10,8 +10,9 @@ shuttle-common = { workspace = true, features = [ "backend", "models", "openapi", + "persist", ] } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["provisioner"] } shuttle-orchestrator = { workspace = true } async-trait = { workspace = true } diff --git a/gateway/src/project.rs b/gateway/src/project.rs index 1f29d1071..2f7f563e0 100644 --- a/gateway/src/project.rs +++ b/gateway/src/project.rs @@ -24,7 +24,8 @@ use once_cell::sync::Lazy; use rand::distributions::{Alphanumeric, DistString}; use serde::{Deserialize, Serialize}; use shuttle_common::backends::headers::{X_SHUTTLE_ACCOUNT_NAME, X_SHUTTLE_ADMIN_SECRET}; -use shuttle_common::models::project::{default_idle_minutes, ProjectName, DEFAULT_IDLE_MINUTES}; +use shuttle_common::constants::{default_idle_minutes, DEFAULT_IDLE_MINUTES}; +use shuttle_common::models::project::ProjectName; use shuttle_common::models::service; use tokio::time::{sleep, timeout}; use tracing::{debug, error, info, instrument, trace, warn}; diff --git a/logger/Cargo.toml b/logger/Cargo.toml index 4772c9c00..508d0db98 100644 --- a/logger/Cargo.toml +++ b/logger/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] shuttle-common = { workspace = true, features = ["backend", "tonic"] } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["logger"] } async-trait = { workspace = true } chrono = { workspace = true } diff --git a/proto/Cargo.toml b/proto/Cargo.toml index 478a88110..c0ac48575 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -6,23 +6,29 @@ license.workspace = true description = "Library for all the gRPC definitions used by shuttle" [dependencies] -shuttle-common = { workspace = true, features = [ - "claims", - "service", - "wasm", - "models", - "backend", -] } +shuttle-common = { workspace = true } -anyhow = { workspace = true } -chrono = { workspace = true } -dunce = { workspace = true } +anyhow = { workspace = true, optional = true } +chrono = { workspace = true, optional = true } futures-core = "0.3.28" -home = { workspace = true } prost = { workspace = true } prost-types = { workspace = true } -tokio = { workspace = true, features = ["process"] } +tokio = { workspace = true, optional = true } tonic = { workspace = true } -tower = { workspace = true } -tracing = { workspace = true } -serde_json = { workspace = true } +tracing = { workspace = true, optional = true } +serde_json = { workspace = true, optional = true } + +[features] +default = [] + +builder = [] +logger = [ + "shuttle-common/service", + "chrono", + "tracing", + "tokio/macros", + "tokio/time", +] +provisioner = [] +resource-recorder = ["anyhow", "serde_json"] +runtime = [] diff --git a/proto/src/lib.rs b/proto/src/lib.rs index 336df66ce..f1c0cfa2f 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -1,5 +1,11 @@ mod generated; +// useful re-exports if types are needed in other crates +pub use prost; +pub use prost_types; +pub use tonic; + +#[cfg(feature = "provisioner")] pub mod provisioner { use std::fmt::Display; @@ -92,106 +98,12 @@ pub mod provisioner { } } +#[cfg(feature = "runtime")] pub mod runtime { - use std::{ - path::{Path, PathBuf}, - process::Stdio, - time::Duration, - }; - - use anyhow::Context; - use shuttle_common::{ - claims::{ClaimLayer, ClaimService, InjectPropagation, InjectPropagationLayer}, - deployment::Environment, - }; - use tokio::process; - use tonic::transport::{Channel, Endpoint}; - use tower::ServiceBuilder; - use tracing::{info, trace}; - pub use super::generated::runtime::*; - - pub async fn start( - wasm: bool, - environment: Environment, - provisioner_address: &str, - auth_uri: Option<&String>, - port: u16, - runtime_executable: PathBuf, - project_path: &Path, - ) -> anyhow::Result<( - process::Child, - runtime_client::RuntimeClient>>, - )> { - let port = &port.to_string(); - let environment = &environment.to_string(); - - let args = if wasm { - vec!["--port", port] - } else { - let mut args = vec![ - "--port", - port, - "--provisioner-address", - provisioner_address, - "--env", - environment, - ]; - - if let Some(auth_uri) = auth_uri { - args.append(&mut vec!["--auth-uri", auth_uri]); - } - - args - }; - - info!( - "Spawning runtime process: {} {}", - runtime_executable.display(), - args.join(" ") - ); - let runtime = process::Command::new( - dunce::canonicalize(runtime_executable).context("canonicalize path of executable")?, - ) - .current_dir(project_path) - .args(&args) - .stdout(Stdio::piped()) - .kill_on_drop(true) - .spawn() - .context("spawning runtime process")?; - - info!("connecting runtime client"); - let conn = Endpoint::new(format!("http://127.0.0.1:{port}")) - .context("creating runtime client endpoint")? - .connect_timeout(Duration::from_secs(5)); - - // Wait for the spawned process to open the control port. - // Connecting instantly does not give it enough time. - let channel = tokio::time::timeout(Duration::from_millis(7000), async move { - let mut ms = 5; - loop { - if let Ok(channel) = conn.connect().await { - break channel; - } - trace!("waiting for runtime control port to open"); - // exponential backoff - tokio::time::sleep(Duration::from_millis(ms)).await; - ms *= 2; - } - }) - .await - .context("runtime control port did not open in time")?; - - let channel = ServiceBuilder::new() - .layer(ClaimLayer) - .layer(InjectPropagationLayer) - .service(channel); - let runtime_client = runtime_client::RuntimeClient::new(channel); - - Ok((runtime, runtime_client)) - } } +#[cfg(feature = "resource-recorder")] pub mod resource_recorder { use anyhow::Context; use std::str::FromStr; @@ -238,10 +150,12 @@ pub mod resource_recorder { } } +#[cfg(feature = "builder")] pub mod builder { pub use super::generated::builder::*; } +#[cfg(feature = "logger")] pub mod logger { use std::str::FromStr; use std::time::Duration; @@ -261,8 +175,6 @@ pub mod logger { DeploymentId, }; - use self::logger_client::LoggerClient; - pub use super::generated::logger::*; impl From for LogItem { @@ -333,7 +245,7 @@ pub mod logger { } #[async_trait] - impl VecReceiver for LoggerClient + impl VecReceiver for logger_client::LoggerClient where T: tonic::client::GrpcService + Send + Sync + Clone, T::Error: Into, diff --git a/provisioner/Cargo.toml b/provisioner/Cargo.toml index 34f2f41c8..aaec177c1 100644 --- a/provisioner/Cargo.toml +++ b/provisioner/Cargo.toml @@ -7,8 +7,8 @@ description = "Service responsible for provisioning and managing resources for s publish = false [dependencies] -shuttle-common = { workspace = true, features = ["backend", "tonic"] } -shuttle-proto = { workspace = true } +shuttle-common = { workspace = true, features = ["backend", "models", "service", "tonic"] } +shuttle-proto = { workspace = true, features = ["provisioner"] } aws-config = "0.56.1" aws-sdk-rds = "0.33.1" diff --git a/resource-recorder/Cargo.toml b/resource-recorder/Cargo.toml index 6d30ce415..f66c651f3 100644 --- a/resource-recorder/Cargo.toml +++ b/resource-recorder/Cargo.toml @@ -7,7 +7,7 @@ repository.workspace = true [dependencies] shuttle-common = { workspace = true, features = ["backend", "tonic"] } -shuttle-proto = { workspace = true } +shuttle-proto = { workspace = true, features = ["resource-recorder"] } async-trait = { workspace = true } chrono = { workspace = true } diff --git a/resources/turso/Cargo.toml b/resources/turso/Cargo.toml index 39895d801..fbe18aa6c 100644 --- a/resources/turso/Cargo.toml +++ b/resources/turso/Cargo.toml @@ -9,12 +9,11 @@ keywords = ["shuttle-service", "turso"] [dependencies] async-trait = "0.1.56" dunce = "1.0.4" -libsql-client = { version = "0.31.0" } +libsql-client = "0.31.0" # Stays on 0.31 until https://github.com/libsql/libsql-client-rs/issues/52 serde = { version = "1.0.148", features = ["derive"] } shuttle-service = { path = "../../service", version = "0.33.0", default-features = false } url = { version = "2.3.1", features = ["serde"] } - [dev-dependencies] tempfile = "3.3.0" tokio = "1.28.2" diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index fccca82f4..4ac724f36 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -14,8 +14,8 @@ doctest = false [dependencies] shuttle-codegen = { workspace = true, features = ["frameworks"] } -shuttle-common = { workspace = true, features = ["claims"] } -shuttle-proto = { workspace = true } +shuttle-common = { workspace = true, features = ["backend", "claims"] } +shuttle-proto = { workspace = true, features = ["provisioner", "runtime"] } shuttle-service = { workspace = true } anyhow = { workspace = true } @@ -43,11 +43,12 @@ wasmtime-wasi = { version = "13.0.0", optional = true } [dev-dependencies] portpicker = "0.1.1" futures = { workspace = true } -shuttle-service = { workspace = true, features = ["builder"] } +shuttle-service = { workspace = true, features = ["builder", "runner"] } uuid = { workspace = true } [features] default = ["setup-tracing"] + next = [ "cap-std", "futures", diff --git a/runtime/tests/integration/helpers.rs b/runtime/tests/integration/helpers.rs index 47552e675..25b7005b3 100644 --- a/runtime/tests/integration/helpers.rs +++ b/runtime/tests/integration/helpers.rs @@ -12,9 +12,9 @@ use shuttle_proto::{ provisioner_server::{Provisioner, ProvisionerServer}, DatabaseDeletionResponse, DatabaseRequest, DatabaseResponse, Ping, Pong, }, - runtime::{self, runtime_client::RuntimeClient}, + runtime::runtime_client::RuntimeClient, }; -use shuttle_service::{builder::build_workspace, Environment}; +use shuttle_service::{builder::build_workspace, runner, Environment}; use tokio::process::Child; use tonic::{ transport::{Channel, Server}, @@ -49,7 +49,7 @@ pub async fn spawn_runtime(project_path: String, service_name: &str) -> Result, + port: u16, + runtime_executable: PathBuf, + project_path: &Path, +) -> anyhow::Result<( + process::Child, + runtime_client::RuntimeClient>>, +)> { + let port = &port.to_string(); + let environment = &environment.to_string(); + + let args = if wasm { + vec!["--port", port] + } else { + let mut args = vec![ + "--port", + port, + "--provisioner-address", + provisioner_address, + "--env", + environment, + ]; + + if let Some(auth_uri) = auth_uri { + args.append(&mut vec!["--auth-uri", auth_uri]); + } + + args + }; + + info!( + "Spawning runtime process: {} {}", + runtime_executable.display(), + args.join(" ") + ); + let runtime = process::Command::new( + dunce::canonicalize(runtime_executable).context("canonicalize path of executable")?, + ) + .current_dir(project_path) + .args(&args) + .stdout(Stdio::piped()) + .kill_on_drop(true) + .spawn() + .context("spawning runtime process")?; + + info!("connecting runtime client"); + let conn = Endpoint::new(format!("http://127.0.0.1:{port}")) + .context("creating runtime client endpoint")? + .connect_timeout(Duration::from_secs(5)); + + // Wait for the spawned process to open the control port. + // Connecting instantly does not give it enough time. + let channel = tokio::time::timeout(Duration::from_millis(7000), async move { + let mut ms = 5; + loop { + if let Ok(channel) = conn.connect().await { + break channel; + } + trace!("waiting for runtime control port to open"); + // exponential backoff + tokio::time::sleep(Duration::from_millis(ms)).await; + ms *= 2; + } + }) + .await + .context("runtime control port did not open in time")?; + + let channel = ServiceBuilder::new() + .layer(ClaimLayer) + .layer(InjectPropagationLayer) + .service(channel); + let runtime_client = runtime_client::RuntimeClient::new(channel); + + Ok((runtime, runtime_client)) +} From 9b1ef53c988f268ed48857a8f54f5e8d8341691d Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Tue, 21 Nov 2023 16:11:17 +0100 Subject: [PATCH 10/23] fix(cargo-shuttle): cargo shuttle clean response type (#1409) --- cargo-shuttle/src/client.rs | 2 +- cargo-shuttle/src/lib.rs | 9 ++------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/cargo-shuttle/src/client.rs b/cargo-shuttle/src/client.rs index 79ea40fdc..f0aef6a90 100644 --- a/cargo-shuttle/src/client.rs +++ b/cargo-shuttle/src/client.rs @@ -140,7 +140,7 @@ impl Client { .await } - pub async fn clean_project(&self, project: &str) -> Result> { + pub async fn clean_project(&self, project: &str) -> Result { let path = format!("/projects/{project}/clean"); self.post(path, Option::::None) diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index dbe1d5c45..29669bc8a 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -635,7 +635,7 @@ impl Shuttle { async fn clean(&self) -> Result { let client = self.client.as_ref().unwrap(); - let lines = client + let message = client .clean_project(self.ctx.project_name()) .await .map_err(|err| { @@ -646,12 +646,7 @@ impl Shuttle { "cleaning your project or checking its status fail repeatedly", ) })?; - - for line in lines { - println!("{line}"); - } - - println!("Cleaning done!"); + println!("{message}"); Ok(CommandOutcome::Ok) } From b32475ffce54b762f57d032130daa4819e49ddf1 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Tue, 21 Nov 2023 16:16:07 +0100 Subject: [PATCH 11/23] chore: Rust 1.74 (#1411) --- Makefile | 2 +- shell.nix | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 68c71243b..e09a765a2 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ BUILDX_FLAGS=$(BUILDX_OP) $(PLATFORM_FLAGS) $(CACHE_FLAGS) # the rust version used by our containers, and as an override for our deployers # ensuring all user crates are compiled with the same rustc toolchain -RUSTUP_TOOLCHAIN=1.73.0 +RUSTUP_TOOLCHAIN=1.74.0 TAG?=$(shell git describe --tags --abbrev=0) AUTH_TAG?=$(TAG) diff --git a/shell.nix b/shell.nix index f2f8d96f0..2c1b9ae44 100644 --- a/shell.nix +++ b/shell.nix @@ -11,7 +11,7 @@ in openssl ]; buildInputs = with nixpkgs; [ - ((rustChannelOf{ channel = "1.73.0"; }).rust.override { + ((rustChannelOf{ channel = "1.74.0"; }).rust.override { extensions = ["rust-src"]; targets = ["wasm32-wasi"]; }) From 8e6deaea60ffc2cba3d4ba136ef095c5fb351e58 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Wed, 22 Nov 2023 16:55:10 +0100 Subject: [PATCH 12/23] feat(cargo-shuttle): ability to force a name to be used in init (#1410) * feat(cargo-shuttle): ability to force a name to be used in init * fix: use forced name in init tests * period * clippy * nit: derive default on some args * fix: repeat input instead of --force * fix: refactorings + fix path confirmation bug * nit: refactor * nit: variable --- cargo-shuttle/src/args.rs | 27 ++--- cargo-shuttle/src/lib.rs | 141 +++++++++++++++--------- cargo-shuttle/tests/integration/init.rs | 43 ++------ 3 files changed, 111 insertions(+), 100 deletions(-) diff --git a/cargo-shuttle/src/args.rs b/cargo-shuttle/src/args.rs index 11803565a..298322fd6 100644 --- a/cargo-shuttle/src/args.rs +++ b/cargo-shuttle/src/args.rs @@ -226,7 +226,7 @@ pub struct ProjectStartArgs { pub idle_minutes: u64, } -#[derive(Parser, Clone, Debug)] +#[derive(Parser, Clone, Debug, Default)] pub struct LoginArgs { /// API key for the Shuttle platform #[arg(long)] @@ -262,7 +262,7 @@ pub struct RunArgs { pub release: bool, } -#[derive(Parser, Clone, Debug)] +#[derive(Parser, Clone, Debug, Default)] pub struct InitArgs { /// Clone a starter template from Shuttle's official examples #[arg(long, short, value_enum, conflicts_with_all = &["from", "subfolder"])] @@ -278,6 +278,9 @@ pub struct InitArgs { #[arg(default_value = ".", value_parser = OsStringValueParser::new().try_map(parse_init_path))] pub path: PathBuf, + /// Don't check the project name's validity or availability and use it anyways + #[arg(long)] + pub force_name: bool, /// Whether to start the container for this project on Shuttle, and claim the project name #[arg(long)] pub create_env: bool, @@ -399,9 +402,7 @@ mod tests { template: Some(InitTemplateArg::Tower), from: None, subfolder: None, - create_env: false, - login_args: LoginArgs { api_key: None }, - path: PathBuf::new(), + ..Default::default() }; assert_eq!( init_args.git_template().unwrap(), @@ -416,9 +417,7 @@ mod tests { template: Some(InitTemplateArg::Axum), from: None, subfolder: None, - create_env: false, - login_args: LoginArgs { api_key: None }, - path: PathBuf::new(), + ..Default::default() }; assert_eq!( init_args.git_template().unwrap(), @@ -433,9 +432,7 @@ mod tests { template: Some(InitTemplateArg::None), from: None, subfolder: None, - create_env: false, - login_args: LoginArgs { api_key: None }, - path: PathBuf::new(), + ..Default::default() }; assert_eq!( init_args.git_template().unwrap(), @@ -450,9 +447,7 @@ mod tests { template: None, from: Some("https://github.com/some/repo".into()), subfolder: Some("some/path".into()), - create_env: false, - login_args: LoginArgs { api_key: None }, - path: PathBuf::new(), + ..Default::default() }; assert_eq!( init_args.git_template().unwrap(), @@ -467,9 +462,7 @@ mod tests { template: None, from: None, subfolder: None, - create_env: false, - login_args: LoginArgs { api_key: None }, - path: PathBuf::new(), + ..Default::default() }; assert_eq!(init_args.git_template().unwrap(), None); } diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 29669bc8a..98f60b5a0 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -304,14 +304,15 @@ impl Shuttle { provided_path_to_init: bool, ) -> Result { // Turns the template or git args (if present) to a repo+folder. - let git_templates = args.git_template()?; + let git_template = args.git_template()?; let unauthorized = self.ctx.api_key().is_err() && args.login_args.api_key.is_none(); - let interactive = project_args.name.is_none() - || git_templates.is_none() - || !provided_path_to_init - || unauthorized; + let needs_name = project_args.name.is_none(); + let needs_template = git_template.is_none(); + let needs_path = !provided_path_to_init; + let needs_login = unauthorized; + let interactive = needs_name || needs_template || needs_path || needs_login; let theme = ColorfulTheme::default(); @@ -320,9 +321,9 @@ impl Shuttle { let login_args = LoginArgs { api_key: Some(api_key.as_ref().to_string()), }; - + // TODO: this re-applies an already loaded API key self.login(login_args).await?; - } else if interactive { + } else if needs_login { println!("First, let's log in to your Shuttle account."); self.login(args.login_args.clone()).await?; println!(); @@ -332,54 +333,55 @@ impl Shuttle { bail!("Tried to login to create a Shuttle environment, but no API key was set.") } - // 2. Ask for project name - if project_args.name.is_none() { + // 2. Ask for project name or validate the given one + if needs_name { printdoc! {" What do you want to name your project? It will be hosted at ${{project_name}}.shuttleapp.rs, so choose something unique! " }; - let client = self.client.as_ref().unwrap(); - loop { - // not using validate_with due to being blocking - let p: String = Input::with_theme(&theme) + } + let mut prev_name: Option = None; + loop { + // prompt if interactive + let name: String = if let Some(name) = project_args.name.clone() { + name + } else { + // not using `validate_with` due to being blocking. + Input::with_theme(&theme) .with_prompt("Project name") - .interact()?; - match client.check_project_name(&p).await { - Ok(true) => { - println!("{} {}", "Project name already taken:".red(), p); - println!("{}", "Try a different name.".yellow()); - } - Ok(false) => { - project_args.name = Some(p); - break; - } - Err(e) => { - // If API error contains message regarding format of error name, print that error and prompt again - if let Ok(api_error) = e.downcast::() { - // If the returned error string changes, this could break - if api_error.message.contains("Invalid project name") { - println!("{}", api_error.message.yellow()); - println!("{}", "Try a different name.".yellow()); - continue; - } - } - // Else, the API error was about something else. - // Ignore and keep going to not prevent the flow of the init command. - project_args.name = Some(p); - println!( - "{}", - "Failed to check if project name is available.".yellow() - ); - break; - } - } + .interact()? + }; + let force_name = args.force_name + || (needs_name && prev_name.as_ref().is_some_and(|prev| prev == &name)); + if force_name { + project_args.name = Some(name); + break; + } + // validate and take action based on result + if self + .check_project_name(&mut project_args, name.clone()) + .await + { + // success + break; + } else if needs_name { + // try again + println!(r#"Type the same name again to use "{}" anyways."#, name); + prev_name = Some(name); + } else { + // don't continue if non-interactive + bail!( + "Invalid or unavailable project name. Use `--force-name` to use this project name anyways." + ); } + } + if needs_name { println!(); } // 3. Confirm the project directory - let path = if interactive { + let path = if needs_path { let path = args .path .to_str() @@ -412,8 +414,8 @@ impl Shuttle { }; // 4. Ask for the framework - let template = match git_templates { - Some(git_templates) => git_templates, + let template = match git_template { + Some(git_template) => git_template, None => { println!( "Shuttle works with a range of web frameworks. Which one do you want to use?" @@ -429,11 +431,10 @@ impl Shuttle { } }; - let serenity_idle_hint = if let Some(s) = template.subfolder.as_ref() { - s.contains("serenity") || s.contains("poise") - } else { - false - }; + let serenity_idle_hint = template + .subfolder + .as_ref() + .is_some_and(|s| s.contains("serenity") || s.contains("poise")); // 5. Initialize locally init::generate_project( @@ -513,6 +514,44 @@ impl Shuttle { Ok(CommandOutcome::Ok) } + /// true -> success/neutral. false -> try again. + async fn check_project_name(&self, project_args: &mut ProjectArgs, name: String) -> bool { + let client = self.client.as_ref().unwrap(); + match client.check_project_name(&name).await { + Ok(true) => { + println!("{} {}", "Project name already taken:".red(), name); + println!("{}", "Try a different name.".yellow()); + + false + } + Ok(false) => { + project_args.name = Some(name); + + true + } + Err(e) => { + // If API error contains message regarding format of error name, print that error and prompt again + if let Ok(api_error) = e.downcast::() { + // If the returned error string changes, this could break + if api_error.message.contains("Invalid project name") { + println!("{}", api_error.message.yellow()); + println!("{}", "Try a different name.".yellow()); + return false; + } + } + // Else, the API error was about something else. + // Ignore and keep going to not prevent the flow of the init command. + project_args.name = Some(name); + println!( + "{}", + "Failed to check if project name is available.".yellow() + ); + + true + } + } + } + pub fn load_project(&mut self, project_args: &ProjectArgs) -> Result<()> { trace!("loading project arguments: {project_args:?}"); diff --git a/cargo-shuttle/tests/integration/init.rs b/cargo-shuttle/tests/integration/init.rs index e787d64bf..a414752cf 100644 --- a/cargo-shuttle/tests/integration/init.rs +++ b/cargo-shuttle/tests/integration/init.rs @@ -19,11 +19,10 @@ async fn non_interactive_basic_init() { let args = ShuttleArgs::parse_from([ "cargo-shuttle", - "--api-url", - "http://shuttle.invalid:80", "init", "--api-key", "dh9z58jttoes3qvt", + "--force-name", "--name", "my-project", "--template", @@ -48,11 +47,10 @@ async fn non_interactive_rocket_init() { let args = ShuttleArgs::parse_from([ "cargo-shuttle", - "--api-url", - "http://shuttle.invalid:80", "init", "--api-key", "dh9z58jttoes3qvt", + "--force-name", "--name", "my-project", "--template", @@ -75,11 +73,10 @@ async fn non_interactive_init_with_from_url() { let args = ShuttleArgs::parse_from([ "cargo-shuttle", - "--api-url", - "http://shuttle.invalid:80", "init", "--api-key", "dh9z58jttoes3qvt", + "--force-name", "--name", "my-project", "--from", @@ -106,11 +103,10 @@ async fn non_interactive_init_with_from_gh() { let args = ShuttleArgs::parse_from([ "cargo-shuttle", - "--api-url", - "http://shuttle.invalid:80", "init", "--api-key", "dh9z58jttoes3qvt", + "--force-name", "--name", "my-project", "--from", @@ -137,11 +133,10 @@ async fn non_interactive_init_with_from_repo_name() { let args = ShuttleArgs::parse_from([ "cargo-shuttle", - "--api-url", - "http://shuttle.invalid:80", "init", "--api-key", "dh9z58jttoes3qvt", + "--force-name", "--name", "my-project", "--from", @@ -168,11 +163,10 @@ async fn non_interactive_init_with_from_local_path() { let args = ShuttleArgs::parse_from([ "cargo-shuttle", - "--api-url", - "http://shuttle.invalid:80", "init", "--api-key", "dh9z58jttoes3qvt", + "--force-name", "--name", "my-project", "--from", @@ -199,13 +193,7 @@ fn interactive_rocket_init() -> Result<(), Box> { let bin_path = assert_cmd::cargo::cargo_bin("cargo-shuttle"); let mut command = Command::new(bin_path); - command.args([ - "--api-url", - "http://shuttle.invalid:80", - "init", - "--api-key", - "dh9z58jttoes3qvt", - ]); + command.args(["init", "--force-name", "--api-key", "dh9z58jttoes3qvt"]); let mut session = rexpect::session::spawn_command(command, Some(EXPECT_TIMEOUT_MS))?; session.exp_string("What do you want to name your project?")?; @@ -240,13 +228,7 @@ fn interactive_rocket_init_manually_choose_template() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Date: Wed, 22 Nov 2023 21:27:07 +0100 Subject: [PATCH 13/23] chore: bump base64 dependency to 0.21.5 (#1403) --- Cargo.lock | 2 +- Cargo.toml | 2 +- common/src/backends/auth.rs | 23 +++++++++++++++-------- 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b4c859d5..88724af96 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5819,7 +5819,7 @@ dependencies = [ "anyhow", "async-trait", "axum", - "base64 0.13.1", + "base64 0.21.5", "bytes", "chrono", "comfy-table", diff --git a/Cargo.toml b/Cargo.toml index 9783e241a..b43fe0c5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,7 +44,7 @@ anyhow = "1.0.66" async-trait = "0.1.58" axum = { version = "0.6.13", default-features = false } axum-sessions = "0.5.0" -base64 = "0.13.1" +base64 = "0.21.5" bollard = "0.15.0" bytes = "1.3.0" cargo_metadata = "0.18.1" diff --git a/common/src/backends/auth.rs b/common/src/backends/auth.rs index 895968964..d5da03f1e 100644 --- a/common/src/backends/auth.rs +++ b/common/src/backends/auth.rs @@ -458,6 +458,7 @@ impl VerifyClaim for tonic::Request { #[cfg(test)] mod tests { use axum::{routing::get, Extension, Router}; + use base64::Engine; use http::{Request, StatusCode}; use hyper::{body, Body}; use jsonwebtoken::EncodingKey; @@ -472,6 +473,12 @@ mod tests { use super::{JwtAuthenticationLayer, ScopedLayer}; + const BASE64_URL_SAFE_ENGINE: base64::engine::GeneralPurpose = + base64::engine::GeneralPurpose::new( + &base64::alphabet::URL_SAFE, + base64::engine::general_purpose::NO_PAD, + ); + #[test] fn to_token_and_back() { let mut claim = Claim::new( @@ -610,14 +617,14 @@ mod tests { let token = claim.into_token(&encoding_key).unwrap(); let (header, rest) = token.split_once('.').unwrap(); - let header = base64::decode_config(header, base64::URL_SAFE_NO_PAD).unwrap(); + let header = BASE64_URL_SAFE_ENGINE.decode(header).unwrap(); let mut header: serde_json::Map = serde_json::from_slice(&header).unwrap(); header["alg"] = json!("HS256"); let header = serde_json::to_vec(&header).unwrap(); - let header = base64::encode_config(header, base64::URL_SAFE_NO_PAD); + let header = BASE64_URL_SAFE_ENGINE.encode(header); let (claim, _sig) = rest.split_once('.').unwrap(); @@ -630,7 +637,7 @@ mod tests { &hmac::Key::new(hmac::HMAC_SHA256, pair.public_key().as_ref()), msg.as_bytes(), ); - let sig = base64::encode_config(sig, base64::URL_SAFE_NO_PAD); + let sig = BASE64_URL_SAFE_ENGINE.encode(sig); let token = format!("{msg}.{sig}"); Claim::from_token(&token, public_key).unwrap(); @@ -652,7 +659,7 @@ mod tests { let token = claim.into_token(&encoding_key).unwrap(); let (header, rest) = token.split_once('.').unwrap(); - let header = base64::decode_config(header, base64::URL_SAFE_NO_PAD).unwrap(); + let header = BASE64_URL_SAFE_ENGINE.decode(header).unwrap(); let (claim, _sig) = rest.split_once('.').unwrap(); let mut header: serde_json::Map = serde_json::from_slice(&header).unwrap(); @@ -660,7 +667,7 @@ mod tests { header["alg"] = json!("none"); let header = serde_json::to_vec(&header).unwrap(); - let header = base64::encode_config(header, base64::URL_SAFE_NO_PAD); + let header = BASE64_URL_SAFE_ENGINE.encode(header); let token = format!("{header}.{claim}."); @@ -712,14 +719,14 @@ mod tests { let (header, rest) = token.split_once('.').unwrap(); let (claim, _sig) = rest.split_once('.').unwrap(); - let claim = base64::decode_config(claim, base64::URL_SAFE_NO_PAD).unwrap(); + let claim = BASE64_URL_SAFE_ENGINE.decode(claim).unwrap(); let mut claim: serde_json::Map = serde_json::from_slice(&claim).unwrap(); claim["iss"] = json!("clone"); let claim = serde_json::to_vec(&claim).unwrap(); - let claim = base64::encode_config(claim, base64::URL_SAFE_NO_PAD); + let claim = BASE64_URL_SAFE_ENGINE.encode(claim); let msg = format!("{header}.{claim}"); @@ -727,7 +734,7 @@ mod tests { let public_key = pair.public_key().as_ref(); let sig = pair.sign(msg.as_bytes()); - let sig = base64::encode_config(sig, base64::URL_SAFE_NO_PAD); + let sig = BASE64_URL_SAFE_ENGINE.encode(sig); let token = format!("{msg}.{sig}"); Claim::from_token(&token, public_key).unwrap(); From 2afaa16b60873878d20a513e64639732e6df12e9 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Wed, 22 Nov 2023 21:28:56 +0100 Subject: [PATCH 14/23] fix: better error hints & formatting + nits (#1412) * fix: better hint on unauthorized error, formatting * better error * deser as String * nit --- .gitattributes | 1 + cargo-shuttle/src/lib.rs | 14 +++++----- common/src/models/error.rs | 54 ++++++++++++++++++------------------ common/src/models/project.rs | 5 ++-- service/src/builder.rs | 2 +- 5 files changed, 39 insertions(+), 37 deletions(-) diff --git a/.gitattributes b/.gitattributes index 84bce1bb0..b4e3de1e9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ proto/src/generated/* linguist-generated=true proto/src/generated.rs linguist-generated=true +CHANGELOG.md linguist-generated=true diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 98f60b5a0..724a0fc9b 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -225,10 +225,10 @@ impl Shuttle { self.resource_delete(&resource_type).await } Command::Project(ProjectCommand::Start(ProjectStartArgs { idle_minutes })) => { - self.project_create(idle_minutes).await + self.project_start(idle_minutes).await } Command::Project(ProjectCommand::Restart(ProjectStartArgs { idle_minutes })) => { - self.project_recreate(idle_minutes).await + self.project_restart(idle_minutes).await } Command::Project(ProjectCommand::Status { follow }) => { self.project_status(follow).await @@ -490,7 +490,7 @@ impl Shuttle { project_args.working_directory = path.clone(); self.load_project(&project_args)?; - self.project_create(DEFAULT_IDLE_MINUTES).await?; + self.project_start(DEFAULT_IDLE_MINUTES).await?; } if std::env::current_dir().is_ok_and(|d| d != path) { @@ -1667,7 +1667,7 @@ impl Shuttle { Ok(CommandOutcome::Ok) } - async fn project_create(&self, idle_minutes: u64) -> Result { + async fn project_start(&self, idle_minutes: u64) -> Result { let client = self.client.as_ref().unwrap(); let config = &project::Config { idle_minutes }; @@ -1721,11 +1721,11 @@ impl Shuttle { Ok(CommandOutcome::Ok) } - async fn project_recreate(&self, idle_minutes: u64) -> Result { + async fn project_restart(&self, idle_minutes: u64) -> Result { self.project_stop() .await .map_err(suggestions::project::project_restart_failure)?; - self.project_create(idle_minutes) + self.project_start(idle_minutes) .await .map_err(suggestions::project::project_restart_failure)?; @@ -2039,7 +2039,7 @@ fn is_dirty(repo: &Repository) -> Result<()> { } writeln!(error).expect("to append error"); - writeln!(error, "To proceed despite this and include the uncommitted changes, pass the `--allow-dirty` flag").expect("to append error"); + writeln!(error, "To proceed despite this and include the uncommitted changes, pass the `--allow-dirty` or `--ad` flag").expect("to append error"); bail!(error); } diff --git a/common/src/models/error.rs b/common/src/models/error.rs index f09bc2c38..d4ee09104 100644 --- a/common/src/models/error.rs +++ b/common/src/models/error.rs @@ -65,27 +65,27 @@ pub enum ErrorKind { impl From for ApiError { fn from(kind: ErrorKind) -> Self { let (status, error_message) = match kind { - ErrorKind::Internal => (StatusCode::INTERNAL_SERVER_ERROR, "internal server error"), - ErrorKind::KeyMissing => (StatusCode::UNAUTHORIZED, "request is missing a key"), + ErrorKind::Internal => (StatusCode::INTERNAL_SERVER_ERROR, "Internal server error"), + ErrorKind::KeyMissing => (StatusCode::UNAUTHORIZED, "Request is missing a key"), ErrorKind::ServiceUnavailable => ( StatusCode::SERVICE_UNAVAILABLE, - "we're experiencing a high workload right now, please try again in a little bit", + "We're experiencing a high workload right now, please try again in a little bit", ), - ErrorKind::KeyMalformed => (StatusCode::BAD_REQUEST, "request has an invalid key"), - ErrorKind::BadHost => (StatusCode::BAD_REQUEST, "the 'Host' header is invalid"), - ErrorKind::UserNotFound => (StatusCode::NOT_FOUND, "user not found"), - ErrorKind::UserAlreadyExists => (StatusCode::BAD_REQUEST, "user already exists"), + ErrorKind::KeyMalformed => (StatusCode::BAD_REQUEST, "Request has an invalid key"), + ErrorKind::BadHost => (StatusCode::BAD_REQUEST, "The 'Host' header is invalid"), + ErrorKind::UserNotFound => (StatusCode::NOT_FOUND, "User not found"), + ErrorKind::UserAlreadyExists => (StatusCode::BAD_REQUEST, "User already exists"), ErrorKind::ProjectNotFound => ( StatusCode::NOT_FOUND, - "project not found. Make sure you are the owner of this project name. Run `cargo shuttle project start` to create a new project.", + "Project not found. Make sure you are the owner of this project name. Run `cargo shuttle project start` to create a new project.", ), ErrorKind::ProjectNotReady => ( StatusCode::SERVICE_UNAVAILABLE, // "not ready" is matched against in cargo-shuttle for giving further instructions on project deletion - "project not ready. Try running `cargo shuttle project restart`.", + "Project not ready. Try running `cargo shuttle project restart`.", ), ErrorKind::ProjectUnavailable => { - (StatusCode::BAD_GATEWAY, "project returned invalid response") + (StatusCode::BAD_GATEWAY, "Project returned invalid response") }, ErrorKind::TooManyProjects => { (StatusCode::FORBIDDEN, "You cannot create more projects. Delete some projects first.") @@ -107,21 +107,21 @@ impl From for ApiError { status_code: StatusCode::BAD_REQUEST.as_u16(), } } - ErrorKind::InvalidOperation => (StatusCode::BAD_REQUEST, "the requested operation is invalid"), - ErrorKind::ProjectAlreadyExists => (StatusCode::BAD_REQUEST, "a project with the same name already exists"), + ErrorKind::InvalidOperation => (StatusCode::BAD_REQUEST, "The requested operation is invalid"), + ErrorKind::ProjectAlreadyExists => (StatusCode::BAD_REQUEST, "A project with the same name already exists"), ErrorKind::OwnProjectAlreadyExists(message) => { return Self { message, status_code: StatusCode::BAD_REQUEST.as_u16(), } } - ErrorKind::InvalidCustomDomain => (StatusCode::BAD_REQUEST, "invalid custom domain"), - ErrorKind::CustomDomainNotFound => (StatusCode::NOT_FOUND, "custom domain not found"), - ErrorKind::CustomDomainAlreadyExists => (StatusCode::BAD_REQUEST, "custom domain already in use"), - ErrorKind::Unauthorized => (StatusCode::UNAUTHORIZED, "unauthorized"), - ErrorKind::Forbidden => (StatusCode::FORBIDDEN, "forbidden"), - ErrorKind::NotReady => (StatusCode::INTERNAL_SERVER_ERROR, "service not ready"), - ErrorKind::DeleteProjectFailed => (StatusCode::INTERNAL_SERVER_ERROR, "deleting project failed"), + ErrorKind::InvalidCustomDomain => (StatusCode::BAD_REQUEST, "Invalid custom domain"), + ErrorKind::CustomDomainNotFound => (StatusCode::NOT_FOUND, "Custom domain not found"), + ErrorKind::CustomDomainAlreadyExists => (StatusCode::BAD_REQUEST, "Custom domain already in use"), + ErrorKind::Unauthorized => (StatusCode::UNAUTHORIZED, "Unauthorized"), + ErrorKind::Forbidden => (StatusCode::FORBIDDEN, "Forbidden"), + ErrorKind::NotReady => (StatusCode::INTERNAL_SERVER_ERROR, "Service not ready"), + ErrorKind::DeleteProjectFailed => (StatusCode::INTERNAL_SERVER_ERROR, "Deleting project failed"), ErrorKind::RateLimited(message) => { return Self { message, @@ -143,28 +143,28 @@ impl From for ApiError { StatusCode::OK | StatusCode::ACCEPTED | StatusCode::FOUND | StatusCode::SWITCHING_PROTOCOLS => { unreachable!("we should not have an API error with a successful status code") } - StatusCode::FORBIDDEN => "this request is not allowed", + StatusCode::FORBIDDEN => "This request is not allowed", StatusCode::UNAUTHORIZED => { - "we were unable to authorize your request. Is your key still valid?" + "we were unable to authorize your request. Check that your API key is set correctly. Use `cargo shuttle login` to set it." }, - StatusCode::INTERNAL_SERVER_ERROR => "our server was unable to handle your request. A ticket should be created for us to fix this.", - StatusCode::SERVICE_UNAVAILABLE => "we're experiencing a high workload right now, please try again in a little bit", + StatusCode::INTERNAL_SERVER_ERROR => "Our server was unable to handle your request. A ticket should be created for us to fix this.", + StatusCode::SERVICE_UNAVAILABLE => "We're experiencing a high workload right now, please try again in a little bit", StatusCode::BAD_REQUEST => { warn!("responding to a BAD_REQUEST request with an unhelpful message. Use ErrorKind instead"); - "this request is invalid" + "This request is invalid" }, StatusCode::NOT_FOUND => { warn!("responding to a NOT_FOUND request with an unhelpful message. Use ErrorKind instead"); - "we don't serve this resource" + "We don't serve this resource" }, StatusCode::BAD_GATEWAY => { warn!("got a bad response from the gateway"); // Gateway's default response when a request handler panicks is a 502 with some HTML. - "response from gateway is invalid. Please create a ticket to report this" + "Response from gateway is invalid. Please create a ticket to report this" }, _ => { error!(%code, "got an unexpected status code"); - "an unexpected error occurred. Please create a ticket to report this" + "An unexpected error occurred. Please create a ticket to report this" }, }; diff --git a/common/src/models/project.rs b/common/src/models/project.rs index 688f8d817..e26a24eff 100644 --- a/common/src/models/project.rs +++ b/common/src/models/project.rs @@ -331,8 +331,9 @@ pub mod name { where D: Deserializer<'de>, { - let s: String = String::deserialize(deserializer)?; - s.parse().map_err(DeError::custom) + String::deserialize(deserializer)? + .parse() + .map_err(DeError::custom) } } diff --git a/service/src/builder.rs b/service/src/builder.rs index cbb548abb..3e1f670a9 100644 --- a/service/src/builder.rs +++ b/service/src/builder.rs @@ -189,7 +189,7 @@ pub async fn clean_crate(project_path: &Path) -> anyhow::Result<()> { .await? .success() { - bail!("cargo clean failed"); + bail!("`cargo clean` failed. Did you build anything yet?"); } Ok(()) From 52c06a7ab9c17398ec8ff5328c99779736e3f819 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Wed, 22 Nov 2023 21:29:36 +0100 Subject: [PATCH 15/23] feat(shuttle-serenity): support serenity 0.11 and 0.12, optional native tls (#1416) --- services/shuttle-serenity/Cargo.toml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/services/shuttle-serenity/Cargo.toml b/services/shuttle-serenity/Cargo.toml index 3f3b3760b..66af85f73 100644 --- a/services/shuttle-serenity/Cargo.toml +++ b/services/shuttle-serenity/Cargo.toml @@ -9,7 +9,7 @@ keywords = ["shuttle-service", "serenity"] [workspace] [dependencies] -serenity = { version = "0.11.5", default-features = false, features = ["client", "gateway", "rustls_backend", "model"] } +serenity = { version = ">=0.11.5,<0.13", default-features = false, features = ["client", "gateway", "model"] } shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } [dev-dependencies] @@ -17,3 +17,9 @@ anyhow = "1.0.69" shuttle-secrets = { path = "../../resources/secrets" } tokio = { version = "1.26.0", features = ["macros", "rt-multi-thread"] } tracing = "0.1.37" + +[features] +default = ["rustls_backend"] + +rustls_backend = ["serenity/rustls_backend"] +native_tls_backend = ["serenity/native_tls_backend"] From 30b6465be9ff55f54a2b0d9f4b33e81d3401a60c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oddbj=C3=B8rn=20Gr=C3=B8dem?= <29732646+oddgrd@users.noreply.github.com> Date: Wed, 22 Nov 2023 21:34:55 +0100 Subject: [PATCH 16/23] feat: limit rds access to pro users (#1398) * feat: limit rds access to pro users * revert(resource-recorder): rds permission guard --- common/src/backends/auth.rs | 22 +++++++++++++++++++++- common/src/limits.rs | 22 +++++++++++++++++++--- provisioner/src/lib.rs | 3 +++ resource-recorder/src/lib.rs | 1 + 4 files changed, 44 insertions(+), 4 deletions(-) diff --git a/common/src/backends/auth.rs b/common/src/backends/auth.rs index d5da03f1e..4ed907129 100644 --- a/common/src/backends/auth.rs +++ b/common/src/backends/auth.rs @@ -15,7 +15,10 @@ use tower::{Layer, Service}; use tracing::{error, trace, Span}; use tracing_opentelemetry::OpenTelemetrySpanExt; -use crate::claims::{Claim, Scope}; +use crate::{ + claims::{Claim, Scope}, + limits::ClaimExt, +}; use super::{ cache::{CacheManagement, CacheManager}, @@ -428,6 +431,8 @@ pub trait VerifyClaim { type Error; fn verify(&self, required_scope: Scope) -> Result<(), Self::Error>; + + fn verify_rds_access(&self) -> Result<(), Self::Error>; } #[cfg(feature = "tonic")] @@ -453,6 +458,21 @@ impl VerifyClaim for tonic::Request { ))) } } + + fn verify_rds_access(&self) -> Result<(), Self::Error> { + let claim = self + .extensions() + .get::() + .ok_or_else(|| tonic::Status::internal("could not get claim"))?; + + if claim.can_provision_rds() { + Ok(()) + } else { + Err(tonic::Status::permission_denied( + "don't have permission to provision rds instances", + )) + } + } } #[cfg(test)] diff --git a/common/src/limits.rs b/common/src/limits.rs index 96dbfd4b5..858fcc297 100644 --- a/common/src/limits.rs +++ b/common/src/limits.rs @@ -9,24 +9,34 @@ use crate::{ pub struct Limits { /// The amount of projects this user can create. project_limit: u32, + /// Whether this user has permission to provision RDS instances. + rds_access: bool, } impl Default for Limits { fn default() -> Self { Self { project_limit: MAX_PROJECTS_DEFAULT, + rds_access: false, } } } impl Limits { - pub fn new(project_limit: u32) -> Self { - Self { project_limit } + pub fn new(project_limit: u32, rds_limit: bool) -> Self { + Self { + project_limit, + rds_access: rds_limit, + } } pub fn project_limit(&self) -> u32 { self.project_limit } + + pub fn rds_access(&self) -> bool { + self.rds_access + } } impl From for Limits { @@ -36,7 +46,7 @@ impl From for Limits { | AccountTier::Basic | AccountTier::PendingPaymentPro | AccountTier::Deployer => Self::default(), - AccountTier::Pro | AccountTier::Team => Self::new(MAX_PROJECTS_EXTRA), + AccountTier::Pro | AccountTier::Team => Self::new(MAX_PROJECTS_EXTRA, true), } } } @@ -46,6 +56,8 @@ pub trait ClaimExt { fn is_admin(&self) -> bool; /// Verify that the user's current project count is lower than the account limit in [Claim::limits]. fn can_create_project(&self, current_count: u32) -> bool; + /// Verify that the user has permission to provision RDS instances. + fn can_provision_rds(&self) -> bool; } impl ClaimExt for Claim { @@ -56,4 +68,8 @@ impl ClaimExt for Claim { fn can_create_project(&self, current_count: u32) -> bool { self.is_admin() || self.limits.project_limit() > current_count } + + fn can_provision_rds(&self) -> bool { + self.is_admin() || self.limits.rds_access + } } diff --git a/provisioner/src/lib.rs b/provisioner/src/lib.rs index 76a3c56d4..1f2f31c54 100644 --- a/provisioner/src/lib.rs +++ b/provisioner/src/lib.rs @@ -456,6 +456,8 @@ impl Provisioner for MyProvisioner { ) -> Result, Status> { request.verify(Scope::ResourcesWrite)?; + let can_provision_rds = request.verify_rds_access(); + let request = request.into_inner(); if !ProjectName::is_valid(&request.project_name) { return Err(Status::invalid_argument("invalid project name")); @@ -468,6 +470,7 @@ impl Provisioner for MyProvisioner { .await? } DbType::AwsRds(AwsRds { engine }) => { + can_provision_rds?; self.request_aws_rds(&request.project_name, engine.expect("engine to be set")) .await? } diff --git a/resource-recorder/src/lib.rs b/resource-recorder/src/lib.rs index cca7461cf..fd19075ab 100644 --- a/resource-recorder/src/lib.rs +++ b/resource-recorder/src/lib.rs @@ -135,6 +135,7 @@ where request.verify(Scope::ResourcesWrite)?; let request = request.into_inner(); + let result = match self.add(request).await { Ok(()) => ResultResponse { success: true, From 4e7dd2d49c7c847d77727ef156b52ebdff6d357c Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Wed, 22 Nov 2023 21:56:16 +0100 Subject: [PATCH 17/23] chore: v0.34.0 (#1417) --- .github/ISSUE_TEMPLATE/BUG-REPORT.yml | 2 +- Cargo.lock | 32 +++++++++---------- Cargo.toml | 14 ++++---- admin/Cargo.toml | 2 +- auth/Cargo.toml | 2 +- builder/Cargo.toml | 2 +- cargo-shuttle/Cargo.toml | 2 +- cargo-shuttle/README.md | 9 +++--- codegen/Cargo.toml | 2 +- common/Cargo.toml | 2 +- deployer/Cargo.toml | 2 +- .../tests/deploy_layer/bind-panic/Cargo.toml | 2 +- .../tests/deploy_layer/main-panic/Cargo.toml | 2 +- .../tests/deploy_layer/self-stop/Cargo.toml | 2 +- .../tests/deploy_layer/sleep-async/Cargo.toml | 2 +- gateway/Cargo.toml | 2 +- logger/Cargo.toml | 2 +- provisioner/Cargo.toml | 2 +- resource-recorder/Cargo.toml | 2 +- resources/aws-rds/Cargo.toml | 4 +-- resources/metadata/Cargo.toml | 4 +-- resources/persist/Cargo.toml | 4 +-- resources/secrets/Cargo.toml | 4 +-- resources/shared-db/Cargo.toml | 4 +-- resources/turso/Cargo.toml | 4 +-- runtime/Cargo.toml | 2 +- runtime/src/lib.rs | 6 ++-- service/Cargo.toml | 2 +- services/shuttle-actix-web/Cargo.toml | 4 +-- services/shuttle-axum/Cargo.toml | 4 +-- services/shuttle-next/Cargo.toml | 6 ++-- services/shuttle-poem/Cargo.toml | 4 +-- services/shuttle-poise/Cargo.toml | 4 +-- services/shuttle-rocket/Cargo.toml | 4 +-- services/shuttle-salvo/Cargo.toml | 4 +-- services/shuttle-serenity/Cargo.toml | 4 +-- services/shuttle-thruster/Cargo.toml | 4 +-- services/shuttle-tide/Cargo.toml | 4 +-- services/shuttle-tower/Cargo.toml | 4 +-- services/shuttle-warp/Cargo.toml | 4 +-- 40 files changed, 85 insertions(+), 86 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml index 61c94077f..884833f6b 100644 --- a/.github/ISSUE_TEMPLATE/BUG-REPORT.yml +++ b/.github/ISSUE_TEMPLATE/BUG-REPORT.yml @@ -20,7 +20,7 @@ body: attributes: label: Version description: What version of `cargo-shuttle` are you running (`cargo shuttle --version`)? - placeholder: "v0.33.0" + placeholder: "v0.34.0" validations: required: true - type: dropdown diff --git a/Cargo.lock b/Cargo.lock index 88724af96..a7d0b9b8a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1121,7 +1121,7 @@ dependencies = [ [[package]] name = "cargo-shuttle" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "assert_cmd", @@ -5723,7 +5723,7 @@ dependencies = [ [[package]] name = "shuttle-admin" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "clap", @@ -5740,7 +5740,7 @@ dependencies = [ [[package]] name = "shuttle-auth" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "async-stripe", @@ -5771,7 +5771,7 @@ dependencies = [ [[package]] name = "shuttle-builder" -version = "0.33.0" +version = "0.34.0" dependencies = [ "async-trait", "clap", @@ -5797,7 +5797,7 @@ dependencies = [ [[package]] name = "shuttle-codegen" -version = "0.33.0" +version = "0.34.0" dependencies = [ "pretty_assertions", "proc-macro-error", @@ -5814,7 +5814,7 @@ dependencies = [ [[package]] name = "shuttle-common" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "async-trait", @@ -5865,7 +5865,7 @@ dependencies = [ [[package]] name = "shuttle-common-tests" -version = "0.33.0" +version = "0.34.0" dependencies = [ "cargo-shuttle", "hyper", @@ -5881,7 +5881,7 @@ dependencies = [ [[package]] name = "shuttle-deployer" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "async-trait", @@ -5933,7 +5933,7 @@ dependencies = [ [[package]] name = "shuttle-gateway" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "async-trait", @@ -5991,7 +5991,7 @@ dependencies = [ [[package]] name = "shuttle-logger" -version = "0.33.0" +version = "0.34.0" dependencies = [ "async-trait", "chrono", @@ -6021,11 +6021,11 @@ dependencies = [ [[package]] name = "shuttle-orchestrator" -version = "0.33.0" +version = "0.34.0" [[package]] name = "shuttle-proto" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "chrono", @@ -6041,7 +6041,7 @@ dependencies = [ [[package]] name = "shuttle-provisioner" -version = "0.33.0" +version = "0.34.0" dependencies = [ "aws-config", "aws-sdk-rds", @@ -6066,7 +6066,7 @@ dependencies = [ [[package]] name = "shuttle-resource-recorder" -version = "0.33.0" +version = "0.34.0" dependencies = [ "async-trait", "chrono", @@ -6090,7 +6090,7 @@ dependencies = [ [[package]] name = "shuttle-runtime" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "async-trait", @@ -6123,7 +6123,7 @@ dependencies = [ [[package]] name = "shuttle-service" -version = "0.33.0" +version = "0.34.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index b43fe0c5d..55c42ba2b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,19 +26,19 @@ exclude = [ ] [workspace.package] -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" repository = "https://github.com/shuttle-hq/shuttle" # https://doc.rust-lang.org/cargo/reference/workspaces.html#the-workspacedependencies-table [workspace.dependencies] -shuttle-codegen = { path = "codegen", version = "0.33.0" } -shuttle-common = { path = "common", version = "0.33.0" } -shuttle-common-tests = { path = "common-tests", version = "0.33.0" } -shuttle-orchestrator = { path = "orchestrator", version = "0.33.0" } -shuttle-proto = { path = "proto", version = "0.33.0" } -shuttle-service = { path = "service", version = "0.33.0" } +shuttle-codegen = { path = "codegen", version = "0.34.0" } +shuttle-common = { path = "common", version = "0.34.0" } +shuttle-common-tests = { path = "common-tests", version = "0.34.0" } +shuttle-orchestrator = { path = "orchestrator", version = "0.34.0" } +shuttle-proto = { path = "proto", version = "0.34.0" } +shuttle-service = { path = "service", version = "0.34.0" } anyhow = "1.0.66" async-trait = "0.1.58" diff --git a/admin/Cargo.toml b/admin/Cargo.toml index 25d44c04d..94b0b1ad3 100644 --- a/admin/Cargo.toml +++ b/admin/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-admin" -version = "0.33.0" +version = "0.34.0" edition = "2021" [dependencies] diff --git a/auth/Cargo.toml b/auth/Cargo.toml index cbcadc077..bce1b4990 100644 --- a/auth/Cargo.toml +++ b/auth/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-auth" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/builder/Cargo.toml b/builder/Cargo.toml index 2bb30e436..7cc13b2e9 100644 --- a/builder/Cargo.toml +++ b/builder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-builder" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index e026ed35e..b443819f1 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "cargo-shuttle" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/cargo-shuttle/README.md b/cargo-shuttle/README.md index 3ef42b518..bbcbb5d14 100644 --- a/cargo-shuttle/README.md +++ b/cargo-shuttle/README.md @@ -98,7 +98,6 @@ Commands: logs View the logs of a deployment in this shuttle service project List or manage projects on shuttle resource Manage resources of a shuttle project - secrets Manage secrets for this shuttle service clean Remove cargo build artifacts in the shuttle environment login Login to the shuttle platform logout Log out of the shuttle platform @@ -142,10 +141,10 @@ cargo shuttle init --template rocket my-rocket-app This should generate the following dependency in `Cargo.toml`: ```toml -rocket = "0.5.0-rc.2" -shuttle-rocket = { version = "0.33.0" } -shuttle-runtime = { version = "0.33.0" } -tokio = { version = "1.26.0" } +rocket = "0.5.0" +shuttle-rocket = "0.34.0" +shuttle-runtime = "0.34.0" +tokio = "1.26.0" ``` The following boilerplate code should be generated into `src/lib.rs`: diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 61a291c61..1fe4402d3 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-codegen" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/common/Cargo.toml b/common/Cargo.toml index 0678142e2..520ff4aa0 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-common" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/deployer/Cargo.toml b/deployer/Cargo.toml index 94b7766d6..e704c92ce 100644 --- a/deployer/Cargo.toml +++ b/deployer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-deployer" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true description = "Service with instances created per project for handling the compilation, loading, and execution of Shuttle services" diff --git a/deployer/tests/deploy_layer/bind-panic/Cargo.toml b/deployer/tests/deploy_layer/bind-panic/Cargo.toml index 64719bf9e..7aab9163f 100644 --- a/deployer/tests/deploy_layer/bind-panic/Cargo.toml +++ b/deployer/tests/deploy_layer/bind-panic/Cargo.toml @@ -7,5 +7,5 @@ edition = "2021" [workspace] [dependencies] -shuttle-runtime = "0.33.0" +shuttle-runtime = "0.34.0" tokio = "1.22" diff --git a/deployer/tests/deploy_layer/main-panic/Cargo.toml b/deployer/tests/deploy_layer/main-panic/Cargo.toml index 49a4c7a85..d734238ff 100644 --- a/deployer/tests/deploy_layer/main-panic/Cargo.toml +++ b/deployer/tests/deploy_layer/main-panic/Cargo.toml @@ -7,5 +7,5 @@ edition = "2021" [workspace] [dependencies] -shuttle-runtime = "0.33.0" +shuttle-runtime = "0.34.0" tokio = "1.22" diff --git a/deployer/tests/deploy_layer/self-stop/Cargo.toml b/deployer/tests/deploy_layer/self-stop/Cargo.toml index 191e27c56..54b29b9ff 100644 --- a/deployer/tests/deploy_layer/self-stop/Cargo.toml +++ b/deployer/tests/deploy_layer/self-stop/Cargo.toml @@ -7,5 +7,5 @@ edition = "2021" [workspace] [dependencies] -shuttle-runtime = "0.33.0" +shuttle-runtime = "0.34.0" tokio = "1.22" diff --git a/deployer/tests/deploy_layer/sleep-async/Cargo.toml b/deployer/tests/deploy_layer/sleep-async/Cargo.toml index be98039e9..c9ec41d63 100644 --- a/deployer/tests/deploy_layer/sleep-async/Cargo.toml +++ b/deployer/tests/deploy_layer/sleep-async/Cargo.toml @@ -7,5 +7,5 @@ edition = "2021" [workspace] [dependencies] -shuttle-runtime = "0.33.0" +shuttle-runtime = "0.34.0" tokio = { version = "1.0", features = ["time"]} diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index b21fa22a8..bb1aeb208 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-gateway" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true publish = false diff --git a/logger/Cargo.toml b/logger/Cargo.toml index 508d0db98..07326a41d 100644 --- a/logger/Cargo.toml +++ b/logger/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-logger" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/provisioner/Cargo.toml b/provisioner/Cargo.toml index aaec177c1..6d114b0df 100644 --- a/provisioner/Cargo.toml +++ b/provisioner/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-provisioner" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true description = "Service responsible for provisioning and managing resources for services" diff --git a/resource-recorder/Cargo.toml b/resource-recorder/Cargo.toml index f66c651f3..d52848d8e 100644 --- a/resource-recorder/Cargo.toml +++ b/resource-recorder/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-resource-recorder" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/resources/aws-rds/Cargo.toml b/resources/aws-rds/Cargo.toml index 0ce90e572..daf76a9e3 100644 --- a/resources/aws-rds/Cargo.toml +++ b/resources/aws-rds/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-aws-rds" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Plugin to provision AWS RDS resources" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "rds"] async-trait = "0.1.56" paste = "1.0.7" serde = { version = "1.0.148", features = ["derive"] } -shuttle-service = { path = "../../service", version = "0.33.0", default-features = false } +shuttle-service = { path = "../../service", version = "0.34.0", default-features = false } sqlx = "0.7.1" [features] diff --git a/resources/metadata/Cargo.toml b/resources/metadata/Cargo.toml index d4330bfb2..cfabc690e 100644 --- a/resources/metadata/Cargo.toml +++ b/resources/metadata/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-metadata" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Plugin to get Shuttle service information" @@ -8,4 +8,4 @@ keywords = ["shuttle-service", "metadata"] [dependencies] async-trait = "0.1.56" -shuttle-service = { path = "../../service", version = "0.33.0" } +shuttle-service = { path = "../../service", version = "0.34.0" } diff --git a/resources/persist/Cargo.toml b/resources/persist/Cargo.toml index 6d7e244dd..71bdab10e 100644 --- a/resources/persist/Cargo.toml +++ b/resources/persist/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-persist" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Plugin for persist objects" @@ -10,5 +10,5 @@ keywords = ["shuttle-service", "persistence"] async-trait = "0.1.56" bincode = "1.2.1" serde = { version = "1.0.0", features = ["derive"] } -shuttle-service = { path = "../../service", version = "0.33.0" } +shuttle-service = { path = "../../service", version = "0.34.0" } thiserror = "1.0.32" diff --git a/resources/secrets/Cargo.toml b/resources/secrets/Cargo.toml index 0f9206adf..14d888b6c 100644 --- a/resources/secrets/Cargo.toml +++ b/resources/secrets/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-secrets" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Plugin to for managing secrets on shuttle" @@ -10,4 +10,4 @@ keywords = ["shuttle-service", "secrets"] async-trait = "0.1.56" secrecy = { version = "0.8.0", features = ["serde"] } serde = { version = "1.0.148", features = ["derive"] } -shuttle-service = { path = "../../service", version = "0.33.0" } +shuttle-service = { path = "../../service", version = "0.34.0" } diff --git a/resources/shared-db/Cargo.toml b/resources/shared-db/Cargo.toml index 99d22f367..2b5f2373f 100644 --- a/resources/shared-db/Cargo.toml +++ b/resources/shared-db/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-shared-db" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Plugin for managing shared databases on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "database"] async-trait = "0.1.56" mongodb = { version = "2.3.0", optional = true } serde = { version = "1.0.148", features = ["derive"] } -shuttle-service = { path = "../../service", version = "0.33.0" } +shuttle-service = { path = "../../service", version = "0.34.0" } sqlx = { version = "0.7.1", optional = true } [features] diff --git a/resources/turso/Cargo.toml b/resources/turso/Cargo.toml index fbe18aa6c..3672769a3 100644 --- a/resources/turso/Cargo.toml +++ b/resources/turso/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-turso" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Plugin to obtain a client connected to a Turso database" @@ -11,7 +11,7 @@ async-trait = "0.1.56" dunce = "1.0.4" libsql-client = "0.31.0" # Stays on 0.31 until https://github.com/libsql/libsql-client-rs/issues/52 serde = { version = "1.0.148", features = ["derive"] } -shuttle-service = { path = "../../service", version = "0.33.0", default-features = false } +shuttle-service = { path = "../../service", version = "0.34.0", default-features = false } url = { version = "2.3.1", features = ["serde"] } [dev-dependencies] diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 4ac724f36..158a9b2e1 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-runtime" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true description = "Runtime to start and manage any service that runs on shuttle" diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 0affedbc1..1afc534c1 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -28,8 +28,8 @@ //! //! ```toml //! axum = "0.6.20" -//! shuttle-axum = "0.33.0" -//! shuttle-runtime = "0.33.0" +//! shuttle-axum = "0.34.0" +//! shuttle-runtime = "0.34.0" //! tokio = "1.28.2" //! ``` //! @@ -112,7 +112,7 @@ //! `runtime-tokio-native-tls` and `postgres` features inside `Cargo.toml`: //! //! ```toml -//! shuttle-shared-db = { version = "0.33.0", features = ["postgres"] } +//! shuttle-shared-db = { version = "0.34.0", features = ["postgres"] } //! sqlx = "0.7.1" //! ``` //! diff --git a/service/Cargo.toml b/service/Cargo.toml index 2e7869a34..8fd338121 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-service" -version = "0.33.0" +version = "0.34.0" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/services/shuttle-actix-web/Cargo.toml b/services/shuttle-actix-web/Cargo.toml index b3878f2a8..f5b96400b 100644 --- a/services/shuttle-actix-web/Cargo.toml +++ b/services/shuttle-actix-web/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-actix-web" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run an actix webserver on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "actix"] [dependencies] actix-web = { version = "4.3.1" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } num_cpus = "1.15.0" [dev-dependencies] diff --git a/services/shuttle-axum/Cargo.toml b/services/shuttle-axum/Cargo.toml index f82de4efa..2f20f7105 100644 --- a/services/shuttle-axum/Cargo.toml +++ b/services/shuttle-axum/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-axum" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run an axum webserver on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "axum"] [dependencies] axum = { version = "0.6.10" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] tokio = { version = "1.26.0", features = ["macros", "rt-multi-thread"] } diff --git a/services/shuttle-next/Cargo.toml b/services/shuttle-next/Cargo.toml index fbddf9cc5..6306a2ce5 100644 --- a/services/shuttle-next/Cargo.toml +++ b/services/shuttle-next/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-next" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Macros and aliases to deploy wasm on the shuttle platform (https://www.shuttle.rs/)" @@ -18,6 +18,6 @@ futures-executor = "0.3.21" http = "0.2.7" rmp-serde = "1.1.1" tower-service = "0.3.1" -shuttle-common = { path = "../../common", version = "0.33.0", features = ["wasm"] } -shuttle-codegen = { path = "../../codegen", version = "0.33.0", features = ["next"] } +shuttle-common = { path = "../../common", version = "0.34.0", features = ["wasm"] } +shuttle-codegen = { path = "../../codegen", version = "0.34.0", features = ["next"] } tracing-subscriber = { version = "0.3.16", features = ["env-filter", "registry"] } diff --git a/services/shuttle-poem/Cargo.toml b/services/shuttle-poem/Cargo.toml index 576e1afe8..a8c902e51 100644 --- a/services/shuttle-poem/Cargo.toml +++ b/services/shuttle-poem/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-poem" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a poem webserver on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "poem"] [dependencies] poem = { version = "1.3.55" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] tokio = { version = "1.26.0", features = ["macros", "rt-multi-thread"] } diff --git a/services/shuttle-poise/Cargo.toml b/services/shuttle-poise/Cargo.toml index 397a46f47..07de97477 100644 --- a/services/shuttle-poise/Cargo.toml +++ b/services/shuttle-poise/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-poise" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a poise discord bot on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "poise", "discord-bot", "serenity"] [dependencies] poise = { version = "0.5.2" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] shuttle-secrets = { path = "../../resources/secrets" } diff --git a/services/shuttle-rocket/Cargo.toml b/services/shuttle-rocket/Cargo.toml index 5d3b65974..ce633d5dd 100644 --- a/services/shuttle-rocket/Cargo.toml +++ b/services/shuttle-rocket/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-rocket" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a rocket webserver on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "rocket"] [dependencies] rocket = { version = "0.5.0" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] tokio = { version = "1.26.0", features = ["macros", "rt-multi-thread"] } diff --git a/services/shuttle-salvo/Cargo.toml b/services/shuttle-salvo/Cargo.toml index 1ca6015c5..6f02dbd88 100644 --- a/services/shuttle-salvo/Cargo.toml +++ b/services/shuttle-salvo/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-salvo" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a salvo webserver on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "salvo"] [dependencies] salvo = { version = "0.41.0" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] tokio = { version = "1.26.0", features = ["macros", "rt-multi-thread"] } diff --git a/services/shuttle-serenity/Cargo.toml b/services/shuttle-serenity/Cargo.toml index 66af85f73..8523acc27 100644 --- a/services/shuttle-serenity/Cargo.toml +++ b/services/shuttle-serenity/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-serenity" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a serenity server on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "serenity"] [dependencies] serenity = { version = ">=0.11.5,<0.13", default-features = false, features = ["client", "gateway", "model"] } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] anyhow = "1.0.69" diff --git a/services/shuttle-thruster/Cargo.toml b/services/shuttle-thruster/Cargo.toml index fccb7ee77..6c7140635 100644 --- a/services/shuttle-thruster/Cargo.toml +++ b/services/shuttle-thruster/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-thruster" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a thruster webserver on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "thruster"] [dependencies] thruster = { version = "1.3.0" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] thruster = { version = "1.3.0", features = ["hyper_server"] } diff --git a/services/shuttle-tide/Cargo.toml b/services/shuttle-tide/Cargo.toml index 7ea75f06e..dff6c9170 100644 --- a/services/shuttle-tide/Cargo.toml +++ b/services/shuttle-tide/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-tide" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a tide webserver on shuttle" @@ -13,7 +13,7 @@ keywords = ["shuttle-service", "tide"] # https://github.com/http-rs/tide/issues/791 async-std = { version = "1.12.0", features = ["tokio1"] } tide = { version = "0.16.0" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] tokio = { version = "1.26.0", features = ["macros", "rt-multi-thread"] } diff --git a/services/shuttle-tower/Cargo.toml b/services/shuttle-tower/Cargo.toml index d7824d261..27f00ca3e 100644 --- a/services/shuttle-tower/Cargo.toml +++ b/services/shuttle-tower/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-tower" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a tower webserver on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "tower"] [dependencies] hyper = { version = "0.14.23", features = ["server", "tcp", "http1"] } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } tower = { version = "0.4.13", features = ["make"] } [dev-dependencies] diff --git a/services/shuttle-warp/Cargo.toml b/services/shuttle-warp/Cargo.toml index 201e5d2d7..bdec524e3 100644 --- a/services/shuttle-warp/Cargo.toml +++ b/services/shuttle-warp/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-warp" -version = "0.33.0" +version = "0.34.0" edition = "2021" license = "Apache-2.0" description = "Service implementation to run a warp webserver on shuttle" @@ -10,7 +10,7 @@ keywords = ["shuttle-service", "warp"] [dependencies] warp = { version = "0.3.3" } -shuttle-runtime = { path = "../../runtime", version = "0.33.0", default-features = false } +shuttle-runtime = { path = "../../runtime", version = "0.34.0", default-features = false } [dev-dependencies] tokio = { version = "1.26.0", features = ["macros", "rt-multi-thread"] } From 45688054408b4a95ead8bfae365f063060ef2cb2 Mon Sep 17 00:00:00 2001 From: Iulian Barbu <14218860+iulianbarbu@users.noreply.github.com> Date: Thu, 23 Nov 2023 02:46:50 +0200 Subject: [PATCH 18/23] fix(auth): comment healthcheck `start_period` & `start_interval` (#1414) --- docker-compose.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f6bc7e10c..6c7bf02e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,8 +58,10 @@ services: interval: 1m timeout: 10s retries: 5 - start_period: 10s - start_interval: 2s + # These options are pending on an upgrade to Docker Engine version 25.0. + # Using them will result in error because they are not handled gracefully. + # start_period: 10s + # start_interval: 2s builder: image: "${CONTAINER_REGISTRY}/builder:${BUILDER_TAG}" depends_on: From da6b8873624530e81033ea2068854610aa3b275f Mon Sep 17 00:00:00 2001 From: Iulian Barbu <14218860+iulianbarbu@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:09:07 +0200 Subject: [PATCH 19/23] fix(common): type conversion from str for a custom resource (#1415) --- common/src/resource.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/resource.rs b/common/src/resource.rs index 8d247dcb6..c8cb0d80f 100644 --- a/common/src/resource.rs +++ b/common/src/resource.rs @@ -56,6 +56,7 @@ impl FromStr for Type { "metadata" => Ok(Self::Metadata), "persist" => Ok(Self::Persist), "turso" => Ok(Self::Turso), + "custom" => Ok(Self::Custom), _ => Err(format!("'{s}' is an unknown resource type")), } } From f37a0e87c06938f0109e718c78085cb4c9267173 Mon Sep 17 00:00:00 2001 From: Pieter Date: Thu, 23 Nov 2023 10:14:41 +0000 Subject: [PATCH 20/23] bug: delete a project even if the current state is destroyed (#1413) * tests: rename delete_project to stop_project since that is more accurate * tests: deleting a running project with nothing in it * refactor: remove unused hyper object * refactor: remove unneeded hyper field * feat: make it easy to recreate a router * refactor: make it easier to get an authorization bearer * refactor: make it easier to create a project for other tests * bug: deleting a project that is destroyed * refactor: missing comments and optimizations * refactor: clippy suggestions * refactor: reduce test setup code --- Cargo.lock | 1 + gateway/Cargo.toml | 1 + gateway/src/api/latest.rs | 46 ++++++- gateway/src/args.rs | 3 + gateway/src/lib.rs | 258 +++++++++++++++++++++++++++++++++++--- gateway/src/project.rs | 11 +- gateway/src/service.rs | 12 ++ gateway/src/task.rs | 16 +++ 8 files changed, 324 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a7d0b9b8a..578a5d5c2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5973,6 +5973,7 @@ dependencies = [ "sqlx", "strum 0.25.0", "tempfile", + "test-context", "tokio", "tonic 0.10.2", "tower", diff --git a/gateway/Cargo.toml b/gateway/Cargo.toml index bb1aeb208..8493d8f84 100644 --- a/gateway/Cargo.toml +++ b/gateway/Cargo.toml @@ -66,3 +66,4 @@ portpicker = { workspace = true } ring = { workspace = true } snailquote = "0.3.1" tempfile = { workspace = true } +test-context = "0.1.4" diff --git a/gateway/src/api/latest.rs b/gateway/src/api/latest.rs index 917c76e82..d052cb4bd 100644 --- a/gateway/src/api/latest.rs +++ b/gateway/src/api/latest.rs @@ -37,6 +37,7 @@ use tokio::sync::{Mutex, MutexGuard}; use tower::ServiceBuilder; use tracing::{field, instrument, trace}; use ttl_cache::TtlCache; +use ulid::Ulid; use utoipa::openapi::security::{ApiKey, ApiKeyValue, SecurityScheme}; use utoipa::IntoParams; use utoipa::{Modify, OpenApi}; @@ -319,6 +320,24 @@ async fn delete_project( req: Request, ) -> Result, Error> { let project_name = scoped_user.scope.clone(); + let project = state.service.find_project(&project_name).await?; + let project_id = + Ulid::from_string(&project.project_id).expect("stored project id to be a valid ULID"); + + // Try to startup a destroyed project first + if project.state.is_destroyed() { + let handle = state + .service + .new_task() + .project(project_name.clone()) + .and_then(task::restart(project_id)) + .send(&state.sender) + .await?; + + // Wait for the project to be ready + handle.await; + } + let service = state.service.clone(); let sender = state.sender.clone(); @@ -1098,17 +1117,19 @@ pub mod tests { use axum::headers::Authorization; use axum::http::Request; use futures::TryFutureExt; + use http::Method; use hyper::body::to_bytes; use hyper::StatusCode; use serde_json::Value; use shuttle_common::constants::limits::{MAX_PROJECTS_DEFAULT, MAX_PROJECTS_EXTRA}; + use test_context::test_context; use tokio::sync::mpsc::channel; use tokio::sync::oneshot; use tower::Service; use super::*; use crate::service::GatewayService; - use crate::tests::{RequestBuilderExt, World}; + use crate::tests::{RequestBuilderExt, TestProject, World}; #[tokio::test] async fn api_create_get_delete_projects() -> anyhow::Result<()> { @@ -1140,7 +1161,7 @@ pub mod tests { .unwrap() }; - let delete_project = |project: &str| { + let stop_project = |project: &str| { Request::builder() .method("DELETE") .uri(format!("/projects/{project}")) @@ -1197,7 +1218,7 @@ pub mod tests { .unwrap(); router - .call(delete_project("matrix").with_header(&authorization)) + .call(stop_project("matrix").with_header(&authorization)) .map_ok(|resp| { assert_eq!(resp.status(), StatusCode::OK); }) @@ -1223,7 +1244,7 @@ pub mod tests { .unwrap(); router - .call(delete_project("reloaded").with_header(&authorization)) + .call(stop_project("reloaded").with_header(&authorization)) .map_ok(|resp| { assert_eq!(resp.status(), StatusCode::NOT_FOUND); }) @@ -1364,6 +1385,23 @@ pub mod tests { Ok(()) } + #[test_context(TestProject)] + #[tokio::test] + async fn api_delete_project_that_is_ready(project: &mut TestProject) -> anyhow::Result<()> { + project.router_call(Method::DELETE, "/delete").await; + + Ok(()) + } + + #[test_context(TestProject)] + #[tokio::test] + async fn api_delete_project_that_is_destroyed(project: &mut TestProject) -> anyhow::Result<()> { + project.destroy_project().await; + project.router_call(Method::DELETE, "/delete").await; + + Ok(()) + } + #[tokio::test(flavor = "multi_thread")] async fn status() { let world = World::new().await; diff --git a/gateway/src/args.rs b/gateway/src/args.rs index abfaa3474..4448a3108 100644 --- a/gateway/src/args.rs +++ b/gateway/src/args.rs @@ -74,4 +74,7 @@ pub struct ContextArgs { /// Api key for the user that has rights to start deploys #[arg(long, default_value = "gateway4deployes")] pub deploys_api_key: String, + + /// Allow tests to set some extra /etc/hosts + pub extra_hosts: Vec, } diff --git a/gateway/src/lib.rs b/gateway/src/lib.rs index 322875c0e..396a613e8 100644 --- a/gateway/src/lib.rs +++ b/gateway/src/lib.rs @@ -267,6 +267,7 @@ pub mod tests { use bollard::Docker; use fqdn::FQDN; use futures::prelude::*; + use http::Method; use hyper::client::HttpConnector; use hyper::http::uri::Scheme; use hyper::http::Uri; @@ -279,7 +280,10 @@ pub mod tests { use shuttle_common::models::project; use sqlx::sqlite::SqliteConnectOptions; use sqlx::SqlitePool; + use test_context::AsyncTestContext; use tokio::sync::mpsc::channel; + use tokio::time::sleep; + use tower::Service; use crate::acme::AcmeClient; use crate::api::latest::ApiBuilder; @@ -478,7 +482,6 @@ pub mod tests { docker: Docker, settings: ContainerSettings, args: StartArgs, - hyper: HyperClient, pool: SqlitePool, acme_client: AcmeClient, auth_service: Arc>, @@ -489,7 +492,6 @@ pub mod tests { pub struct WorldContext { pub docker: Docker, pub container_settings: ContainerSettings, - pub hyper: HyperClient, pub auth_uri: Uri, } @@ -506,11 +508,11 @@ pub mod tests { let control: i16 = Uniform::from(9000..10000).sample(&mut rand::thread_rng()); let user = control + 1; let bouncer = user + 1; - let auth = bouncer + 1; + let auth_port = bouncer + 1; let control = format!("127.0.0.1:{control}").parse().unwrap(); let user = format!("127.0.0.1:{user}").parse().unwrap(); let bouncer = format!("127.0.0.1:{bouncer}").parse().unwrap(); - let auth: SocketAddr = format!("127.0.0.1:{auth}").parse().unwrap(); + let auth: SocketAddr = format!("0.0.0.0:{auth_port}").parse().unwrap(); let auth_uri: Uri = format!("http://{auth}").parse().unwrap(); let auth_service = AuthService::new(auth); @@ -547,17 +549,31 @@ pub mod tests { prefix, provisioner_host, builder_host, - auth_uri: auth_uri.clone(), + // The started containers need to reach auth on the host. + // For this to work, the firewall should not be blocking traffic on the `SHUTTLE_TEST_NETWORK` interface. + // The following command can be used on NixOs to allow traffic on the interface. + // ``` + // sudo iptables -I nixos-fw -i -j nixos-fw-accept + // ``` + // + // Something like this should work on other systems. + // ``` + // sudo iptables -I INPUT -i -j ACCEPT + // ``` + auth_uri: format!("http://host.docker.internal:{auth_port}") + .parse() + .unwrap(), network_name, proxy_fqdn: FQDN::from_str("test.shuttleapp.rs").unwrap(), deploys_api_key: "gateway".to_string(), + + // Allow access to the auth on the host + extra_hosts: vec!["host.docker.internal:host-gateway".to_string()], }, }; let settings = ContainerSettings::builder().from_args(&args.context).await; - let hyper = HyperClient::builder().build(HttpConnector::new()); - let pool = SqlitePool::connect_with( SqliteConnectOptions::from_str("sqlite::memory:") .unwrap() @@ -576,7 +592,6 @@ pub mod tests { docker, settings, args, - hyper, pool, acme_client, auth_service, @@ -592,10 +607,6 @@ pub mod tests { self.pool.clone() } - pub fn client>(&self, addr: A) -> Client { - Client::new(addr).with_hyper_client(self.hyper.clone()) - } - pub fn fqdn(&self) -> FQDN { self.args().proxy_fqdn } @@ -609,24 +620,67 @@ pub mod tests { .lock() .unwrap() .users - .insert(user.to_string(), vec![Scope::Project, Scope::ProjectWrite]); + .insert(user.to_string(), AccountTier::Basic.into()); user.to_string() } + /// Create with the given name and return the authorization bearer for the user + pub fn create_authorization_bearer(&self, user: &str) -> Authorization { + let user_key = self.create_user(user); + Authorization::bearer(&user_key).unwrap() + } + pub fn set_super_user(&self, user: &str) { if let Some(scopes) = self.auth_service.lock().unwrap().users.get_mut(user) { scopes.push(Scope::Admin) } } - } - impl World { + /// Create a router to make API calls against. Also starts up a worker to create actual Docker containers for all requests + pub async fn router(&self) -> Router { + let service = Arc::new(GatewayService::init(self.args(), self.pool(), "".into()).await); + let worker = Worker::new(); + + let (sender, mut receiver) = channel(256); + tokio::spawn({ + let worker_sender = worker.sender(); + async move { + while let Some(work) = receiver.recv().await { + // Forward tasks to an actual worker + worker_sender + .send(work) + .await + .map_err(|_| "could not send work") + .unwrap(); + } + } + }); + + let _worker = tokio::spawn(async move { + worker.start().await.unwrap(); + }); + + // Allow the spawns to start + tokio::time::sleep(Duration::from_secs(1)).await; + + ApiBuilder::new() + .with_service(Arc::clone(&service)) + .with_sender(sender) + .with_default_routes() + .with_auth_service(self.context().auth_uri) + .into_router() + } + + pub fn client>(addr: A) -> Client { + let hyper = HyperClient::builder().build(HttpConnector::new()); + Client::new(addr).with_hyper_client(hyper) + } + pub fn context(&self) -> WorldContext { WorldContext { docker: self.docker.clone(), container_settings: self.settings.clone(), - hyper: self.hyper.clone(), auth_uri: self.auth_uri.clone(), } } @@ -696,6 +750,176 @@ pub mod tests { } } + /// Make it easy to perform common requests against the router for testing purposes + #[async_trait] + pub trait RouterExt { + /// Create a project and put it in the ready state + async fn create_project( + &mut self, + authorization: &Authorization, + project_name: &str, + ) -> TestProject; + } + + /// Helper struct to wrap a bunch of commands to run against a test project + pub struct TestProject { + router: Router, + authorization: Authorization, + project_name: String, + } + + impl TestProject { + /// Wait a few seconds for the project to enter the desired state + pub async fn wait_for_state(&mut self, state: project::State) { + let mut tries = 0; + let project_name = &self.project_name; + + loop { + let resp = self + .router + .call( + Request::get(format!("/projects/{project_name}")) + .with_header(&self.authorization) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(resp.status(), StatusCode::OK); + let body = hyper::body::to_bytes(resp.into_body()).await.unwrap(); + let project: project::Response = serde_json::from_slice(&body).unwrap(); + + if project.state == state { + break; + } + + tries += 1; + if tries > 12 { + panic!("timed out waiting for state {state}"); + } + + sleep(Duration::from_secs(1)).await; + } + } + + /// Is this project still available - aka has it been deleted + pub async fn is_missing(&mut self) -> bool { + let project_name = &self.project_name; + + let resp = self + .router + .call( + Request::get(format!("/projects/{project_name}")) + .with_header(&self.authorization) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + resp.status() == StatusCode::NOT_FOUND + } + + /// Destroy / stop a project. Like `cargo shuttle project stop` + pub async fn destroy_project(&mut self) { + let TestProject { + router, + authorization, + project_name, + } = self; + + router + .call( + Request::builder() + .method("DELETE") + .uri(format!("/projects/{project_name}")) + .body(Body::empty()) + .unwrap() + .with_header(authorization), + ) + .map_ok(|resp| { + assert_eq!(resp.status(), StatusCode::OK); + }) + .await + .unwrap(); + + self.wait_for_state(project::State::Destroyed).await; + } + + /// Send a request to the router for this project + pub async fn router_call(&mut self, method: Method, sub_path: &str) { + let project_name = &self.project_name; + + self.router + .call( + Request::builder() + .method(method) + .uri(format!("/projects/{project_name}{sub_path}")) + .body(Body::empty()) + .unwrap() + .with_header(&self.authorization), + ) + .map_ok(|resp| { + assert_eq!(resp.status(), StatusCode::OK); + }) + .await + .unwrap(); + } + } + + #[async_trait] + impl AsyncTestContext for TestProject { + async fn setup() -> Self { + let world = World::new().await; + + let mut router = world.router().await; + let authorization = world.create_authorization_bearer("neo"); + + router.create_project(&authorization, "matrix").await + } + + async fn teardown(mut self) { + assert!(self.is_missing().await, "test left a dangling project"); + } + } + + #[async_trait] + impl RouterExt for Router { + async fn create_project( + &mut self, + authorization: &Authorization, + project_name: &str, + ) -> TestProject { + let authorization = authorization.clone(); + + self.call( + Request::builder() + .method("POST") + .uri(format!("/projects/{project_name}")) + .header("Content-Type", "application/json") + .body("{\"idle_minutes\": 3}".into()) + .unwrap() + .with_header(&authorization), + ) + .map_ok(|resp| { + assert_eq!(resp.status(), StatusCode::OK); + }) + .await + .unwrap(); + + let mut this = TestProject { + authorization, + project_name: project_name.to_string(), + router: self.clone(), + }; + + this.wait_for_state(project::State::Ready).await; + + this + } + } + #[tokio::test] async fn end_to_end() { let world = World::new().await; @@ -716,7 +940,7 @@ pub mod tests { } }); - let api_client = world.client(world.args.control); + let api_client = World::client(world.args.control); let api = ApiBuilder::new() .with_service(Arc::clone(&service)) .with_sender(log_out.clone()) diff --git a/gateway/src/project.rs b/gateway/src/project.rs index 2f7f563e0..b13d16489 100644 --- a/gateway/src/project.rs +++ b/gateway/src/project.rs @@ -373,7 +373,10 @@ impl Project { | Self::Stopping(ProjectStopping { container, .. }) | Self::Stopped(ProjectStopped { container, .. }) | Self::Rebooting(ProjectRebooting { container, .. }) - | Self::Destroying(ProjectDestroying { container }) => Some(container.clone()), + | Self::Destroying(ProjectDestroying { container }) + | Self::Destroyed(ProjectDestroyed { + destroyed: Some(container), + }) => Some(container.clone()), Self::Errored(ProjectError { ctx: Some(ctx), .. }) => ctx.container(), Self::Errored(_) | Self::Creating(_) | Self::Destroyed(_) | Self::Deleted => None, } @@ -793,6 +796,7 @@ impl ProjectCreating { builder_host, auth_uri, fqdn: public, + extra_hosts, .. } = ctx.container_settings(); @@ -864,7 +868,8 @@ impl ProjectCreating { "MemoryReservation": 4295000000i64, // 4 GiB soft limit, applied if host is low on memory // https://docs.docker.com/config/containers/resource_constraints/#cpu "CpuPeriod": 100000i64, - "CpuQuota": 400000i64 + "CpuQuota": 400000i64, + "ExtraHosts": extra_hosts, }); debug!( @@ -1987,7 +1992,7 @@ pub mod tests { .unwrap() .unwrap(); - let client = world.client(target_addr); + let client = World::client(target_addr); client .request( diff --git a/gateway/src/service.rs b/gateway/src/service.rs index 2ae1e3991..bbb74135a 100644 --- a/gateway/src/service.rs +++ b/gateway/src/service.rs @@ -65,6 +65,7 @@ pub struct ContainerSettingsBuilder { auth_uri: Option, network_name: Option, fqdn: Option, + extra_hosts: Option>, } impl Default for ContainerSettingsBuilder { @@ -83,6 +84,7 @@ impl ContainerSettingsBuilder { auth_uri: None, network_name: None, fqdn: None, + extra_hosts: None, } } @@ -95,6 +97,7 @@ impl ContainerSettingsBuilder { auth_uri, image, proxy_fqdn, + extra_hosts, .. } = args; self.prefix(prefix) @@ -104,6 +107,7 @@ impl ContainerSettingsBuilder { .auth_uri(auth_uri) .network_name(network_name) .fqdn(proxy_fqdn) + .extra_hosts(extra_hosts) .build() .await } @@ -143,12 +147,18 @@ impl ContainerSettingsBuilder { self } + pub fn extra_hosts(mut self, extra_hosts: &[S]) -> Self { + self.extra_hosts = Some(extra_hosts.iter().map(ToString::to_string).collect()); + self + } + pub async fn build(mut self) -> ContainerSettings { let prefix = self.prefix.take().unwrap(); let image = self.image.take().unwrap(); let provisioner_host = self.provisioner.take().unwrap(); let builder_host = self.builder.take().unwrap(); let auth_uri = self.auth_uri.take().unwrap(); + let extra_hosts = self.extra_hosts.take().unwrap(); let network_name = self.network_name.take().unwrap(); let fqdn = self.fqdn.take().unwrap(); @@ -161,6 +171,7 @@ impl ContainerSettingsBuilder { auth_uri, network_name, fqdn, + extra_hosts, } } } @@ -174,6 +185,7 @@ pub struct ContainerSettings { pub auth_uri: String, pub network_name: String, pub fqdn: String, + pub extra_hosts: Vec, } impl ContainerSettings { diff --git a/gateway/src/task.rs b/gateway/src/task.rs index 5c7785bb6..aeae7f43e 100644 --- a/gateway/src/task.rs +++ b/gateway/src/task.rs @@ -10,6 +10,7 @@ use tokio::sync::mpsc::Sender; use tokio::sync::oneshot; use tokio::time::{sleep, timeout}; use tracing::{error, trace, trace_span, warn}; +use ulid::Ulid; use uuid::Uuid; use crate::project::*; @@ -128,6 +129,21 @@ pub fn start() -> impl Task { }) } +/// Will force restart a project no matter the state it is in +pub fn restart(project_id: Ulid) -> impl Task { + run(move |ctx| async move { + let state = ctx + .state + .container() + .and_then(|container| ProjectCreating::from_container(container, 0).ok()) + .unwrap_or_else(|| { + ProjectCreating::new_with_random_initial_key(ctx.project_name, project_id, 1) + }); + + TaskResult::Done(Project::Creating(state)) + }) +} + pub fn start_idle_deploys() -> impl Task { run(|ctx| async move { match ctx.state { From b7471ac2d135c9a681462775b481d55faceb18af Mon Sep 17 00:00:00 2001 From: Iulian Barbu <14218860+iulianbarbu@users.noreply.github.com> Date: Thu, 23 Nov 2023 12:39:05 +0200 Subject: [PATCH 21/23] auth: use a centrally stored jwt signing private key (#1402) * auth: add JWT signing private key arg * auth: added jwt signing private key flag to docker-compose * tests(auth): fix by starting auth server with jwt secret key * auth/circleci: add jwt signing key env variables for deploy * auth: added instructions about JWT signing private key generation * auth: consume AUTH_JWTSIGNING_PRIVATE_KEY from environment * fix(common): type conversion from str for a custom resource * auth: replaced deprecated base64::decode call * ci: comment main filter for staging release * Revert "auth: consume AUTH_JWTSIGNING_PRIVATE_KEY from environment" This reverts commit 3f766d349d7afe51d0f2153031965456a33cb42e. * Revert "ci: comment main filter for staging release" This reverts commit 220df0d363b697d6c53846e9b31824483ef2e6a7. * nit: simplify comment Co-authored-by: Pieter --------- Co-authored-by: Pieter --- .circleci/config.yml | 6 ++++++ Cargo.lock | 12 ++++++++++++ Makefile | 3 ++- auth/Cargo.toml | 2 ++ auth/README | 13 +++++++++++++ auth/src/api/builder.rs | 12 +++++++++++- auth/src/args.rs | 5 +++++ auth/src/lib.rs | 1 + auth/src/secrets.rs | 28 ++++++++++++++++++++-------- auth/tests/api/helpers.rs | 1 + docker-compose.dev.yml | 7 +++++++ docker-compose.yml | 1 + 12 files changed, 81 insertions(+), 10 deletions(-) create mode 100644 auth/README diff --git a/.circleci/config.yml b/.circleci/config.yml index c15035081..5987b6e7a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -420,6 +420,9 @@ jobs: stripe-secret-key: description: "Stripe secret key used to connect a client to Stripe backend" type: string + jwt-signing-private-key: + description: "Auth private key used for JWT signing" + type: string production: description: "Push and deploy to production" type: boolean @@ -450,6 +453,7 @@ jobs: DEPLOYS_API_KEY=${<< parameters.deploys-api-key >>} \ LOGGER_POSTGRES_URI=${<< parameters.logger-postgres-uri >>} \ STRIPE_SECRET_KEY=${<< parameters.stripe-secret-key >>} \ + AUTH_JWTSIGNING_PRIVATE_KEY=${<< parameters.jwt-signing-private-key >>} \ make deploy - when: condition: << parameters.production >> @@ -817,6 +821,7 @@ workflows: deploys-api-key: DEV_DEPLOYS_API_KEY logger-postgres-uri: DEV_LOGGER_POSTGRES_URI stripe-secret-key: DEV_STRIPE_SECRET_KEY + jwt-signing-private-key: DEV_AUTH_JWTSIGNING_PRIVATE_KEY requires: - build-and-push-unstable release: @@ -895,6 +900,7 @@ workflows: deploys-api-key: PROD_DEPLOYS_API_KEY logger-postgres-uri: PROD_LOGGER_POSTGRES_URI stripe-secret-key: PROD_STRIPE_SECRET_KEY + jwt-signing-private-key: PROD_AUTH_JWTSIGNING_PRIVATE_KEY ssh-fingerprint: 6a:c5:33:fe:5b:c9:06:df:99:64:ca:17:0d:32:18:2e ssh-config-script: production-ssh-config.sh ssh-host: shuttle.prod.internal diff --git a/Cargo.lock b/Cargo.lock index 578a5d5c2..51a5157ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4505,6 +4505,16 @@ dependencies = [ "base64 0.13.1", ] +[[package]] +name = "pem" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b13fe415cdf3c8e44518e18a7c95a13431d9bdf6d15367d82b23c377fdd441a" +dependencies = [ + "base64 0.21.5", + "serde", +] + [[package]] name = "pem" version = "3.0.2" @@ -5748,11 +5758,13 @@ dependencies = [ "axum", "axum-extra", "axum-sessions", + "base64 0.21.5", "clap", "http", "hyper", "jsonwebtoken", "opentelemetry", + "pem 2.0.1", "portpicker", "rand 0.8.5", "ring 0.17.5", diff --git a/Makefile b/Makefile index e09a765a2..9b1449472 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ POSTGRES_PASSWORD?=postgres MONGO_INITDB_ROOT_USERNAME?=mongodb MONGO_INITDB_ROOT_PASSWORD?=password STRIPE_SECRET_KEY?="" - +AUTH_JWTSIGNING_PRIVATE_KEY?="" ifeq ($(PROD),true) DOCKER_COMPOSE_FILES=docker-compose.yml @@ -137,6 +137,7 @@ DOCKER_COMPOSE_ENV=\ MONGO_INITDB_ROOT_USERNAME=$(MONGO_INITDB_ROOT_USERNAME)\ MONGO_INITDB_ROOT_PASSWORD=$(MONGO_INITDB_ROOT_PASSWORD)\ STRIPE_SECRET_KEY=$(STRIPE_SECRET_KEY)\ + AUTH_JWTSIGNING_PRIVATE_KEY=$(AUTH_JWTSIGNING_PRIVATE_KEY)\ DD_ENV=$(DD_ENV)\ USE_TLS=$(USE_TLS)\ COMPOSE_PROFILES=$(COMPOSE_PROFILES)\ diff --git a/auth/Cargo.toml b/auth/Cargo.toml index bce1b4990..85321503d 100644 --- a/auth/Cargo.toml +++ b/auth/Cargo.toml @@ -17,10 +17,12 @@ async-stripe = { version = "0.25.1", default-features = false, features = ["chec async-trait = { workspace = true } axum = { workspace = true, features = ["headers"] } axum-sessions = { workspace = true } +base64 = { workspace = true } clap = { workspace = true } http = { workspace = true } jsonwebtoken = { workspace = true } opentelemetry = { workspace = true } +pem = "2" rand = { workspace = true } ring = { workspace = true } serde = { workspace = true, features = ["derive"] } diff --git a/auth/README b/auth/README new file mode 100644 index 000000000..3aeaadae7 --- /dev/null +++ b/auth/README @@ -0,0 +1,13 @@ +# Auth service considerations + +## JWT signing private key + +Starting the service locally requires provisioning of a base64 encoded PEM encoded PKCS#8 v1 unencrypted private key. +The service was tested with keys generated as follows: + +```bash +openssl genpkey -algorithm ED25519 -out auth_jwtsigning_private_key.pem +base64 < auth_jwtsigning_private_key.pem +``` + +Used `OpenSSL 3.1.2 1 Aug 2023 (Library: OpenSSL 3.1.2 1 Aug 2023)` and `FreeBSD base64`, on a `macOS Sonoma 14.1.1`. \ No newline at end of file diff --git a/auth/src/api/builder.rs b/auth/src/api/builder.rs index 7baf14775..30c588da2 100644 --- a/auth/src/api/builder.rs +++ b/auth/src/api/builder.rs @@ -54,6 +54,7 @@ pub struct ApiBuilder { pool: Option, session_layer: Option>, stripe_client: Option, + jwt_signing_private_key: Option, } impl Default for ApiBuilder { @@ -95,6 +96,7 @@ impl ApiBuilder { pool: None, session_layer: None, stripe_client: None, + jwt_signing_private_key: None, } } @@ -122,15 +124,23 @@ impl ApiBuilder { self } + pub fn with_jwt_signing_private_key(mut self, private_key: String) -> Self { + self.jwt_signing_private_key = Some(private_key); + self + } + pub fn into_router(self) -> Router { let pool = self.pool.expect("an sqlite pool is required"); let session_layer = self.session_layer.expect("a session layer is required"); let stripe_client = self.stripe_client.expect("a stripe client is required"); + let jwt_signing_private_key = self + .jwt_signing_private_key + .expect("a jwt signing private key"); let user_manager = UserManager { pool, stripe_client, }; - let key_manager = EdDsaManager::new(); + let key_manager = EdDsaManager::new(jwt_signing_private_key); let state = RouterState { user_manager: Arc::new(Box::new(user_manager)), diff --git a/auth/src/args.rs b/auth/src/args.rs index 77c12909b..e78a10833 100644 --- a/auth/src/args.rs +++ b/auth/src/args.rs @@ -28,6 +28,11 @@ pub struct StartArgs { /// Stripe client secret key #[arg(long, default_value = "")] pub stripe_secret_key: String, + + /// Auth JWT signing private key, as a base64 encoding of + /// a PEM encoded PKCS#8 v1 formatted unencrypted private key. + #[arg(long, default_value = "")] + pub jwt_signing_private_key: String, } #[derive(clap::Args, Debug, Clone)] diff --git a/auth/src/lib.rs b/auth/src/lib.rs index 5989c761c..72a058bfe 100644 --- a/auth/src/lib.rs +++ b/auth/src/lib.rs @@ -29,6 +29,7 @@ pub async fn start(pool: SqlitePool, args: StartArgs) -> io::Result<()> { .with_sqlite_pool(pool) .with_sessions() .with_stripe_client(stripe::Client::new(args.stripe_secret_key)) + .with_jwt_signing_private_key(args.jwt_signing_private_key) .into_router(); info!(address=%args.address, "Binding to and listening at address"); diff --git a/auth/src/secrets.rs b/auth/src/secrets.rs index 6ddfc3917..d131f2642 100644 --- a/auth/src/secrets.rs +++ b/auth/src/secrets.rs @@ -1,3 +1,4 @@ +use base64::{engine::general_purpose, Engine}; use jsonwebtoken::EncodingKey; use ring::signature::{Ed25519KeyPair, KeyPair}; @@ -15,16 +16,27 @@ pub struct EdDsaManager { } impl EdDsaManager { - pub fn new() -> Self { - let doc = Ed25519KeyPair::generate_pkcs8(&ring::rand::SystemRandom::new()) - .expect("to create a PKCS8 for edDSA"); - let encoding_key = EncodingKey::from_ed_der(doc.as_ref()); - let pair = Ed25519KeyPair::from_pkcs8(doc.as_ref()).expect("to create a key pair"); - let public_key = pair.public_key(); + /// Create a new manager from a base64 PEM encoded private key. This key can be generated using: + /// ```bash + /// openssl genpkey -algorithm ED25519 -out auth_jwtsigning_private_key.pem + /// base64 < auth_jwtsigning_private_key.pem + /// ``` + pub fn new(jwt_signing_private_key: String) -> Self { + // Decode the base64 encoding. + let pk_bytes = general_purpose::STANDARD + .decode(jwt_signing_private_key) + .expect("to decode base64 pem encoded private key"); + + // Parse the pem file and the ed25519 private key contained. + let pem_keypair = pem::parse(pk_bytes.clone()).expect("to parse pem encoded private key"); + let ed_keypair = Ed25519KeyPair::from_pkcs8_maybe_unchecked(pem_keypair.contents()) + .expect("to get PKCS#8 v1 formatted private key from pem encoded key"); Self { - encoding_key, - public_key: public_key.as_ref().to_vec(), + // Wrap the private key as a jwt encoding key. + encoding_key: EncodingKey::from_ed_pem(pk_bytes.as_slice()) + .expect("to get an encoding key from pem encoded ed25519 private key"), + public_key: ed_keypair.public_key().as_ref().to_vec(), } } } diff --git a/auth/tests/api/helpers.rs b/auth/tests/api/helpers.rs index 2490017f5..09d4a4474 100644 --- a/auth/tests/api/helpers.rs +++ b/auth/tests/api/helpers.rs @@ -41,6 +41,7 @@ pub(crate) async fn app() -> TestApp { mocked_stripe_server.uri.to_string().as_str(), "", )) + .with_jwt_signing_private_key("LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSUR5V0ZFYzhKYm05NnA0ZGNLTEwvQWNvVUVsbUF0MVVKSTU4WTc4d1FpWk4KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=".to_string()) .into_router(); TestApp { diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 573b3aba4..b37d27d31 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -61,6 +61,13 @@ services: >&2 echo "The control DB is available - starting shuttle-auth" exec /usr/local/bin/shuttle-auth "$${@:0}" + command: + - "--state=/var/lib/shuttle-auth" + - "start" + - "--address=0.0.0.0:8000" + - "--stripe-secret-key=${STRIPE_SECRET_KEY}" + # used only for local development + - "--jwt-signing-private-key=LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1DNENBUUF3QlFZREsyVndCQ0lFSUR5V0ZFYzhKYm05NnA0ZGNLTEwvQWNvVUVsbUF0MVVKSTU4WTc4d1FpWk4KLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLQo=" otel-collector: ports: - 4317:4317 diff --git a/docker-compose.yml b/docker-compose.yml index 6c7bf02e0..ae693a104 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -53,6 +53,7 @@ services: - "start" - "--address=0.0.0.0:8000" - "--stripe-secret-key=${STRIPE_SECRET_KEY}" + - "--jwt-signing-private-key=${AUTH_JWTSIGNING_PRIVATE_KEY}" healthcheck: test: curl --fail http://localhost:8000/ || exit 1 interval: 1m From c677290887d29a3c3968ceff4fff942fa2c8168e Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Thu, 23 Nov 2023 12:17:32 +0100 Subject: [PATCH 22/23] feat(cargo-shuttle): better handling of runtime version checks (#1418) --- cargo-shuttle/src/lib.rs | 44 ++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 17 deletions(-) diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 724a0fc9b..9552a2237 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -7,6 +7,7 @@ mod suggestions; use std::collections::{BTreeMap, HashMap}; use std::ffi::OsString; +use std::fmt::Write as FmtWrite; use std::fs::{read_to_string, File}; use std::io::stdout; use std::net::{Ipv4Addr, SocketAddr}; @@ -57,12 +58,12 @@ use ignore::overrides::OverrideBuilder; use ignore::WalkBuilder; use indicatif::ProgressBar; use indoc::{formatdoc, printdoc}; -use std::fmt::Write as FmtWrite; use strum::IntoEnumIterator; use tar::Builder; use tokio::io::{AsyncBufReadExt, BufReader}; use tokio::process::Child; use tokio::task::JoinHandle; +use tokio::time::{sleep, Duration}; use tonic::transport::Channel; use tonic::Status; use tracing::{debug, error, trace, warn}; @@ -907,7 +908,7 @@ impl Shuttle { let path = dunce::canonicalize(format!("{MANIFEST_DIR}/../runtime")) .expect("path to shuttle-runtime does not exist or is invalid"); - std::process::Command::new("cargo") + tokio::process::Command::new("cargo") .arg("install") .arg("shuttle-runtime") .arg("--path") @@ -917,15 +918,16 @@ impl Shuttle { .arg("--features") .arg("next") .output() + .await .expect("failed to install the shuttle runtime"); } else { // If the version of cargo-shuttle is different from shuttle-runtime, // or it isn't installed, try to install shuttle-runtime from crates.io. - if let Err(err) = check_version(&runtime_path) { + if let Err(err) = check_version(&runtime_path).await { warn!("{}", err); trace!("installing shuttle-runtime"); - std::process::Command::new("cargo") + tokio::process::Command::new("cargo") .arg("install") .arg("shuttle-runtime") .arg("--bin") @@ -933,6 +935,7 @@ impl Shuttle { .arg("--features") .arg("next") .output() + .await .expect("failed to install the shuttle runtime"); }; }; @@ -940,7 +943,7 @@ impl Shuttle { runtime_path } else { trace!(path = ?service.executable_path, "using alpha runtime"); - if let Err(err) = check_version(&service.executable_path) { + if let Err(err) = check_version(&service.executable_path).await { warn!("{}", err); if let Some(mismatch) = err.downcast_ref::() { println!("Warning: {}.", mismatch); @@ -961,7 +964,7 @@ impl Shuttle { } else { return Err(err.context( format!( - "Failed to verify the version of shuttle-runtime in {}. Is cargo targeting the correct binary?", + "Failed to verify the version of shuttle-runtime in {}. Is cargo targeting the correct executable?", service.executable_path.display() ) )); @@ -1591,7 +1594,7 @@ impl Shuttle { eprintln!("--- Reconnecting websockets logging ---"); // A wait time short enough for not much state to have changed, long enough that // the terminal isn't completely spammed - tokio::time::sleep(std::time::Duration::from_millis(100)).await; + sleep(Duration::from_millis(100)).await; stream = client .get_logs_ws(self.ctx.project_name(), &deployment.id) .await @@ -1607,7 +1610,7 @@ impl Shuttle { // Temporary fix. // TODO: Make get_service_summary endpoint wait for a bit and see if it entered Running/Crashed state. // Note: Will otherwise be possible when health checks are supported - tokio::time::sleep(std::time::Duration::from_millis(500)).await; + sleep(Duration::from_millis(500)).await; let deployment = client .get_deployment_details(self.ctx.project_name(), &deployment.id) @@ -2047,7 +2050,7 @@ fn is_dirty(repo: &Repository) -> Result<()> { Ok(()) } -fn check_version(runtime_path: &Path) -> Result<()> { +async fn check_version(runtime_path: &Path) -> Result<()> { debug!( "Checking version of runtime binary at {}", runtime_path.display() @@ -2057,20 +2060,27 @@ fn check_version(runtime_path: &Path) -> Result<()> { let my_version = semver::Version::from_str(VERSION).unwrap(); if !runtime_path.try_exists()? { - bail!("shuttle-runtime is not installed"); + bail!("shuttle-runtime binary not found"); } // Get runtime version from shuttle-runtime cli - let runtime_version = std::process::Command::new(runtime_path) - .arg("--version") - .output() - .context("failed to run the shuttle-runtime binary to check its version")? - .stdout; + // It should print the version and exit immediately, so a timeout is used to not launch servers with non-Shuttle setups + let stdout = tokio::time::timeout(Duration::from_millis(500), async move { + tokio::process::Command::new(runtime_path) + .arg("--version") + .kill_on_drop(true) // if the binary does not halt on its own, not killing it will leak child processes + .output() + .await + .context("Failed to run the shuttle-runtime binary to check its version") + .map(|o| o.stdout) + }) + .await + .context("Checking the version of shuttle-runtime timed out. Make sure the executable is using #[shuttle-runtime::main].")??; // Parse the version, splitting the version from the name and // and pass it to `to_semver()`. let runtime_version = semver::Version::from_str( - std::str::from_utf8(&runtime_version) + std::str::from_utf8(&stdout) .context("shuttle-runtime version should be valid utf8")? .split_once(' ') .context("shuttle-runtime version should be in the `name version` format")? @@ -2127,7 +2137,7 @@ where break cleanup; } count += 1; - tokio::time::sleep(tokio::time::Duration::from_millis(300)).await; + sleep(Duration::from_millis(300)).await; }; progress_bar.finish_and_clear(); From 6f167685e5760584b97f6034640d833874cdb192 Mon Sep 17 00:00:00 2001 From: jonaro00 <54029719+jonaro00@users.noreply.github.com> Date: Thu, 23 Nov 2023 12:19:00 +0100 Subject: [PATCH 23/23] chore: examples v0.34.0 --- examples | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples b/examples index 7b9dde8e5..541d2006f 160000 --- a/examples +++ b/examples @@ -1 +1 @@ -Subproject commit 7b9dde8e5c7a0f235277e58a2ff9ea96e4ccfd4d +Subproject commit 541d2006fc425df26bbfe41003e7f279848e35db