diff --git a/.dockerignore b/.dockerignore index cf9e8b506..56f73f37f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,6 @@ **/target/ -Dockerfile +Containerfile +Containerfile.dev terraform/ examples/ -.cargo/ \ No newline at end of file +.cargo/ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1004e5b00..df7cedcae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,7 +8,7 @@ jobs: Build-And-Deploy: runs-on: self-hosted env: - CONTAINER_REGISTRY: public.ecr.aws/shuttle/backend + CONTAINER_REGISTRY: public.ecr.aws/shuttle steps: - uses: shuttle-hq/checkout@v2 - name: Configure AWS Credentials @@ -20,12 +20,20 @@ jobs: - name: Docker Login run: | aws ecr-public get-login-password | docker login --username AWS --password-stdin $CONTAINER_REGISTRY - - name: Build And Push Image + - name: Build And Push Backend Image uses: shuttle-hq/build-push-action@v2 with: context: . + file: api/Containerfile push: true - tags: ${{ env.CONTAINER_REGISTRY }}:${{ github.sha }},${{ env.CONTAINER_REGISTRY }}:latest + tags: ${{ env.CONTAINER_REGISTRY }}/backend:${{ github.sha }},${{ env.CONTAINER_REGISTRY }}/backend:latest + - name: Build And Push Provisioner Image + uses: shuttle-hq/build-push-action@v2 + with: + context: . + file: provisioner/Containerfile + push: true + tags: ${{ env.CONTAINER_REGISTRY }}/provisioner:${{ github.sha }},${{ env.CONTAINER_REGISTRY }}/provisioner:latest - name: Deploy Image run: | ssh ubuntu@prod sudo systemctl restart shuttle-backend.service diff --git a/.github/workflows/fmt.yml b/.github/workflows/fmt.yml index 62e69c646..b7f94aa03 100644 --- a/.github/workflows/fmt.yml +++ b/.github/workflows/fmt.yml @@ -18,6 +18,17 @@ jobs: override: true components: rustfmt - run: cargo fmt --all -- --check + cargo_sort_test: + runs-on: ubuntu-latest + steps: + - uses: shuttle-hq/checkout@v2 + - uses: shuttle-hq/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt + - run: cargo install cargo-sort + - run: cargo sort --check --workspace clippy_test: strategy: fail-fast: true @@ -32,4 +43,4 @@ jobs: override: true components: clippy - name: Run test with ${{ matrix.features}} - run: cargo clippy --tests --all-targets --features="codegen,loader,sqlx-integration,sqlx-postgres,secrets,${{ matrix.features }}" --no-deps -- --D warnings \ No newline at end of file + run: cargo clippy --tests --all-targets --features="codegen,loader,sqlx-integration,sqlx-postgres,secrets,${{ matrix.features }}" --no-deps -- --D warnings diff --git a/.gitignore b/.gitignore index eadf7a754..f219b0348 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,9 @@ Cargo.lock # VS Code .vscode/ +# OS Specific +.DS_Store + # Terraform **/secret* **/.terraform* diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 000000000..6b04091d2 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,81 @@ +# Code of conduct + +* We are committed to providing a friendly, safe and welcoming environment for +all, regardless of level of experience, gender identity and expression, sexual +orientation, disability, personal appearance, body size, race, ethnicity, age, +religion, nationality, or other similar characteristics. +* Please avoid using overtly sexual aliases or other nicknames that might +detract from a friendly, safe and welcoming environment for all. +* Please be kind and courteous. There’s no need to be mean or rude. +* Respect that people have differences of opinion and that every design or +implementation choice carries a trade-off and numerous costs. There is seldom +a right answer. +* Please keep unstructured critique to a minimum. If you have solid ideas you +want to experiment with, make a fork and see how it works. +* We will exclude you from interaction if you insult, demean or harass anyone. +That is not welcome behavior. We interpret the term “harassment” as including +the definition in the +[Citizen Code of Conduct](https://github.com/stumpsyn/policies/blob/master/citizen_code_of_conduct.md); +if you have any lack of clarity about what might be included in that concept, +please read their definition. In particular, we don’t tolerate behavior that +excludes people in socially marginalised groups. +* Private harassment is also unacceptable. No matter who you are, if you feel +you have been or are being harassed or made uncomfortable by a community member, +please contact one of the channel ops or any of the +[shuttle moderators](mailto:opensource@shuttle.rs) immediately. Whether you’re a +regular contributor or a newcomer, we care about making this community a safe +place for you and we’ve got your back. +* Likewise any spamming, trolling, flaming, baiting or other +attention-stealing behavior is not welcome. + +## Moderation + +These are the policies for upholding our community’s standards of conduct. If +you feel that a thread needs moderation, please contact the +[shuttle moderators](mailto:opensource@shuttle.rs). + +1. Remarks that violate the shuttle standards of conduct, including hateful, +hurtful, oppressive, or exclusionary remarks, are not allowed. (Cursing is +allowed, but never targeting another user, and never in a hateful manner.) +2. Remarks that moderators find inappropriate, whether listed in the code of +conduct or not, are also not allowed. +3. Moderators will first respond to such remarks with a warning. +4. If the warning is unheeded, the user will be “kicked,” i.e., kicked out of +the communication channel to cool off. +5. If the user comes back and continues to make trouble, they will be banned, +i.e., indefinitely excluded. +6. Moderators may choose at their discretion to un-ban the user if it was a +first offense and they offer the offended party a genuine apology. +7. If a moderator bans someone and you think it was unjustified, please take +it up with that moderator, or with a different moderator, *in private*. +Complaints about bans in-channel are not allowed. +8. Moderators are held to a higher standard than other community members. If a +moderator creates an inappropriate situation, they should expect less leeway +than others. + +In the shuttle community we strive to go the extra step to look out for each +other. Don’t just aim to be technically unimpeachable, try to be your best +self. In particular, avoid flirting with offensive or sensitive issues, +particularly if they’re off-topic; this all too often leads to unnecessary +fights, hurt feelings, and damaged trust; worse, it can drive people away from +the community entirely. + +And if someone takes issue with something you said or did, resist the urge to +be defensive. Just stop doing what it was they complained about and apologise. +Even if you feel you were misinterpreted or unfairly accused, chances are good +there was something you could’ve communicated better — remember that it’s your +responsibility to make your fellow contributor comfortable. Everyone wants to +get along and we are all here first and foremost because we want to talk about +cool technology. You will find that people will be eager to assume good intent +and forgive as long as you earn their trust. + +The enforcement policies listed above apply to all official shuttle venues; +including [Discord channels](https://discord.gg/H33rRDTm3p) and +[GitHub repositories under shuttle](https://github.com/shuttle-hq). For other +projects adopting the shuttle Code of Conduct, please contact the maintainers of +those projects for enforcement. If you wish to use this code of conduct for +your own project, consider explicitly mentioning your moderation policy or +making a copy with your own moderation policy so as to avoid confusion. + + *Adapted from the +[Rust Code of Conduct](https://www.rust-lang.org/policies/code-of-conduct)* diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..c51c1e886 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,112 @@ +# Contributing + +## Raise an Issue + +Raising [issues](https://github.com/shuttle-hq/shuttle/issues) is encouraged. We have some templates to help you get started. + +## Running Locally +You can use Docker and docker-compose to test shuttle locally during development. See the [Docker install](https://docs.docker.com/get-docker/) +and [docker-compose install](https://docs.docker.com/compose/install/) instructions if you do not have them installed already. + +You should now be set to run shuttle locally as follow: + +```bash +# clone the repo +git clone git@github.com:shuttle-hq/shuttle.git + +# cd into the repo +cd shuttle + +# start the shuttle services +docker-compose up --build + +# login to shuttle service in a new terminal window +cd path/to/shuttle/repo +cargo run --bin cargo-shuttle -- login --api-key "ci-test" + +# cd into one of the examples +cd examples/rocket/hello-world/ + +# deploy the example +# the --manifest-path is used to locate the root of the shuttle workspace +cargo run --manifest-path ../../../Cargo.toml --bin cargo-shuttle -- deploy + +# test if the deploy is working +# (the Host header should match the Host from the deploy output) +curl --header "Host: hello-world-rocket-app.teste.rs" localhost:8000/hello +``` +### Using Podman instead of Docker +If you are using Podman over Docker, then expose a rootless socket of Podman using the following command: + +```bash +podman system service --time=0 unix:///tmp/podman.sock +``` + +Now make docker-compose use this socket by setting the following environment variable: + +```bash +export DOCKER_HOST=unix:///tmp/podman.sock +``` + +shuttle can now be run locally using the steps shown earlier. + +## Running Tests + +shuttle has reasonable test coverage - and we are working on improving this +every day. We encourage PRs to come with tests. If you're not sure about +what a test should look like, feel free to [get in touch](https://discord.gg/H33rRDTm3p). + +To run the test suite - just run `cargo test -- --nocapture` at the root of the repository. + +## Committing + +We use the [Angular Commit Guidelines](https://github.com/angular/angular/blob/master/CONTRIBUTING.md#commit). We expect all commits to conform to these guidelines. + +Furthermore, commits should be squashed before being merged to master. + +Also, make sure your commits don't trigger any warnings from Clippy by running: `cargo clippy --tests --all-targets`. If you have a good reason to contradict Clippy, insert an #allow[] macro, so that it won't complain. + +## Project Layout +The folders in this repository relate to each other as follow: + +```mermaid +graph BT + classDef default fill:#1f1f1f,stroke-width:0; + classDef binary fill:#f25100,font-weight:bold,stroke-width:0; + classDef external fill:#343434,font-style:italic,stroke:#f25100; + + api:::binary + cargo-shuttle:::binary + common + codegen + e2e + proto + provisioner:::binary + service + user([user service]):::external + api --> proto + api -.->|calls| provisioner + service ---> common + api --> common + cargo-shuttle --->|"features = ['loader']"| service + api -->|"features = ['loader', 'secrets']"| service + cargo-shuttle --> common + service --> codegen + proto ---> common + provisioner --> proto + e2e -.->|starts up| api + e2e -.->|calls| cargo-shuttle + user -->|"features = ['codegen']"| service +``` + +First, `provisioner`, `api`, and `cargo-shuttle` are binary crates with `provisioner` and `api` being backend services. The `cargo-shuttle` binary is the `cargo shuttle` command used by users. + +The rest are the following libraries: +- `common` contains shared models and functions used by the other libraries and binaries. +- `codegen` contains our proc-macro code which gets exposed to user services from `service` by the `codegen` feature flag. The redirect through `service` is to make it available under the prettier name of `shuttle_service::main`. +- `service` is where our special `Service` trait is defined. Anything implementing this `Service` can be loaded by the `api` and the local runner in `cargo-shuttle`. + The `codegen` automatically implements the `Service` trait for any user service. +- `proto` contains the gRPC server and client definitions to allow `api` to communicate with `provisioner`. +- `e2e` just contains tests which starts up the `api` in a container and then deploys services to it using `cargo-shuttle`. + +Lastly, the `user service` is not a folder in this repository, but is the user service that will be deployed by `api`. diff --git a/Cargo.lock b/Cargo.lock index 9860d0559..da9b1aba3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "aead" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877" +dependencies = [ + "generic-array", +] + [[package]] name = "aes" version = "0.6.0" @@ -25,7 +34,19 @@ checksum = "884391ef1066acaa41e766ba8f596341b96e93ce34f9a43e7d24bf0a0eaf0561" dependencies = [ "aes-soft", "aesni", - "cipher", + "cipher 0.2.5", +] + +[[package]] +name = "aes" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8" +dependencies = [ + "cfg-if 1.0.0", + "cipher 0.3.0", + "cpufeatures", + "opaque-debug", ] [[package]] @@ -34,11 +55,25 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5278b5fabbb9bd46e24aa69b2fdea62c99088e0a950a9be40e3e0101298f88da" dependencies = [ - "aead", - "aes", - "cipher", - "ctr", - "ghash", + "aead 0.3.2", + "aes 0.6.0", + "cipher 0.2.5", + "ctr 0.6.0", + "ghash 0.3.1", + "subtle", +] + +[[package]] +name = "aes-gcm" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" +dependencies = [ + "aead 0.4.3", + "aes 0.7.5", + "cipher 0.3.0", + "ctr 0.8.0", + "ghash 0.4.4", "subtle", ] @@ -48,7 +83,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be14c7498ea50828a38d0e24a765ed2effe92a705885b57d029cd67d45744072" dependencies = [ - "cipher", + "cipher 0.2.5", "opaque-debug", ] @@ -58,7 +93,7 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2e11f5e94c2f7d386164cc2aa1f97823fed6f259e486940a71c174dd01b0ce" dependencies = [ - "cipher", + "cipher 0.2.5", "opaque-debug", ] @@ -93,13 +128,13 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4361135be9122e0870de935d7c439aef945b9f9ddd4199a553b5270b49c82a27" +checksum = "08f9b8508dccb7687a1d6c4ce66b2b0ecef467c94667de27d8d7fe1f8d2a9cdc" [[package]] name = "api" -version = "0.1.0" +version = "0.1.2" dependencies = [ "anyhow", "async-mutex", @@ -116,6 +151,7 @@ dependencies = [ "lazy_static", "libloading", "log", + "proto", "rand 0.8.5", "rocket", "serde", @@ -125,6 +161,7 @@ dependencies = [ "structopt", "tokio", "toml", + "tonic", "uuid", ] @@ -140,20 +177,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" -[[package]] -name = "assert_cmd" -version = "2.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93ae1ddd39efd67689deb1979d80bad3bf7f2b09c6e6117c8d1f2443b5e2f83e" -dependencies = [ - "bstr", - "doc-comment", - "predicates", - "predicates-core", - "predicates-tree", - "wait-timeout", -] - [[package]] name = "async-channel" version = "1.6.1" @@ -293,7 +316,7 @@ dependencies = [ "rand 0.7.3", "serde", "serde_json", - "sha2", + "sha2 0.9.9", ] [[package]] @@ -367,9 +390,9 @@ checksum = "30696a84d817107fc028e049980e09d5e140e8da8f1caeb17e8e950658a3cea9" [[package]] name = "async-trait" -version = "0.1.52" +version = "0.1.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061a7acccaa286c011ddc30970520b98fa40e00c9d644633fb26b5fc63a265e3" +checksum = "96cf8829f67d2eab0b2dfa42c5d0ef737e0724e4a82b01b3e292456202b19716" dependencies = [ "proc-macro2", "quote", @@ -419,9 +442,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "axum" -version = "0.5.1" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47594e438a243791dba58124b6669561f5baa14cb12046641d8008bf035e5a25" +checksum = "dc47084705629d09d15060d70a8dbfce479c842303d05929ce29c74c995916ae" dependencies = [ "async-trait", "axum-core", @@ -443,16 +466,16 @@ dependencies = [ "sync_wrapper", "tokio", "tower", - "tower-http", + "tower-http 0.3.4", "tower-layer", "tower-service", ] [[package]] name = "axum-core" -version = "0.2.1" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a671c9ae99531afdd5d3ee8340b8da547779430689947144c140fc74a740244" +checksum = "c2efed1c501becea07ce48118786ebcf229531d0d3b28edf224a720020d9e106" dependencies = [ "async-trait", "bytes", @@ -522,7 +545,7 @@ dependencies = [ "cfg-if 0.1.10", "constant_time_eq", "crypto-mac 0.8.0", - "digest", + "digest 0.9.0", ] [[package]] @@ -534,6 +557,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf7fe51849ea569fd452f37822f606a5cabb684dc918707a0193fd4664ff324" +dependencies = [ + "generic-array", +] + [[package]] name = "blocking" version = "1.2.0" @@ -548,6 +580,46 @@ dependencies = [ "once_cell", ] +[[package]] +name = "bollard" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b1d4b9e55620571c2200f4be87db2a9a69e2a107fc7d206a6accad58c3536cb" +dependencies = [ + "base64 0.13.0", + "bollard-stubs", + "bytes", + "chrono", + "futures-core", + "futures-util", + "hex 0.4.3", + "http", + "hyper", + "hyperlocal", + "log", + "pin-project-lite 0.2.8", + "serde", + "serde_derive", + "serde_json", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util 0.7.1", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.42.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4295240332c78d04291f3ac857a281d5534a8e036f3dfcdaa294b22c0d424427" +dependencies = [ + "chrono", + "serde", + "serde_with", +] + [[package]] name = "bstr" version = "0.2.17" @@ -600,16 +672,16 @@ dependencies = [ [[package]] name = "cargo" -version = "0.59.0" +version = "0.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d08705bf607eb738364863d8435e69f88dd5c131b7f9752777a7ce328085cf95" +checksum = "79bc435c2de37f164b5c36420d9e1dd65cd5acbd41b2df10fdc02dbf75ed9efc" dependencies = [ "anyhow", "atty", "bytesize", "cargo-platform", "cargo-util", - "clap", + "clap 3.1.18", "crates-io", "crossbeam-utils", "curl", @@ -634,13 +706,12 @@ dependencies = [ "libgit2-sys", "log", "memchr", - "num_cpus", "opener", "os_info", "percent-encoding", "rustc-workspace-hack", "rustfix", - "semver 1.0.6", + "semver 1.0.9", "serde", "serde_ignored", "serde_json", @@ -649,7 +720,7 @@ dependencies = [ "tar", "tempfile", "termcolor", - "toml", + "toml_edit 0.13.4", "unicode-width", "unicode-xid", "url", @@ -657,6 +728,37 @@ dependencies = [ "winapi", ] +[[package]] +name = "cargo-edit" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34965aea9b211a43ace81105894e2e2d15103be244884b31f26065e7538fe30b" +dependencies = [ + "anyhow", + "cargo_metadata", + "clap 3.1.18", + "concolor-control", + "crates-index", + "dirs-next", + "dunce", + "env_proxy", + "git2", + "hex 0.4.3", + "indexmap", + "native-tls", + "pathdiff", + "regex", + "semver 1.0.9", + "serde", + "serde_derive", + "serde_json", + "subprocess", + "termcolor", + "toml_edit 0.13.4", + "ureq", + "url", +] + [[package]] name = "cargo-platform" version = "0.1.2" @@ -668,27 +770,37 @@ dependencies = [ [[package]] name = "cargo-shuttle" -version = "0.3.0" +version = "0.3.2" dependencies = [ "anyhow", - "assert_cmd", + "async-trait", + "bollard", "cargo", + "cargo-edit", "cargo_metadata", "chrono", "colored", + "crossterm", "dirs", + "env_logger", "futures", "log", - "predicates", + "portpicker", "reqwest", "reqwest-middleware", "reqwest-retry", + "semver 1.0.9", "serde", "serde_json", "shuttle-common", + "shuttle-service", "structopt", + "test-context", "tokio", + "tokiotest-httpserver", "toml", + "toml_edit 0.14.4", + "uuid", "webbrowser", ] @@ -722,7 +834,7 @@ checksum = "4acbb09d9ee8e23699b9634375c72795d095bf268439da88562cf9b501f181fa" dependencies = [ "camino", "cargo-platform", - "semver 1.0.6", + "semver 1.0.9", "serde", "serde_json", ] @@ -768,6 +880,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "cipher" version = "0.2.5" @@ -777,6 +895,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "cipher" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7" +dependencies = [ + "generic-array", +] + [[package]] name = "clap" version = "2.34.0" @@ -787,11 +914,60 @@ dependencies = [ "atty", "bitflags", "strsim 0.8.0", - "textwrap", + "textwrap 0.11.0", "unicode-width", "vec_map", ] +[[package]] +name = "clap" +version = "3.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b" +dependencies = [ + "atty", + "bitflags", + "clap_derive", + "clap_lex", + "indexmap", + "lazy_static", + "strsim 0.10.0", + "termcolor", + "terminal_size", + "textwrap 0.15.0", +] + +[[package]] +name = "clap_derive" +version = "3.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25320346e922cffe59c0bbc5410c8d8784509efb321488971081313cb1e1a33c" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213" +dependencies = [ + "os_str_bytes", +] + +[[package]] +name = "cmake" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8ad8cef104ac57b68b89df3208164d228503abbdce70f6880ffa3d970e7443a" +dependencies = [ + "cc", +] + [[package]] name = "colored" version = "2.0.0" @@ -831,6 +1007,23 @@ dependencies = [ "libc", ] +[[package]] +name = "concolor-control" +version = "0.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7104119c2f80d887239879d0c50e033cd40eac9a3f3561e0684ba7d5d654f4da" +dependencies = [ + "atty", + "bitflags", + "concolor-query", +] + +[[package]] +name = "concolor-query" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad159cc964ac8f9d407cbc0aa44b02436c054b541f2b4b5f06972e1efdc54bc7" + [[package]] name = "concurrent-queue" version = "1.2.2" @@ -858,25 +1051,32 @@ version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03a5d7b21829bc7b4bf4754a978a241ae54ea55a40f92bb20216e54096f4b951" dependencies = [ - "aes-gcm", + "aes-gcm 0.8.0", "base64 0.13.0", - "hkdf", + "hkdf 0.10.0", "hmac 0.10.1", "percent-encoding", "rand 0.8.5", - "sha2", + "sha2 0.9.9", "time 0.2.27", "version_check", ] [[package]] name = "cookie" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f1c7727e460397e56abc4bddc1d49e07a1ad78fc98eb2e1c8f032a58a2f80d" +checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" dependencies = [ + "aes-gcm 0.9.4", + "base64 0.13.0", + "hkdf 0.12.3", + "hmac 0.12.1", "percent-encoding", - "time 0.2.27", + "rand 0.8.5", + "sha2 0.10.2", + "subtle", + "time 0.3.9", "version_check", ] @@ -911,11 +1111,31 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb25d077389e53838a8158c8e99174c5a9d902dee4904320db714f3c653ffba" +[[package]] +name = "crates-index" +version = "0.18.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0044896374c388ccbf1497dad6384bf6111dbcad9d7069506df7450ce9b62ea3" +dependencies = [ + "git2", + "hex 0.4.3", + "home", + "memchr", + "num_cpus", + "rayon", + "rustc-hash", + "semver 1.0.9", + "serde", + "serde_derive", + "serde_json", + "smartstring", +] + [[package]] name = "crates-io" -version = "0.33.1" +version = "0.34.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2d7714dc2b336c5a579a1a2aa2d41c7cd7a31ccb25e2ea908dba8934cfeb75a" +checksum = "6b4a87459133b2e708195eaab34be55039bc30e0d120658bd40794bb00b6328d" dependencies = [ "anyhow", "curl", @@ -949,6 +1169,41 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c" +dependencies = [ + "autocfg", + "cfg-if 1.0.0", + "crossbeam-utils", + "lazy_static", + "memoffset", + "scopeguard", +] + [[package]] name = "crossbeam-queue" version = "0.3.5" @@ -969,6 +1224,41 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "crossterm" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2102ea4f781910f8a5b98dd061f4c2023f479ce7bb1236330099ceb5a93cf17" +dependencies = [ + "bitflags", + "crossterm_winapi", + "libc", + "mio", + "parking_lot 0.12.0", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ae1b35a484aa10e07fe0638d02301c5ad24de82d310ccbd2f3693da5f09bf1c" +dependencies = [ + "winapi", +] + +[[package]] +name = "crypto-common" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57952ca27b5e3606ff4dd79b0020231aaf9d6aa76dc05fd30137538c50bd3ce8" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-hash" version = "0.3.4" @@ -1001,16 +1291,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "crypto-mac" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" -dependencies = [ - "generic-array", - "subtle", -] - [[package]] name = "ctor" version = "0.1.22" @@ -1027,7 +1307,16 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb4a30d54f7443bf3d6191dcd486aca19e67cb3c49fa7a06a319966346707e7f" dependencies = [ - "cipher", + "cipher 0.2.5", +] + +[[package]] +name = "ctr" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea" +dependencies = [ + "cipher 0.3.0", ] [[package]] @@ -1135,12 +1424,6 @@ version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499" -[[package]] -name = "difflib" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" - [[package]] name = "digest" version = "0.9.0" @@ -1151,12 +1434,33 @@ dependencies = [ ] [[package]] -name = "dirs" -version = "4.0.0" +name = "digest" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +checksum = "f2fb860ca6fafa5552fb6d0e816a69c8e49f0908bf524e30a90d97c85892d506" dependencies = [ - "dirs-sys", + "block-buffer 0.10.2", + "crypto-common", + "subtle", +] + +[[package]] +name = "dirs" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-next" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1" +dependencies = [ + "cfg-if 1.0.0", + "dirs-sys-next", ] [[package]] @@ -1171,16 +1475,21 @@ dependencies = [ ] [[package]] -name = "discard" -version = "1.0.4" +name = "dirs-sys-next" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d" +dependencies = [ + "libc", + "redox_users", + "winapi", +] [[package]] -name = "doc-comment" -version = "0.3.3" +name = "discard" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fea41bba32d969b513997752735605054bc0dfa92b4c56bf1189f2e174be7a10" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" [[package]] name = "dotenv" @@ -1188,6 +1497,12 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" +[[package]] +name = "dunce" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "453440c271cf5577fd2a40e4942540cb7d0d2f85e27c8d07dd0023c925a67541" + [[package]] name = "e2e" version = "0.1.0" @@ -1226,6 +1541,16 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_proxy" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a5019be18538406a43b5419a5501461f0c8b49ea7dfda0cfc32f4e51fc44be1" +dependencies = [ + "log", + "url", +] + [[package]] name = "erased-serde" version = "0.3.20" @@ -1292,6 +1617,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fixedbitset" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "279fb028e20b3c4c320317955b77c5e0c9701f05a1d309905d6fc702cdc5053e" + [[package]] name = "flate2" version = "1.0.22" @@ -1305,15 +1636,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "float-cmp" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" -dependencies = [ - "num-traits", -] - [[package]] name = "fnv" version = "1.0.7" @@ -1347,9 +1669,9 @@ dependencies = [ [[package]] name = "fqdn" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e6d13224e56390ed1c582c513492b8b4acc7865034a7f4a2077e52585175ea8" +checksum = "9b4b9cc8d7db413f35e3647159e2f726741d605522b7de6f2c59056da0badf79" [[package]] name = "futures" @@ -1528,14 +1850,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97304e4cd182c3846f7575ced3890c53012ce534ad9114046b0a9e00bb30a375" dependencies = [ "opaque-debug", - "polyval", + "polyval 0.4.5", +] + +[[package]] +name = "ghash" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99" +dependencies = [ + "opaque-debug", + "polyval 0.5.3", ] [[package]] name = "git2" -version = "0.13.25" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29229cc1b24c0e6062f6e742aa3e256492a5323365e5ed3413599f8a5eff7d6" +checksum = "d0155506aab710a86160ddb504a480d2964d7ab5b9e62419be69e0032bc5931c" dependencies = [ "bitflags", "libc", @@ -1548,9 +1880,9 @@ dependencies = [ [[package]] name = "git2-curl" -version = "0.14.1" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "883539cb0ea94bab3f8371a98cd8e937bbe9ee7c044499184aa4c17deb643a50" +checksum = "1ee51709364c341fbb6fe2a385a290fb9196753bdde2fc45447d27cd31b11b13" dependencies = [ "curl", "git2", @@ -1635,6 +1967,15 @@ dependencies = [ "unicode-segmentation", ] +[[package]] +name = "heck" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1655,6 +1996,9 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] [[package]] name = "hkdf" @@ -1662,10 +2006,19 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51ab2f639c231793c5f6114bdb9bbe50a7dbbfcd7c7c6bd8475dec2d991e964f" dependencies = [ - "digest", + "digest 0.9.0", "hmac 0.10.1", ] +[[package]] +name = "hkdf" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "791a029f6b9fc27657f6f188ec6e5e43f6911f6f878e0dc5501396e09809d437" +dependencies = [ + "hmac 0.12.1", +] + [[package]] name = "hmac" version = "0.8.1" @@ -1673,7 +2026,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "126888268dcc288495a26bf004b38c5fdbb31682f992c84ceb046a1f0fe38840" dependencies = [ "crypto-mac 0.8.0", - "digest", + "digest 0.9.0", ] [[package]] @@ -1683,17 +2036,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ "crypto-mac 0.10.1", - "digest", + "digest 0.9.0", ] [[package]] name = "hmac" -version = "0.11.0" +version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "crypto-mac 0.11.1", - "digest", + "digest 0.10.3", ] [[package]] @@ -1707,9 +2059,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" +checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" dependencies = [ "bytes", "fnv", @@ -1718,9 +2070,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ "bytes", "http", @@ -1787,9 +2139,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.18" +version = "0.14.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b26ae0a80afebe130861d90abf98e3814a4f28a4c6ffeb5ab8ebb2be311e0ef2" +checksum = "42dc3c131584288d375f2d07f822b0cb012d8c6fb899a5b9fdb3cb7eb9b6004f" dependencies = [ "bytes", "futures-channel", @@ -1811,12 +2163,25 @@ dependencies = [ [[package]] name = "hyper-reverse-proxy" -version = "0.6.0" -source = "git+https://github.com/jtroo/hyper-reverse-proxy#91adb990ed8c8be8b27bd797d0bd0491ad30d006" +version = "0.5.2-dev" +source = "git+https://github.com/chesedo/hyper-reverse-proxy?branch=master#a4deffef77685b37fda7224ae678d3d9f00d391e" dependencies = [ "hyper", "lazy_static", - "unicase", + "tokio", + "tracing", +] + +[[package]] +name = "hyper-timeout" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb958482e8c7be4bc3cf272a766a2b0bf1a6755e7a6ae777f017a31d11b13b1" +dependencies = [ + "hyper", + "pin-project-lite 0.2.8", + "tokio", + "tokio-io-timeout", ] [[package]] @@ -1832,6 +2197,19 @@ dependencies = [ "tokio-native-tls", ] +[[package]] +name = "hyperlocal" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" +dependencies = [ + "futures-util", + "hex 0.4.3", + "hyper", + "pin-project", + "tokio", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -1972,6 +2350,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "kstring" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b310ccceade8121d7d77fee406160e457c2f4e7c7982d589da3499bc7ea4526" +dependencies = [ + "serde", +] + [[package]] name = "kv-log-macro" version = "1.0.7" @@ -2001,9 +2388,9 @@ checksum = "efaa7b300f3b5fe8eb6bf21ce3895e1751d9665086af2d64b42f19701015ff4f" [[package]] name = "libgit2-sys" -version = "0.12.26+1.3.0" +version = "0.13.4+1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e1c899248e606fbfe68dcb31d8b0176ebab833b103824af31bddf4b7457494" +checksum = "d0fa6563431ede25f5cc7f6d803c6afbc1c5d3ad3d4925d12c882bf2b526f5d1" dependencies = [ "cc", "libc", @@ -2070,9 +2457,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.16" +version = "0.4.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6389c490849ff5bc16be905ae24bc913a9c8892e19b2341dbc175e14c341c2b8" +checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" dependencies = [ "cfg-if 1.0.0", "serde", @@ -2117,13 +2504,11 @@ checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb" [[package]] name = "md-5" -version = "0.9.1" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +checksum = "658646b21e0b72f7866c7038ab086d3d5e1cd6271f060fd37defb241949d0582" dependencies = [ - "block-buffer", - "digest", - "opaque-debug", + "digest 0.10.3", ] [[package]] @@ -2132,6 +2517,15 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" +[[package]] +name = "memoffset" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa361d4faea93603064a027415f07bd8e1d5c88c9fbf68bf56a285428fd79ce" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.16" @@ -2201,12 +2595,18 @@ dependencies = [ "log", "memchr", "mime", - "spin", + "spin 0.9.2", "tokio", "tokio-util 0.6.9", "version_check", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "native-tls" version = "0.2.8" @@ -2291,12 +2691,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "normalize-line-endings" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" - [[package]] name = "ntapi" version = "0.3.7" @@ -2356,11 +2750,20 @@ dependencies = [ "syn", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "opaque-debug" @@ -2422,6 +2825,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "os_str_bytes" +version = "6.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21326818e99cfe6ce1e524c2a805c189a99b5ae555a35d19f9a284b427d86afa" + [[package]] name = "output_vt100" version = "0.1.3" @@ -2491,6 +2900,12 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0744126afe1a6dd7f394cb50a716dbe086cb06e255e53d8d0185d82828358fb5" +[[package]] +name = "pathdiff" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8835116a5c179084a830efb3adc117ab007512b535bc1a21c991d3b32a6b44dd" + [[package]] name = "pear" version = "0.2.3" @@ -2520,6 +2935,16 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "petgraph" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pin-project" version = "1.0.10" @@ -2588,6 +3013,18 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "opaque-debug", + "universal-hash", +] + [[package]] name = "portpicker" version = "0.1.1" @@ -2603,41 +3040,11 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" -[[package]] -name = "predicates" -version = "2.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5aab5be6e4732b473071984b3164dbbfb7a3674d30ea5ff44410b6bcd960c3c" -dependencies = [ - "difflib", - "float-cmp", - "itertools", - "normalize-line-endings", - "predicates-core", - "regex", -] - -[[package]] -name = "predicates-core" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da1c2388b1513e1b605fcec39a95e0a9e8ef088f71443ef37099fa9ae6673fcb" - -[[package]] -name = "predicates-tree" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d86de6de25020a36c6d3643a86d9a6a9f552107c0559c60ea03551b5e16c032" -dependencies = [ - "predicates-core", - "termtree", -] - [[package]] name = "pretty_assertions" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c038cb5319b9c704bf9c227c261d275bfec0ad438118a2787ce47944fb228b" +checksum = "c89f989ac94207d048d92db058e4f6ec7342b0971fc58d1271ca148b799b3563" dependencies = [ "ansi_term", "ctor", @@ -2645,6 +3052,16 @@ dependencies = [ "output_vt100", ] +[[package]] +name = "prettyplease" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28f53e8b192565862cf99343194579a022eb9c7dd3a8d03134734803c7b3125" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro-crate" version = "1.1.3" @@ -2687,11 +3104,11 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro2" -version = "1.0.36" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" +checksum = "c54b25569025b7fc9651de43004ae593a75ad88543b17178aa5e1b9c4f15f56f" dependencies = [ - "unicode-xid", + "unicode-ident", ] [[package]] @@ -2707,11 +3124,102 @@ dependencies = [ "yansi", ] +[[package]] +name = "prost" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71adf41db68aa0daaefc69bb30bcd68ded9b9abaad5d1fbb6304c4fb390e083e" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae5a4388762d5815a9fc0dea33c56b021cdc8dde0c55e0c9ca57197254b0cab" +dependencies = [ + "bytes", + "cfg-if 1.0.0", + "cmake", + "heck 0.4.0", + "itertools", + "lazy_static", + "log", + "multimap", + "petgraph", + "prost", + "prost-types", + "regex", + "tempfile", + "which", +] + +[[package]] +name = "prost-derive" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b670f45da57fb8542ebdbb6105a925fe571b67f9e7ed9f47a06a84e72b4e7cc" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d0a014229361011dc8e69c8a1ec6c2e8d0f2af7c91e3ea3f5b2170298461e68" +dependencies = [ + "bytes", + "prost", +] + +[[package]] +name = "proto" +version = "0.1.0" +dependencies = [ + "prost", + "shuttle-common", + "tonic", + "tonic-build", +] + +[[package]] +name = "provisioner" +version = "0.1.0" +dependencies = [ + "clap 3.1.18", + "ctor", + "lazy_static", + "portpicker", + "prost", + "proto", + "rand 0.8.5", + "sqlx", + "thiserror", + "tokio", + "tonic", + "tonic-build", + "tracing", + "tracing-subscriber", +] + +[[package]] +name = "queues" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1475abae4f8ad4998590fe3acfe20104f0a5d48fc420c817cd2c09c3f56151f0" + [[package]] name = "quote" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632d02bff7f874a36f33ea8bb416cd484b90cc66c1194b1a1110d067a7013f58" +checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1" dependencies = [ "proc-macro2", ] @@ -2797,12 +3305,36 @@ dependencies = [ ] [[package]] -name = "redox_syscall" -version = "0.2.12" +name = "rayon" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" +checksum = "bd99e5772ead8baa5215278c9b15bf92087709e9c1b2d1f97cdb5a183c933a7d" dependencies = [ - "bitflags", + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "258bcdb5ac6dad48491bb2992db6b7cf74878b0384908af124823d118c99683f" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", + "num_cpus", +] + +[[package]] +name = "redox_syscall" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae183fc1b06c149f0c1793e1eb447c8b04bfe46d48e9e48bfb8d2d7ed64ecf0" +dependencies = [ + "bitflags", ] [[package]] @@ -2838,9 +3370,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.5.5" +version = "1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a11647b6b25ff05a515cb92c365cec08801e83423a235b51e231e1808747286" +checksum = "d83f127d94bdbcda4c8cc2e50f6f84f4b611f69c902699ca385a39c3a75f9ff1" dependencies = [ "aho-corasick", "memchr", @@ -2858,9 +3390,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.25" +version = "0.6.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "remove_dir_all" @@ -2910,9 +3442,9 @@ dependencies = [ [[package]] name = "reqwest-middleware" -version = "0.1.5" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b58621b8223cfc85b63d38b8d335c69b96a666d9b7561aa30a3b070ce1df31c" +checksum = "69539cea4148dce683bec9dc95be3f0397a9bb2c248a49c8296a9d21659a8cdd" dependencies = [ "anyhow", "async-trait", @@ -2926,9 +3458,9 @@ dependencies = [ [[package]] name = "reqwest-retry" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d5cb0170c4fa3f251c036ef482b35abc550afac6305cf875541c4b78dd76c6c" +checksum = "ce246a729eaa6aff5e215aee42845bf5fed9893cc6cd51aeeb712f34e04dd9f3" dependencies = [ "anyhow", "async-trait", @@ -2955,11 +3487,26 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin 0.5.2", + "untrusted", + "web-sys", + "winapi", +] + [[package]] name = "rocket" -version = "0.5.0-rc.1" +version = "0.5.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a71c18c42a0eb15bf3816831caf0dad11e7966f2a41aaf486a701979c4dd1f2" +checksum = "98ead083fce4a405feb349cf09abdf64471c6077f14e0ce59364aa90d4b99317" dependencies = [ "async-stream", "async-trait", @@ -2975,7 +3522,7 @@ dependencies = [ "memchr", "multer", "num_cpus", - "parking_lot 0.11.2", + "parking_lot 0.12.0", "pin-project-lite 0.2.8", "rand 0.8.5", "ref-cast", @@ -2985,10 +3532,10 @@ dependencies = [ "serde_json", "state", "tempfile", - "time 0.2.27", + "time 0.3.9", "tokio", "tokio-stream", - "tokio-util 0.6.9", + "tokio-util 0.7.1", "ubyte", "uuid", "version_check", @@ -2997,9 +3544,9 @@ dependencies = [ [[package]] name = "rocket_codegen" -version = "0.5.0-rc.1" +version = "0.5.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66f5fa462f7eb958bba8710c17c5d774bbbd59809fa76fb1957af7e545aea8bb" +checksum = "d6aeb6bb9c61e9cd2c00d70ea267bf36f76a4cc615e5908b349c2f9d93999b47" dependencies = [ "devise", "glob", @@ -3013,19 +3560,18 @@ dependencies = [ [[package]] name = "rocket_http" -version = "0.5.0-rc.1" +version = "0.5.0-rc.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23c8b7d512d2fcac2316ebe590cde67573844b99e6cc9ee0f53375fa16e25ebd" +checksum = "2ded65d127954de3c12471630bf4b81a2792f065984461e65b91d0fdaafc17a2" dependencies = [ - "cookie 0.15.1", + "cookie 0.16.0", "either", + "futures", "http", "hyper", "indexmap", "log", "memchr", - "mime", - "parking_lot 0.11.2", "pear", "percent-encoding", "pin-project-lite 0.2.8", @@ -3034,7 +3580,7 @@ dependencies = [ "smallvec", "stable-pattern", "state", - "time 0.2.27", + "time 0.3.9", "tokio", "uncased", "uuid", @@ -3046,6 +3592,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56770675ebc04927ded3e60633437841581c285dc6236109ea25fbf3beb7b59e" +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + [[package]] name = "rustc-workspace-hack" version = "1.0.0" @@ -3073,6 +3625,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustls" +version = "0.20.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aab8ee6c7097ed6057f43c187a62418d0c05a4bd5f18b3571db50ee0f9ce033" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "rustversion" version = "1.0.6" @@ -3116,6 +3680,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "security-framework" version = "2.6.1" @@ -3150,9 +3724,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.6" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a3381e03edd24287172047536f20cabde766e2cd3e65e6b00fb3af51c4f38d" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" dependencies = [ "serde", ] @@ -3165,18 +3739,18 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce31e24b01e1e524df96f1c2fdd054405f8d7376249a5110886fb4b658484789" +checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.136" +version = "1.0.137" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08597e7152fcd306f41838ed3e37be9eaeed2b61c42e2117266a554fab4662f9" +checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be" dependencies = [ "proc-macro2", "quote", @@ -3203,9 +3777,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.79" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e8d9fa5c3b304765ce1fd9c4c8a3de2c8db365a5b91be52f186efc675681d95" +checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ "itoa", "ryu", @@ -3235,17 +3809,38 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_with" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "946fa04a8ac43ff78a1f4b811990afb9ddbdf5890b46d6dda0ba1998230138b7" +dependencies = [ + "rustversion", + "serde", + "serde_with_macros", +] + +[[package]] +name = "serde_with_macros" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12e47be9471c72889ebafb5e14d5ff930d89ae7a67bbdb5f8abb564f845a927e" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha-1" -version = "0.9.8" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99cd6713db3cf16b6c84e06321e049a9b9f699826e16096d23bbcc44d15d51a6" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" dependencies = [ - "block-buffer", "cfg-if 1.0.0", "cpufeatures", - "digest", - "opaque-debug", + "digest 0.10.3", ] [[package]] @@ -3269,13 +3864,24 @@ version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800" dependencies = [ - "block-buffer", + "block-buffer 0.9.0", "cfg-if 1.0.0", "cpufeatures", - "digest", + "digest 0.9.0", "opaque-debug", ] +[[package]] +name = "sha2" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55deaec60f81eefe3cce0dc50bda92d6d8e88f2a27df7c5033b42afeb1ed2676" +dependencies = [ + "cfg-if 1.0.0", + "cpufeatures", + "digest 0.10.3", +] + [[package]] name = "sharded-slab" version = "0.1.4" @@ -3293,7 +3899,7 @@ checksum = "45bb67a18fa91266cc7807181f62f9178a6873bfad7dc788c42e6430db40184f" [[package]] name = "shuttle-codegen" -version = "0.3.0" +version = "0.3.1" dependencies = [ "pretty_assertions", "proc-macro2", @@ -3303,7 +3909,7 @@ dependencies = [ [[package]] name = "shuttle-common" -version = "0.3.0" +version = "0.3.1" dependencies = [ "anyhow", "chrono", @@ -3317,12 +3923,14 @@ dependencies = [ [[package]] name = "shuttle-service" -version = "0.3.1" +version = "0.3.3" dependencies = [ "anyhow", "async-trait", "axum", + "cargo", "chrono", + "futures", "hyper", "lazy_static", "libloading", @@ -3351,6 +3959,17 @@ dependencies = [ "signal-hook-registry", ] +[[package]] +name = "signal-hook-mio" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29ad2e15f37ec9a6cc544097b78a1ec90001e9f71b81338ca39f430adaca99af" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + [[package]] name = "signal-hook-registry" version = "1.4.0" @@ -3391,6 +4010,18 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "serde", + "static_assertions", + "version_check", +] + [[package]] name = "socket2" version = "0.4.4" @@ -3401,6 +4032,23 @@ dependencies = [ "winapi", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "spin" version = "0.9.2" @@ -3420,9 +4068,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc15591eb44ffb5816a4a70a7efd5dd87bfd3aa84c4c200401c4396140525826" +checksum = "551873805652ba0d912fec5bbb0f8b4cdd96baf8e2ebf5970e5671092966019b" dependencies = [ "sqlx-core", "sqlx-macros", @@ -3430,9 +4078,9 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "195183bf6ff8328bb82c0511a83faf60aacf75840103388851db61d7a9854ae3" +checksum = "e48c61941ccf5ddcada342cd59e3e5173b007c509e1e8e990dafc830294d9dc5" dependencies = [ "ahash", "atoi", @@ -3444,13 +4092,15 @@ dependencies = [ "crossbeam-queue", "dirs", "either", + "event-listener", "futures-channel", "futures-core", "futures-intrusive", "futures-util", "hashlink", "hex 0.4.3", - "hmac 0.11.0", + "hkdf 0.12.3", + "hmac 0.12.1", "indexmap", "itoa", "libc", @@ -3464,7 +4114,7 @@ dependencies = [ "serde", "serde_json", "sha-1", - "sha2", + "sha2 0.10.2", "smallvec", "sqlformat", "sqlx-rt", @@ -3477,17 +4127,17 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee35713129561f5e55c554bba1c378e2a7e67f81257b7311183de98c50e6f94" +checksum = "bc0fba2b0cae21fc00fe6046f8baa4c7fcb49e379f0f592b04696607f69ed2e1" dependencies = [ "dotenv", "either", - "heck", + "heck 0.4.0", "once_cell", "proc-macro2", "quote", - "sha2", + "sha2 0.10.2", "sqlx-core", "sqlx-rt", "syn", @@ -3496,9 +4146,9 @@ dependencies = [ [[package]] name = "sqlx-rt" -version = "0.5.11" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b555e70fbbf84e269ec3858b7a6515bcfe7a166a7cc9c636dd6efd20431678b6" +checksum = "4db708cd3e459078f85f39f96a00960bd841f66ee2a669e90bf36907f5a79aae" dependencies = [ "native-tls", "once_cell", @@ -3526,13 +4176,19 @@ dependencies = [ [[package]] name = "state" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cf4f5369e6d3044b5e365c9690f451516ac8f0954084622b49ea3fde2f6de5" +checksum = "dbe866e1e51e8260c9eed836a042a5e7f6726bb2b411dffeaa712e19c388f23b" dependencies = [ "loom", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stdweb" version = "0.4.20" @@ -3619,7 +4275,7 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c6b5c64445ba8094a6ab0c3cd2ad323e07171012d9c98b0b15651daf1787a10" dependencies = [ - "clap", + "clap 2.34.0", "lazy_static", "structopt-derive", ] @@ -3630,13 +4286,23 @@ version = "0.4.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ - "heck", + "heck 0.3.3", "proc-macro-error", "proc-macro2", "quote", "syn", ] +[[package]] +name = "subprocess" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2e86926081dda636c546d8c5e641661049d7562a68f5488be4a1f7f66f6086" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "subtle" version = "2.4.1" @@ -3654,13 +4320,13 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.90" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "704df27628939572cd88d33f171cd6f896f4eaca85252c6e0a72d8d8287ee86f" +checksum = "0748dd251e24453cb8717f0354206b91557e4ec8703673a4b30208f2abaf1ebf" dependencies = [ "proc-macro2", "quote", - "unicode-xid", + "unicode-ident", ] [[package]] @@ -3712,10 +4378,35 @@ dependencies = [ ] [[package]] -name = "termtree" -version = "0.2.4" +name = "terminal_size" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "test-context" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d813064aec716c2ed0e46f10c1e11143c8a73d394ec139d85487cf9c7c0abfa3" +dependencies = [ + "async-trait", + "futures", + "test-context-macros", +] + +[[package]] +name = "test-context-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f5c0709159d0fc65bd87254492efb5f53b84424321c4b4d316fe8508628fa5e" +dependencies = [ + "quote", + "syn", +] [[package]] name = "textwrap" @@ -3726,20 +4417,29 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "textwrap" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb" +dependencies = [ + "terminal_size", +] + [[package]] name = "thiserror" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" +checksum = "bd829fe32373d27f76265620b5309d0340cb8550f523c1dda251d6298069069a" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.30" +version = "1.0.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" +checksum = "0396bc89e626244658bef819e22d0cc459e795a5ebe878e6ec336d1674a8d79a" dependencies = [ "proc-macro2", "quote", @@ -3798,11 +4498,23 @@ dependencies = [ "libc", "standback", "stdweb", - "time-macros", + "time-macros 0.1.1", "version_check", "winapi", ] +[[package]] +name = "time" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2702e08a7a860f005826c6815dcac101b19b5eb330c27fe4a5928fec1d20ddd" +dependencies = [ + "itoa", + "libc", + "num_threads", + "time-macros 0.2.4", +] + [[package]] name = "time-macros" version = "0.1.1" @@ -3813,6 +4525,12 @@ dependencies = [ "time-macros-impl", ] +[[package]] +name = "time-macros" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42657b1a6f4d817cda8e7a0ace261fe0cc946cf3a80314390b22cc61ae080792" + [[package]] name = "time-macros-impl" version = "0.1.2" @@ -3843,9 +4561,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.17.0" +version = "1.19.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2af73ac49756f3f7c01172e34a23e5d0216f6c32333757c2c61feb2bbff5a5ee" +checksum = "c51a52ed6686dd62c320f9b89299e9dfb46f730c7a48e635c19f21d116cb1439" dependencies = [ "bytes", "libc", @@ -3861,6 +4579,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "tokio-io-timeout" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30b74022ada614a1b4834de765f9bb43877f910cc8ce4be40e89042c9223a8bf" +dependencies = [ + "pin-project-lite 0.2.8", + "tokio", +] + [[package]] name = "tokio-macros" version = "1.7.0" @@ -3893,6 +4621,19 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-test" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53474327ae5e166530d17f2d956afcb4f8a004de581b3cae10f12006bc8163e3" +dependencies = [ + "async-stream", + "bytes", + "futures-core", + "tokio", + "tokio-stream", +] + [[package]] name = "tokio-util" version = "0.6.9" @@ -3918,17 +4659,104 @@ dependencies = [ "futures-sink", "pin-project-lite 0.2.8", "tokio", + "tracing", +] + +[[package]] +name = "tokiotest-httpserver" +version = "0.2.1" +source = "git+https://github.com/shuttle-hq/tokiotest-httpserver?branch=feat/body#ca413a227397f0d0441b4454581ddd803acabdd7" +dependencies = [ + "async-trait", + "futures", + "hyper", + "lazy_static", + "queues", + "serde_json", + "test-context", + "tokio", + "tokio-test", + "tower-http 0.2.5", ] [[package]] name = "toml" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa" +checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7" dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744e9ed5b352340aa47ce033716991b5589e23781acb97cad37d4ea70560f55b" +dependencies = [ + "combine", + "indexmap", + "itertools", + "kstring", + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5376256e44f2443f8896ac012507c19a012df0fe8758b55246ae51a2279db51f" +dependencies = [ + "combine", + "indexmap", + "itertools", +] + +[[package]] +name = "tonic" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5be9d60db39854b30b835107500cf0aca0b0d14d6e1c3de124217c23a29c2ddb" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.13.0", + "bytes", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost", + "prost-derive", + "tokio", + "tokio-stream", + "tokio-util 0.7.1", + "tower", + "tower-layer", + "tower-service", + "tracing", + "tracing-futures", +] + +[[package]] +name = "tonic-build" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9263bf4c9bfaae7317c1c2faf7f18491d2fe476f70c414b73bf5d445b00ffa1" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn", +] + [[package]] name = "tower" version = "0.4.12" @@ -3937,8 +4765,11 @@ checksum = "9a89fd63ad6adf737582df5db40d286574513c69a11dac5214dc3b5603d6713e" dependencies = [ "futures-core", "futures-util", + "indexmap", "pin-project", "pin-project-lite 0.2.8", + "rand 0.8.5", + "slab", "tokio", "tokio-util 0.7.1", "tower-layer", @@ -3951,6 +4782,24 @@ name = "tower-http" version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aba3f3efabf7fb41fae8534fc20a817013dd1c12cb45441efb6c82e6556b4cd8" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-range-header", + "pin-project-lite 0.2.8", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c530c8675c1dbf98facee631536fa116b5fb6382d7dd6dc1b118d970eafe3ba" dependencies = [ "bitflags", "bytes", @@ -3979,9 +4828,9 @@ checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.32" +version = "0.1.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a1bdf54a7c28a2bbf701e1d2233f6c77f473486b94bee4f9678da5a148dca7f" +checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" dependencies = [ "cfg-if 1.0.0", "log", @@ -4003,14 +4852,24 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.23" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa31669fa42c09c34d94d8165dd2012e8ff3c66aca50f3bb226b68f216f2706c" +checksum = "7709595b8878a4965ce5e87ebf880a7d39c9afc6837721b21a5a816a8117d921" dependencies = [ - "lazy_static", + "once_cell", "valuable", ] +[[package]] +name = "tracing-futures" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" +dependencies = [ + "pin-project", + "tracing", +] + [[package]] name = "tracing-log" version = "0.1.2" @@ -4024,9 +4883,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.9" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0ab7bdc962035a87fba73f3acca9b8a8d0034c2e6f60b84aeaaddddc155dce" +checksum = "4bc28f93baff38037f64e6f43d34cfa1605f27a49c34e8a04c5e78b0babf2596" dependencies = [ "ansi_term", "lazy_static", @@ -4086,6 +4945,12 @@ version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" +[[package]] +name = "unicode-ident" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22af068fba1eb5edcb4aea19d382b2a3deb4c8f9d475c589b6ada9e0fd493ee" + [[package]] name = "unicode-normalization" version = "0.1.19" @@ -4129,6 +4994,32 @@ dependencies = [ "subtle", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9399fa2f927a3d327187cbd201480cee55bee6ac5d3c77dd27f0c6814cff16d5" +dependencies = [ + "base64 0.13.0", + "chunked_transfer", + "log", + "native-tls", + "once_cell", + "rustls", + "serde", + "serde_json", + "socks", + "url", + "webpki", + "webpki-roots", +] + [[package]] name = "url" version = "2.2.2" @@ -4150,9 +5041,9 @@ checksum = "936e4b492acfd135421d8dca4b1aa80a7bfc26e702ef3af710e0752684df5372" [[package]] name = "uuid" -version = "0.8.2" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +checksum = "c6d5d669b51467dcf7b2f1a796ce0f955f05f01cafda6c19d6e95f730df29238" dependencies = [ "getrandom 0.2.5", "serde", @@ -4166,9 +5057,9 @@ checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d" [[package]] name = "value-bag" -version = "1.0.0-alpha.8" +version = "1.0.0-alpha.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79923f7731dc61ebfba3633098bf3ac533bbd35ccd8c57e7088d9a5eebe0263f" +checksum = "2209b78d1249f7e6f3293657c9779fe31ced465df091bbd433a1cf88e916ec55" dependencies = [ "ctor", "erased-serde", @@ -4217,15 +5108,6 @@ dependencies = [ "quote", ] -[[package]] -name = "wait-timeout" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" -dependencies = [ - "libc", -] - [[package]] name = "waker-fn" version = "1.1.0" @@ -4351,9 +5233,9 @@ dependencies = [ [[package]] name = "webbrowser" -version = "0.6.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9c28b6b6a78440b02647358625e3febc90724126480b9da6a967b5f674b3554" +checksum = "fc6a3cffdb686fbb24d9fb8f03a213803277ed2300f11026a3afe1f108dc021b" dependencies = [ "jni", "ndk-glue", @@ -4363,6 +5245,25 @@ dependencies = [ "winapi", ] +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44d8de8415c823c8abd270ad483c6feeac771fad964890779f9a8cb24fbbc1bf" +dependencies = [ + "webpki", +] + [[package]] name = "wepoll-ffi" version = "0.1.2" @@ -4372,6 +5273,17 @@ dependencies = [ "cc", ] +[[package]] +name = "which" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +dependencies = [ + "either", + "lazy_static", + "libc", +] + [[package]] name = "whoami" version = "1.2.1" diff --git a/Cargo.toml b/Cargo.toml index c7a9060c5..868ead674 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,13 @@ [workspace] members = [ "api", - "service", "cargo-shuttle", - "common", "codegen", - "e2e" + "common", + "e2e", + "proto", + "provisioner", + "service" ] exclude = [ "examples" diff --git a/Dockerfile.nochef b/Dockerfile.nochef deleted file mode 100644 index 418ce034d..000000000 --- a/Dockerfile.nochef +++ /dev/null @@ -1,15 +0,0 @@ -FROM rust:buster AS runtime -RUN apt-get update &&\ - apt-get install -y curl postgresql supervisor -RUN pg_dropcluster $(pg_lsclusters -h | cut -d' ' -f-2 | head -n1) - -FROM rust:buster as builder -WORKDIR app -COPY . . -RUN cargo build --release --bin api - -FROM runtime -COPY --from=builder /app/target/release/api /usr/local/bin/unveil-backend -COPY docker/entrypoint.sh /bin/entrypoint.sh -COPY docker/supervisord.conf /usr/share/supervisord/supervisord.conf -ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/README.md b/README.md index 0aa971a3f..4ecbe3392 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ crate-type = ["cdylib"] [dependencies] rocket = "0.5.0-rc.1" -shuttle-service = { version = "0.3", features = ["web-rocket"] } +shuttle-service = { version = "0.3.3", features = ["web-rocket"] } ``` diff --git a/api/Cargo.toml b/api/Cargo.toml index 2dd3ad215..67d887442 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -1,33 +1,44 @@ [package] name = "api" -version = "0.1.0" +version = "0.1.2" edition = "2021" [dependencies] -rocket = { version = "0.5.0-rc.1", features = ["uuid", "serde_json", "json"] } -cargo = "0.59.0" -cargo-util = "0.1" -chrono = "0.4" -anyhow = "1.0.54" -libloading = "0.7.3" -futures = "0.3" -fqdn = "0.1" -uuid = { version = "0.8.2", features = ["v4"] } +anyhow = "1.0.57" async-mutex = "1.4.0" -serde = "1.0.136" -structopt = "0.3.26" -sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres"] } -tokio = { version = "1.15.0", features = ["full"] } -# not great, but the changes for hyper 0.14 and tokio 1 have not been merged on the main repo -hyper-reverse-proxy = { git = "https://github.com/jtroo/hyper-reverse-proxy" } -hyper = { version = "0.14", features = ["client", "http1", "http2", "tcp" ] } # for reverse proxying -log = "0.4.14" +async-trait = "0.1.56" +base64 = "0.13.0" +cargo = "0.62.0" +cargo-util = "0.1.2" +chrono = "0.4.19" env_logger = "0.9.0" -async-trait = "0.1.52" +fqdn = "0.1.9" +futures = "0.3.21" +hyper = { version = "0.14.19", features = ["client", "http1", "http2", "tcp" ] } # for reverse proxying +# not great, but waiting for WebSocket changes to be merged +hyper-reverse-proxy = { git = "https://github.com/chesedo/hyper-reverse-proxy", branch = "master" } lazy_static = "1.4.0" -toml = "0.5.8" -base64 = "0.13.0" +libloading = "0.7.3" +log = "0.4.17" rand = "0.8.5" +rocket = { version = "0.5.0-rc.2", features = ["uuid", "serde_json", "json"] } +serde = "1.0.137" +sqlx = { version = "0.5.13", features = ["runtime-tokio-native-tls", "postgres"] } +structopt = "0.3.26" +tokio = { version = "1.19.2", features = ["full"] } +toml = "0.5.9" +tonic = "0.7.2" +uuid = { version = "1.1.1", features = ["v4"] } + +[dependencies.proto] +version = "0.1.0" +path = "../proto" + +[dependencies.shuttle-common] +version = "0.3.1" +path = "../common" -shuttle-common = { version = "0.3.0", path = "../common" } -shuttle-service = { version = "0.3.0", path = "../service", features = [ "loader", "secrets" ] } +[dependencies.shuttle-service] +version = "0.3.3" +path = "../service" +features = ["loader", "secrets"] diff --git a/Dockerfile b/api/Containerfile similarity index 62% rename from Dockerfile rename to api/Containerfile index ac50fa987..dcb9b878e 100644 --- a/Dockerfile +++ b/api/Containerfile @@ -1,11 +1,12 @@ FROM rust:buster as chef +RUN apt-get update &&\ + apt-get install -y protobuf-compiler RUN cargo install cargo-chef WORKDIR app FROM rust:buster AS runtime RUN apt-get update &&\ - apt-get install -y curl postgresql supervisor -RUN pg_dropcluster $(pg_lsclusters -h | cut -d' ' -f-2 | head -n1) + apt-get install -y supervisor FROM chef AS planner COPY . . @@ -20,7 +21,6 @@ RUN cargo build --bin api FROM runtime COPY --from=builder /app/target/debug/api /usr/local/bin/shuttle-backend -COPY docker/entrypoint.sh /bin/entrypoint.sh -COPY docker/wait-for-pg-then /usr/bin/wait-for-pg-then -COPY docker/supervisord.conf /usr/share/supervisord/supervisord.conf +COPY api/docker/entrypoint.sh /bin/entrypoint.sh +COPY api/docker/supervisord.conf /usr/share/supervisord/supervisord.conf ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/Dockerfile.dev b/api/Containerfile.dev similarity index 62% rename from Dockerfile.dev rename to api/Containerfile.dev index 84d0fe7f7..32158c0b7 100644 --- a/Dockerfile.dev +++ b/api/Containerfile.dev @@ -1,16 +1,17 @@ FROM rust:buster as runtime RUN apt-get update &&\ - apt-get install -y curl postgresql supervisor -RUN pg_dropcluster $(pg_lsclusters -h | cut -d' ' -f-2 | head -n1) + apt-get install -y supervisor RUN rustup component add rust-src FROM rust:buster AS chef +RUN apt-get update &&\ + apt-get install -y protobuf-compiler WORKDIR app RUN cargo install cargo-chef FROM chef AS planner COPY . . -COPY ./docker/config.toml $CARGO_HOME/config.toml +COPY ./api/docker/config.toml $CARGO_HOME/config.toml RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder @@ -26,8 +27,7 @@ COPY --from=builder /app/service /app/service COPY --from=builder /app/common /app/common COPY --from=builder /app/codegen /app/codegen -COPY docker/config.toml $CARGO_HOME/config.toml -COPY docker/entrypoint.sh /bin/entrypoint.sh -COPY docker/wait-for-pg-then /usr/bin/wait-for-pg-then -COPY docker/supervisord.conf /usr/share/supervisord/supervisord.conf +COPY api/docker/config.toml $CARGO_HOME/config.toml +COPY api/docker/entrypoint.sh /bin/entrypoint.sh +COPY api/docker/supervisord.conf /usr/share/supervisord/supervisord.conf ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/docker/config.toml b/api/docker/config.toml similarity index 100% rename from docker/config.toml rename to api/docker/config.toml diff --git a/api/docker/entrypoint.sh b/api/docker/entrypoint.sh new file mode 100755 index 000000000..601e68f95 --- /dev/null +++ b/api/docker/entrypoint.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +if [ -z $PROXY_FQDN ] +then + echo "The variable 'PROXY_FQDN' is missing" + exit 1 +fi + +if [ -z $PROVISIONER_ADDRESS ] +then + echo "The variable 'PROVISIONER_ADDRESS' is missing" + exit 1 +fi + +export CRATES_PATH=${CRATES_PATH:-/var/lib/shuttle/crates} + +mkdir -p $CRATES_PATH + +export PROXY_PORT=${PROXY_PORT:-8000} + +export API_PORT=${API_PORT:-8001} + +if [[ ! -z "${SHUTTLE_USERS_TOML}" && ! -s "${SHUTTLE_USERS_TOML}" ]] +then + if [[ -z "${SHUTTLE_INITIAL_KEY}" ]] + then + echo "\$SHUTTLE_INITIAL_KEY is not set to create initial user's key" + exit 1 + fi + + echo "Creating a first user with key '${SHUTTLE_INITIAL_KEY}' at '${SHUTTLE_USERS_TOML}'" + mkdir -p $(dirname "${SHUTTLE_USERS_TOML}") + echo -e "[$SHUTTLE_INITIAL_KEY]\nname = 'first-user'\nprojects = []" > "${SHUTTLE_USERS_TOML}" +fi + +exec supervisord -n -c /usr/share/supervisord/supervisord.conf diff --git a/api/docker/supervisord.conf b/api/docker/supervisord.conf new file mode 100644 index 000000000..bd8c834ed --- /dev/null +++ b/api/docker/supervisord.conf @@ -0,0 +1,14 @@ +[supervisord] +loglevel=debug + +[program:shuttle-api] +command=/usr/local/bin/shuttle-backend --path %(ENV_CRATES_PATH)s --bind-addr 0.0.0.0 --api-port %(ENV_API_PORT)s --proxy-port %(ENV_PROXY_PORT)s --proxy-fqdn %(ENV_PROXY_FQDN)s --provisioner-address %(ENV_PROVISIONER_ADDRESS)s +redirect_stderr=true +environment=RUST_BACKTRACE="1",RUST_LOG="debug" +startretries=3 +startsecs=5 +autorestart=true + +[eventlistener:quit_on_failure] +events=PROCESS_STATE_FATAL +command=sh -c 'while true; do echo "READY"; read line; kill -15 1; echo "RESULT 2"; echo "OK"; done' diff --git a/api/src/args.rs b/api/src/args.rs index 3a838d8b0..1d20a6498 100644 --- a/api/src/args.rs +++ b/api/src/args.rs @@ -31,6 +31,14 @@ pub struct Args { pub(crate) bind_addr: IpAddr, #[structopt(long, about = "Fully qualified domain name deployed services are reachable at", parse(try_from_str = parse_fqdn))] pub(crate) proxy_fqdn: FQDN, + #[structopt(long, about = "Address to connect to the provisioning service")] + pub(crate) provisioner_address: String, + #[structopt( + long, + about = "Port provisioner is reachable at", + default_value = "5001" + )] + pub(crate) provisioner_port: Port, } fn parse_fqdn(src: &str) -> Result { diff --git a/api/src/build.rs b/api/src/build.rs index 5ffe772f9..0006da0d1 100644 --- a/api/src/build.rs +++ b/api/src/build.rs @@ -3,11 +3,9 @@ use std::path::{Path, PathBuf}; use std::process::Command; use anyhow::{anyhow, Context, Result}; -use cargo::core::compiler::CompileMode; -use cargo::core::Workspace; -use cargo::ops::CompileOptions; use rocket::tokio; use rocket::tokio::io::AsyncWriteExt; +use shuttle_service::loader::build_crate; use uuid::Uuid; #[cfg(debug_assertions)] @@ -190,31 +188,3 @@ fn extract_tarball(crate_path: &Path, project_path: &Path) -> Result<()> { Ok(()) } } - -/// Given a project directory path, builds the crate -fn build_crate(project_path: &Path, buf: Box) -> Result { - let mut shell = cargo::core::Shell::from_write(buf); - shell.set_verbosity(cargo::core::Verbosity::Normal); - - let cwd = std::env::current_dir() - .with_context(|| "couldn't get the current directory of the process")?; - let homedir = cargo::util::homedir(&cwd).ok_or_else(|| { - anyhow!( - "Cargo couldn't find your home directory. \ - This probably means that $HOME was not set." - ) - })?; - - let config = cargo::Config::new(shell, cwd, homedir); - let manifest_path = project_path.join("Cargo.toml"); - - let ws = Workspace::new(&manifest_path, &config)?; - let opts = CompileOptions::new(&config, CompileMode::Build)?; - let compilation = cargo::ops::compile(&ws, &opts)?; - - if compilation.cdylibs.is_empty() { - return Err(anyhow!("a cdylib was not created")); - } - - Ok(compilation.cdylibs[0].path.clone()) -} diff --git a/api/src/database.rs b/api/src/database.rs deleted file mode 100644 index 66420d48e..000000000 --- a/api/src/database.rs +++ /dev/null @@ -1,137 +0,0 @@ -use std::time::Duration; - -use lazy_static::lazy_static; -use rand::Rng; -use shuttle_common::project::ProjectName; -use shuttle_common::DatabaseReadyInfo; -use sqlx::postgres::{PgPool, PgPoolOptions}; - -lazy_static! { - static ref SUDO_POSTGRES_CONNECTION_STRING: String = format!( - "postgres://postgres:{}@localhost", - std::env::var("PG_PASSWORD").expect( - "superuser postgres role password expected as environment variable PG_PASSWORD" - ) - ); -} - -fn generate_role_password() -> String { - rand::thread_rng() - .sample_iter(&rand::distributions::Alphanumeric) - .take(12) - .map(char::from) - .collect() -} - -pub(crate) struct State { - project: ProjectName, - context: Context, - info: Option, -} - -impl State { - pub(crate) fn new(project: &ProjectName, context: &Context) -> Self { - Self { - project: project.clone(), - context: context.clone(), - info: None, - } - } - - pub(crate) async fn request(&mut self) -> sqlx::Result { - if self.info.is_some() { - return Ok(self.info.clone().unwrap()); - } - - let role_name = format!("user-{}", self.project); - let role_password = generate_role_password(); - let database_name = format!("db-{}", self.project); - - let pool = &self.context.sudo_pool; - - // Check if this deployment already has its own role: - let rows = sqlx::query("SELECT * FROM pg_roles WHERE rolname = $1") - .bind(&role_name) - .fetch_all(pool) - .await?; - - if rows.is_empty() { - // Create role if it does not already exist: - // TODO: Should be able to use `.bind` instead of `format!` but doesn't seem to - // insert quotes correctly. - let create_role_query = format!( - "CREATE ROLE \"{}\" PASSWORD '{}' LOGIN", - role_name, role_password - ); - sqlx::query(&create_role_query).execute(pool).await?; - - debug!( - "created new role '{}' in database for project '{}'", - role_name, database_name - ); - } else { - // If the role already exists then change its password: - let alter_password_query = format!( - "ALTER ROLE \"{}\" WITH PASSWORD '{}'", - role_name, role_password - ); - sqlx::query(&alter_password_query).execute(pool).await?; - - debug!( - "role '{}' already exists so updating their password", - role_name - ); - } - - // Since user creation is not atomic, need to separately check for DB existence - let get_database_query = "SELECT 1 FROM pg_database WHERE datname = $1"; - let database = sqlx::query(get_database_query) - .bind(&database_name) - .fetch_all(pool) - .await?; - if database.is_empty() { - debug!("database '{}' does not exist, creating", database_name); - // Create the database (owned by the new role): - let create_database_query = format!( - "CREATE DATABASE \"{}\" OWNER '{}'", - database_name, role_name - ); - sqlx::query(&create_database_query).execute(pool).await?; - - debug!( - "created database '{}' belonging to '{}'", - database_name, role_name - ); - } else { - debug!( - "database '{}' already exists, not recreating", - database_name - ); - } - - let info = DatabaseReadyInfo::new(role_name, role_password, database_name); - self.info = Some(info.clone()); - Ok(info) - } - - pub(crate) fn to_info(&self) -> Option { - self.info.clone() - } -} - -#[derive(Clone)] -pub struct Context { - sudo_pool: PgPool, -} - -impl Context { - pub async fn new() -> sqlx::Result { - Ok(Context { - sudo_pool: PgPoolOptions::new() - .min_connections(4) - .max_connections(12) - .connect_timeout(Duration::from_secs(60)) - .connect_lazy(&SUDO_POSTGRES_CONNECTION_STRING)?, - }) - } -} diff --git a/api/src/deployment.rs b/api/src/deployment.rs index 337608910..28977ab14 100644 --- a/api/src/deployment.rs +++ b/api/src/deployment.rs @@ -4,13 +4,13 @@ use std::fs::DirEntry; use std::io::Write; use std::net::{Ipv4Addr, SocketAddr, SocketAddrV4, TcpListener}; use std::path::{Path, PathBuf}; -use std::sync::mpsc::SyncSender; use std::sync::Arc; use anyhow::{anyhow, Context as AnyhowContext}; use chrono::{DateTime, Utc}; use futures::prelude::*; use libloading::Library; +use proto::provisioner::provisioner_client::ProvisionerClient; use rocket::data::ByteUnit; use rocket::{tokio, Data}; use shuttle_common::project::ProjectName; @@ -20,11 +20,13 @@ use shuttle_common::{ use shuttle_service::loader::Loader; use shuttle_service::logger::Log; use shuttle_service::ServeHandle; +use tokio::sync::mpsc::UnboundedSender; use tokio::sync::{mpsc, RwLock}; +use tonic::transport::{Channel, Endpoint}; use crate::build::Build; use crate::router::Router; -use crate::{database, BuildSystem, ShuttleFactory}; +use crate::{BuildSystem, ShuttleFactory}; // This controls the maximum number of deploys an api instance can run // This is mainly needed because tokio::task::spawn_blocking keeps an internal pool for the number of blocking threads @@ -105,12 +107,7 @@ impl Deployment { /// Tries to advance the deployment one stage. Does nothing if the deployment /// is in a terminal state. - pub(crate) async fn advance( - &self, - context: &Context, - db_context: &database::Context, - run_logs_tx: SyncSender, - ) { + pub(crate) async fn advance(&self, context: &Context, run_logs_tx: UnboundedSender) { { trace!("waiting to get write on the state"); let meta = self.meta().await; @@ -160,9 +157,12 @@ impl Deployment { ); debug!("{}: factory phase", meta.project); - let db_state = database::State::new(&meta.project, db_context); - let mut factory = ShuttleFactory::new(db_state); + let mut factory = ShuttleFactory::new( + context.provisioner_client.clone(), + context.provisioner_address.clone(), + meta.project.clone(), + ); let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), port); match loader.load(&mut factory, addr, run_logs_tx, meta.id).await { Err(e) => { @@ -172,7 +172,7 @@ impl Deployment { Ok((handle, so)) => { debug!("{}: factory phase DONE", meta.project); self.meta.write().await.database_deployment = - factory.to_database_info(); + factory.into_database_info(); // Remove stale active deployments if let Some(stale_id) = context.router.promote(meta.host, meta.id).await @@ -289,6 +289,7 @@ pub(crate) struct DeploymentSystem { job_queue: JobQueue, router: Arc, fqdn: String, + pub(crate) provisioner_address: String, } const JOB_QUEUE_SIZE: usize = 200; @@ -298,11 +299,7 @@ struct JobQueue { } impl JobQueue { - async fn new( - context: Context, - db_context: database::Context, - run_logs_tx: SyncSender, - ) -> Self { + async fn new(context: Context, run_logs_tx: UnboundedSender) -> Self { let (send, mut recv) = mpsc::channel::>(JOB_QUEUE_SIZE); log::debug!("starting job processor task"); @@ -316,7 +313,7 @@ impl JobQueue { while !deployment.deployment_finished().await { let run_logs_tx = run_logs_tx.clone(); - deployment.advance(&context, &db_context, run_logs_tx).await; + deployment.advance(&context, run_logs_tx).await; } debug!("ended deployment job for id: '{}'", id); @@ -342,12 +339,19 @@ pub(crate) struct Context { router: Arc, build_system: Box, deployments: Arc>, + provisioner_client: ProvisionerClient, + provisioner_address: String, } impl DeploymentSystem { - pub(crate) async fn new(build_system: Box, fqdn: String) -> Self { + pub(crate) async fn new( + build_system: Box, + fqdn: String, + provisioner_address: String, + provisioner_port: Port, + ) -> Self { let router: Arc = Default::default(); - let (tx, rx) = std::sync::mpsc::sync_channel::(64); + let (tx, mut rx) = mpsc::unbounded_channel::(); let deployments = Arc::new(RwLock::new( Self::initialise_from_fs(&build_system.fs_root(), &fqdn).await, @@ -356,31 +360,34 @@ impl DeploymentSystem { let deployments_log = deployments.clone(); tokio::spawn(async move { - loop { - let res = rx.recv(); + while let Some(log) = rx.recv().await { + let mut deployments_log = deployments_log.write().await; - if let Ok(log) = res { - let mut deployments_log = deployments_log.write().await; - if let Some(deployment) = deployments_log.get_mut(&log.deployment_id) { - deployment.add_runtime_log(log.datetime, log.item).await; - } - } else { - break; + if let Some(deployment) = deployments_log.get_mut(&log.deployment_id) { + deployment.add_runtime_log(log.datetime, log.item).await; } } }); + let provisioner_uri = Endpoint::try_from(format!( + "http://{}:{}", + provisioner_address, provisioner_port + )) + .expect("provisioner uri to be valid"); + + let provisioner_client = ProvisionerClient::connect(provisioner_uri) + .await + .expect("failed to connect to provisioner"); + let context = Context { router: router.clone(), build_system, deployments: deployments.clone(), + provisioner_client, + provisioner_address: provisioner_address.clone(), }; - let db_context = database::Context::new() - .await - .expect("failed to create lazy connection to database"); - - let job_queue = JobQueue::new(context, db_context, tx).await; + let job_queue = JobQueue::new(context, tx).await; debug!("loading deployments into job processor"); for deployment in deployments.read().await.values() { @@ -393,6 +400,7 @@ impl DeploymentSystem { job_queue, router, fqdn, + provisioner_address, } } diff --git a/api/src/factory.rs b/api/src/factory.rs index 9f26f321e..f55dbf250 100644 --- a/api/src/factory.rs +++ b/api/src/factory.rs @@ -1,34 +1,57 @@ use async_trait::async_trait; -use shuttle_common::DatabaseReadyInfo; +use proto::provisioner::{provisioner_client::ProvisionerClient, DatabaseRequest}; +use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; use shuttle_service::Factory; - -use crate::database; +use tonic::{transport::Channel, Request}; pub(crate) struct ShuttleFactory { - database: database::State, + project_name: ProjectName, + provisioner_client: ProvisionerClient, + provisioner_address: String, + info: Option, } impl ShuttleFactory { - pub(crate) fn new(database: database::State) -> Self { - Self { database } + pub(crate) fn new( + provisioner_client: ProvisionerClient, + provisioner_address: String, + project_name: ProjectName, + ) -> Self { + Self { + provisioner_client, + provisioner_address, + project_name, + info: None, + } } -} -impl ShuttleFactory { - pub(crate) fn to_database_info(&self) -> Option { - self.database.to_info() + pub(crate) fn into_database_info(self) -> Option { + self.info } } #[async_trait] impl Factory for ShuttleFactory { async fn get_sql_connection_string(&mut self) -> Result { - let conn_str = self - .database - .request() + if let Some(ref info) = self.info { + return Ok(info.connection_string(&self.provisioner_address)); + } + + let request = Request::new(DatabaseRequest { + project_name: self.project_name.to_string(), + }); + + let response = self + .provisioner_client + .provision_database(request) .await .map_err(shuttle_service::error::CustomError::new)? - .connection_string("localhost"); + .into_inner(); + + let info: DatabaseReadyInfo = response.into(); + let conn_str = info.connection_string(&self.provisioner_address); + self.info = Some(info); + debug!("giving a sql connection string: {}", conn_str); Ok(conn_str) } diff --git a/api/src/main.rs b/api/src/main.rs index 36d1b9d68..f14170793 100644 --- a/api/src/main.rs +++ b/api/src/main.rs @@ -8,7 +8,6 @@ mod args; mod auth; mod auth_admin; mod build; -mod database; mod deployment; mod factory; mod proxy; @@ -52,6 +51,11 @@ async fn get_or_create_user( #[get("/status")] async fn status() {} +#[get("/version")] +async fn version() -> String { + String::from(shuttle_service::VERSION) +} + #[get("/<_>/deployments/")] async fn get_deployment( state: &State, @@ -144,7 +148,8 @@ async fn project_secrets( .await?; if let Some(database_deployment) = &deployment.database_deployment { - let conn_str = database_deployment.connection_string("localhost"); + let conn_str = + database_deployment.connection_string(&state.deployment_manager.provisioner_address); let conn = sqlx::PgPool::connect(&conn_str) .await .map_err(|e| DeploymentApiError::Internal(e.to_string()))?; @@ -171,7 +176,7 @@ fn main() -> Result<(), Box> { .build() .unwrap() .block_on(async { - rocket().await.launch().await?; + let _rocket = rocket().await.launch().await?; Ok(()) }) @@ -187,8 +192,15 @@ async fn rocket() -> Rocket { let args: Args = Args::from_args(); let build_system = FsBuildSystem::initialise(args.path).unwrap(); - let deployment_manager = - Arc::new(DeploymentSystem::new(Box::new(build_system), args.proxy_fqdn.to_string()).await); + let deployment_manager = Arc::new( + DeploymentSystem::new( + Box::new(build_system), + args.proxy_fqdn.to_string(), + args.provisioner_address, + args.provisioner_port, + ) + .await, + ); start_proxy(args.bind_addr, args.proxy_port, deployment_manager.clone()).await; @@ -214,7 +226,7 @@ async fn rocket() -> Rocket { project_secrets ], ) - .mount("/", routes![get_or_create_user, status]) + .mount("/", routes![get_or_create_user, status, version]) .manage(state) .manage(user_directory) } diff --git a/api/src/proxy.rs b/api/src/proxy.rs index e77f97ad1..d9d2fd885 100644 --- a/api/src/proxy.rs +++ b/api/src/proxy.rs @@ -6,8 +6,11 @@ use ::hyper::server::conn::AddrStream; use ::hyper::server::Server; use ::hyper::service::{make_service_fn, service_fn}; use ::hyper::{Body, Request, Response, StatusCode}; +use hyper::client::connect::dns::GaiResolver; +use hyper::client::HttpConnector; use hyper::header::{HeaderValue, SERVER}; -use hyper_reverse_proxy::ProxyError; +use hyper::Client; +use hyper_reverse_proxy::{ProxyError, ReverseProxy}; use lazy_static::lazy_static; use shuttle_common::Port; @@ -15,6 +18,8 @@ use crate::DeploymentSystem; lazy_static! { static ref HEADER_SERVER: HeaderValue = "shuttle.rs".parse().unwrap(); + static ref PROXY_CLIENT: ReverseProxy> = + ReverseProxy::new(Client::new()); } pub(crate) async fn start( @@ -94,6 +99,10 @@ async fn handle( ProxyError::ForwardHeaderError => { log::warn!("error while handling request in reverse proxy: 'fwd header error'"); } + ProxyError::UpgradeError(e) => log::warn!( + "error while handling request needing upgrade in reverse proxy: {}", + e + ), }; Ok(Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) @@ -109,7 +118,7 @@ async fn reverse_proxy( req: Request, ) -> Result, ProxyError> { let forward_uri = format!("http://127.0.0.1:{}", port); - let mut response = hyper_reverse_proxy::call(ip, &forward_uri, req).await?; + let mut response = PROXY_CLIENT.call(ip, &forward_uri, req).await?; response.headers_mut().insert(SERVER, HEADER_SERVER.clone()); diff --git a/api/users.toml b/api/users.toml index 32852fb78..9646d9e58 100644 --- a/api/users.toml +++ b/api/users.toml @@ -4,6 +4,7 @@ projects = [ 'hello-world-rocket-app', 'postgres-rocket-app', 'hello-world-axum-app', + 'websocket-axum-app', 'authentication-rocket-app', 'hello-world-tide-app', 'hello-world-tower-app', diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index 587a576cc..c80d73c2d 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -1,32 +1,49 @@ [package] name = "cargo-shuttle" -version = "0.3.0" +version = "0.3.2" edition = "2021" license = "Apache-2.0" description = "A cargo command for the shuttle platform (https://www.shuttle.rs/)" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = "1.0.53" -chrono = "0.4" -colored = "2.0" +anyhow = "1.0.57" +async-trait = "0.1.56" +bollard = "0.12.0" +cargo = "0.62.0" +cargo-edit = { version = "0.9.1", features = ["cli"] } +cargo_metadata = "0.14.2" +chrono = "0.4.19" +colored = "2.0.0" +crossterm = "0.23.2" dirs = "4.0.0" -log = "0.4" -serde_json = "1.0.79" -serde = { version = "1.0.136", features = ["derive"] } -reqwest = { version = "0.11.9", features = ["json"] } -reqwest-middleware = "0.1" -reqwest-retry = "0.1" -cargo = "0.59.0" -tokio = "1.0" -futures = "0.3" -toml = "0.5.8" +env_logger = "0.9.0" +futures = "0.3.21" +log = "0.4.17" +portpicker = "0.1.1" +reqwest = { version = "0.11.10", features = ["json"] } +reqwest-middleware = "0.1.6" +reqwest-retry = "0.1.5" +semver = "1.0.9" +serde = { version = "1.0.137", features = ["derive"] } +serde_json = "1.0.81" structopt = "0.3.26" -cargo_metadata = "0.14.2" -webbrowser = "0.6" +tokio = "1.19.2" +toml = "0.5.9" +toml_edit = "0.14.4" +uuid = { version = "1.1.1", features = ["v4"] } +webbrowser = "0.7.1" + +[dependencies.shuttle-common] +version = "0.3.1" +path = "../common" -shuttle-common = { version = "0.3.0", path = "../common" } +[dependencies.shuttle-service] +version = "0.3.3" +path = "../service" +features = ["loader"] [dev-dependencies] -assert_cmd = "2.0.4" -predicates = "2.1.1" +test-context = "0.1.3" +# Tmp until this branch is merged and released +tokiotest-httpserver = { git = "https://github.com/shuttle-hq/tokiotest-httpserver", branch = "feat/body" } diff --git a/cargo-shuttle/src/args.rs b/cargo-shuttle/src/args.rs index abb2acd4a..e52bfaf7b 100644 --- a/cargo-shuttle/src/args.rs +++ b/cargo-shuttle/src/args.rs @@ -1,6 +1,6 @@ use std::{ ffi::{OsStr, OsString}, - fs::canonicalize, + fs::{canonicalize, create_dir_all}, path::PathBuf, }; @@ -31,12 +31,12 @@ pub struct Args { } // Common args for subcommands that deal with projects. -#[derive(StructOpt)] +#[derive(StructOpt, Debug)] pub struct ProjectArgs { #[structopt( global = true, long, - parse(try_from_os_str = parse_working_directory), + parse(try_from_os_str = parse_path), default_value = ".", )] /// Specify the working directory @@ -46,15 +46,12 @@ pub struct ProjectArgs { pub name: Option, } -fn parse_working_directory(working_directory: &OsStr) -> Result { - canonicalize(working_directory) - .map_err(|e| format!("could not turn {working_directory:?} into a real path: {e}").into()) -} - #[derive(StructOpt)] pub enum Command { #[structopt(about = "deploy a shuttle project")] Deploy(DeployArgs), + #[structopt(about = "create a new shuttle project")] + Init(InitArgs), #[structopt(about = "view the status of a shuttle project")] Status, #[structopt(about = "view the logs of a shuttle project")] @@ -65,6 +62,8 @@ pub enum Command { Auth(AuthArgs), #[structopt(about = "login to the shuttle platform")] Login(LoginArgs), + #[structopt(about = "run a shuttle project locally")] + Run(RunArgs), } #[derive(StructOpt)] @@ -83,4 +82,35 @@ pub struct AuthArgs { pub struct DeployArgs { #[structopt(long, about = "allow dirty working directories to be packaged")] pub allow_dirty: bool, + #[structopt(long, about = "allows pre-deploy tests to be skipped")] + pub no_test: bool, +} + +#[derive(StructOpt, Debug)] +pub struct RunArgs { + #[structopt(long, about = "port to start service on", default_value = "8000")] + pub port: u16, +} + +#[derive(StructOpt)] +pub struct InitArgs { + #[structopt( + about = "the path to initialize a new shuttle project", + parse(try_from_os_str = parse_init_path), + default_value = ".", + )] + pub path: PathBuf, +} + +// Helper function to parse and return the absolute path +fn parse_path(path: &OsStr) -> Result { + canonicalize(path).map_err(|e| format!("could not turn {path:?} into a real path: {e}").into()) +} + +// Helper function to parse, create if not exists, and return the absolute path +fn parse_init_path(path: &OsStr) -> Result { + // Create the directory if does not exist + create_dir_all(path).expect("could not find or create a directory with the given path"); + + parse_path(path) } diff --git a/cargo-shuttle/src/client.rs b/cargo-shuttle/src/client.rs index 2997358f1..fefbf26b7 100644 --- a/cargo-shuttle/src/client.rs +++ b/cargo-shuttle/src/client.rs @@ -4,9 +4,6 @@ use std::io::Read; use std::time::Duration; use anyhow::{anyhow, Context, Result}; -use chrono::{DateTime, Local}; -use colored::{ColoredString, Colorize}; -use log::Level; use reqwest::{Response, StatusCode}; use reqwest_middleware::{ClientBuilder, ClientWithMiddleware}; use reqwest_retry::policies::ExponentialBackoff; @@ -15,6 +12,8 @@ use shuttle_common::project::ProjectName; use shuttle_common::{ApiKey, ApiUrl, DeploymentMeta, DeploymentStateMeta, SHUTTLE_PROJECT_HEADER}; use tokio::time::sleep; +use crate::print; + pub(crate) async fn auth(api_url: ApiUrl, username: String) -> Result { let client = get_retry_client(); let mut api_url = api_url; @@ -70,37 +69,41 @@ pub(crate) async fn status(api_url: ApiUrl, api_key: &ApiKey, project: &ProjectN Ok(()) } +pub(crate) async fn shuttle_version(mut api_url: ApiUrl) -> Result { + let client = get_retry_client(); + api_url.push_str("/version"); + + let res: Response = client + .get(api_url) + .send() + .await + .context("failed to get version from Shuttle server")?; + + let response_status = res.status(); + + if response_status == StatusCode::OK { + Ok(res.text().await?) + } else { + Err(anyhow!( + "status: {}, body: {}", + response_status, + res.text().await? + )) + } +} + pub(crate) async fn logs(api_url: ApiUrl, api_key: &ApiKey, project: &ProjectName) -> Result<()> { let client = get_retry_client(); let deployment_meta = get_deployment_meta(api_url, api_key, project, &client).await?; - for (datetime, log) in deployment_meta.runtime_logs { - let datetime: DateTime = DateTime::from(datetime); - println!( - "{}{} {:<5} {}{} {}", - "[".bright_black(), - datetime.format("%Y-%m-%dT%H:%M:%SZ"), - get_colored_level(&log.level), - log.target, - "]".bright_black(), - log.body - ); + for (datetime, log_item) in deployment_meta.runtime_logs { + print::log(datetime, log_item); } Ok(()) } -fn get_colored_level(level: &Level) -> ColoredString { - match level { - Level::Trace => level.to_string().bright_black(), - Level::Debug => level.to_string().blue(), - Level::Info => level.to_string().green(), - Level::Warn => level.to_string().yellow(), - Level::Error => level.to_string().red(), - } -} - async fn get_deployment_meta( api_url: ApiUrl, api_key: &ApiKey, diff --git a/cargo-shuttle/src/config.rs b/cargo-shuttle/src/config.rs index 9536ba1ab..f8ec34a45 100644 --- a/cargo-shuttle/src/config.rs +++ b/cargo-shuttle/src/config.rs @@ -152,6 +152,8 @@ pub type SecretsConfig = HashMap; /// /// # Usage /// ```rust,no_run +/// # use cargo_shuttle::config::{Config, GlobalConfig, GlobalConfigManager}; +/// # /// let mut config = Config::new(GlobalConfigManager); /// config.open().unwrap(); /// let content: &GlobalConfig = config.as_ref().unwrap(); @@ -278,6 +280,7 @@ impl RequestContext { let mut secrets = Config::new(secrets_manager); if secrets.exists() { + trace!("found secrets"); secrets.open()?; self.secrets = Some(secrets); } @@ -300,17 +303,26 @@ impl RequestContext { if !project.exists() { project.replace(ProjectConfig::default()); } else { + trace!("found a local Shuttle.toml"); project.open()?; } let config = project.as_mut().unwrap(); match (&project_args.name, &config.name) { // Command-line name parameter trumps everything - (Some(name_from_args), _) => config.name = Some(name_from_args.clone()), + (Some(name_from_args), _) => { + trace!("using command-line project name"); + config.name = Some(name_from_args.clone()); + } // If key exists in config then keep it as it is - (None, Some(_)) => {} + (None, Some(_)) => { + trace!("using Shuttle.toml project name"); + } // If name key is not in project config, then we infer from crate name - (None, None) => config.name = Some(find_crate_name(&project_args.working_directory)?), + (None, None) => { + trace!("using crate name as project name"); + config.name = Some(find_crate_name(&project_args.working_directory)?); + } }; Ok(project) } diff --git a/cargo-shuttle/src/factory.rs b/cargo-shuttle/src/factory.rs new file mode 100644 index 000000000..bfa3fb043 --- /dev/null +++ b/cargo-shuttle/src/factory.rs @@ -0,0 +1,256 @@ +use anyhow::Result; +use async_trait::async_trait; +use bollard::{ + container::{Config, CreateContainerOptions, StartContainerOptions}, + exec::{CreateExecOptions, CreateExecResults}, + image::CreateImageOptions, + models::{CreateImageInfo, HostConfig, PortBinding, ProgressDetail}, + Docker, +}; +use colored::Colorize; +use crossterm::{ + cursor::MoveUp, + terminal::{Clear, ClearType}, + QueueableCommand, +}; +use futures::StreamExt; +use portpicker::pick_unused_port; +use shuttle_common::{project::ProjectName, DatabaseReadyInfo}; +use shuttle_service::{error::CustomError, Factory}; +use std::{collections::HashMap, io::stdout, time::Duration}; +use tokio::time::sleep; + +pub struct LocalFactory { + docker: Docker, + project: ProjectName, +} + +impl LocalFactory { + pub fn new(project: ProjectName) -> Result { + Ok(Self { + docker: Docker::connect_with_local_defaults()?, + project, + }) + } +} + +const PG_PASSWORD: &str = "password"; +const PG_IMAGE: &str = "postgres:11"; + +#[async_trait] +impl Factory for LocalFactory { + async fn get_sql_connection_string(&mut self) -> Result { + trace!("getting sql string for project '{}'", self.project); + let container_name = format!("shuttle_{}_postgres", self.project); + + let container = match self.docker.inspect_container(&container_name, None).await { + Ok(container) => { + trace!("found DB container {container_name}"); + container + } + Err(bollard::errors::Error::DockerResponseServerError { status_code, .. }) + if status_code == 404 => + { + self.pull_image(PG_IMAGE) + .await + .expect("failed to pull image"); + trace!("will create DB container {container_name}"); + let options = Some(CreateContainerOptions { + name: container_name.clone(), + }); + let mut port_bindings = HashMap::new(); + let host_port = pick_unused_port().expect("system to have a free port"); + port_bindings.insert( + "5432/tcp".to_string(), + Some(vec![PortBinding { + host_port: Some(host_port.to_string()), + ..Default::default() + }]), + ); + let host_config = HostConfig { + port_bindings: Some(port_bindings), + ..Default::default() + }; + + let password_env = format!("POSTGRES_PASSWORD={PG_PASSWORD}"); + let config = Config { + image: Some(PG_IMAGE), + env: Some(vec![&password_env]), + host_config: Some(host_config), + ..Default::default() + }; + + self.docker + .create_container(options, config) + .await + .expect("to be able to create container"); + + self.docker + .inspect_container(&container_name, None) + .await + .expect("container to be created") + } + Err(error) => { + error!("got unexpected error while inspecting docker container: {error}"); + return Err(shuttle_service::Error::Custom(CustomError::new(error))); + } + }; + + let port = container + .host_config + .expect("container to have host config") + .port_bindings + .expect("port bindings on container") + .get("5432/tcp") + .expect("a '5432/tcp' port bindings entry") + .as_ref() + .expect("a '5432/tcp' port bindings") + .first() + .expect("at least one port binding") + .host_port + .as_ref() + .expect("a host port") + .clone(); + + if !container + .state + .expect("container to have a state") + .running + .expect("state to have a running key") + { + trace!("DB container '{container_name}' not running, so starting it"); + self.docker + .start_container(&container_name, None::>) + .await + .expect("failed to start none running container"); + } + + self.wait_for_ready(&container_name).await?; + + let db_info = DatabaseReadyInfo::new( + "postgres".to_string(), + PG_PASSWORD.to_string(), + "postgres".to_string(), + port, + ); + + let conn_str = db_info.connection_string("localhost"); + + println!( + "{:>12} can be reached at {}\n", + "DB ready".bold().cyan(), + conn_str + ); + + Ok(conn_str) + } +} + +impl LocalFactory { + async fn wait_for_ready(&self, container_name: &str) -> Result<(), shuttle_service::Error> { + loop { + trace!("waiting for '{container_name}' to be ready for connections"); + + let config = CreateExecOptions { + cmd: Some(vec!["pg_isready"]), + attach_stdout: Some(true), + ..Default::default() + }; + + let CreateExecResults { id } = self + .docker + .create_exec(container_name, config) + .await + .expect("failed to create exec to check if container is ready"); + + let ready_result = self + .docker + .start_exec(&id, None) + .await + .expect("failed to execute ready command"); + + if let bollard::exec::StartExecResults::Attached { mut output, .. } = ready_result { + while let Some(line) = output.next().await { + if let bollard::container::LogOutput::StdOut { message } = + line.expect("output to have a log line") + { + if message.ends_with(b"accepting connections\n") { + return Ok(()); + } + } + } + } + + sleep(Duration::from_millis(500)).await; + } + } + + async fn pull_image(&self, image: &str) -> Result<(), String> { + trace!("pulling latest image for '{image}'"); + let mut layers = Vec::new(); + + let create_image_options = Some(CreateImageOptions { + from_image: image, + ..Default::default() + }); + let mut output = self.docker.create_image(create_image_options, None, None); + + while let Some(line) = output.next().await { + let info = line.expect("failed to create image"); + + if let Some(id) = info.id.as_ref() { + match layers + .iter_mut() + .find(|item: &&mut CreateImageInfo| item.id.as_deref() == Some(id)) + { + Some(item) => *item = info, + None => layers.push(info), + } + } else { + layers.push(info); + } + + print_layers(&layers); + } + + Ok(()) + } +} + +fn print_layers(layers: &Vec) { + for info in layers { + stdout() + .queue(Clear(ClearType::CurrentLine)) + .expect("to be able to clear line"); + + if let Some(id) = info.id.as_ref() { + let text = match (info.status.as_deref(), info.progress_detail.as_ref()) { + ( + Some("Downloading"), + Some(ProgressDetail { + current: Some(c), + total: Some(t), + }), + ) => { + let percent = *c as f64 / *t as f64 * 100.0; + let progress = (percent as i64 / 10) as usize; + let remaining = 10 - progress; + format!("{:={:remaining$} {percent:.0}%", "", "") + } + (Some(status), _) => status.to_string(), + _ => "Unknown".to_string(), + }; + println!("[{id} {}]", text); + } else { + println!( + "{}", + info.status.as_ref().expect("image info to have a status") + ) + } + } + stdout() + .queue(MoveUp( + layers.len().try_into().expect("to convert usize to u16"), + )) + .expect("to reset cursor position"); +} diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs new file mode 100644 index 000000000..acc426c08 --- /dev/null +++ b/cargo-shuttle/src/lib.rs @@ -0,0 +1,341 @@ +mod args; +mod client; +pub mod config; +mod factory; +mod print; + +use std::fs::{read_to_string, File}; +use std::io::Write; +use std::io::{self, stdout}; +use std::net::{Ipv4Addr, SocketAddr}; +use std::rc::Rc; + +use anyhow::{anyhow, Context, Result}; +pub use args::{Args, Command, DeployArgs, InitArgs, ProjectArgs, RunArgs}; +use args::{AuthArgs, LoginArgs}; +use cargo::core::compiler::CompileMode; +use cargo::core::resolver::CliFeatures; +use cargo::core::Workspace; +use cargo::ops::{CompileOptions, NewOptions, PackageOpts, Packages, TestOptions}; +use cargo_edit::{find, get_latest_dependency, registry_url}; +use colored::Colorize; +use config::RequestContext; +use factory::LocalFactory; +use futures::future::TryFutureExt; +use semver::{Version, VersionReq}; +use shuttle_service::loader::{build_crate, Loader}; +use tokio::sync::mpsc; +use toml_edit::{value, Array, Document, Item, Table, Value}; +use uuid::Uuid; + +#[macro_use] +extern crate log; + +pub struct Shuttle { + ctx: RequestContext, +} + +impl Default for Shuttle { + fn default() -> Self { + Self::new() + } +} + +impl Shuttle { + pub fn new() -> Self { + let ctx = RequestContext::load_global().unwrap(); + Self { ctx } + } + + pub async fn run(mut self, args: Args) -> Result<()> { + trace!("running local client"); + if matches!( + args.cmd, + Command::Deploy(..) + | Command::Delete + | Command::Status + | Command::Logs + | Command::Run(..) + ) { + self.load_project(&args.project_args)?; + } + + self.ctx.set_api_url(args.api_url); + + match args.cmd { + Command::Deploy(deploy_args) => { + self.check_lib_version(args.project_args).await?; + self.deploy(deploy_args).await + } + Command::Init(init_args) => self.init(init_args).await, + Command::Status => self.status().await, + Command::Logs => self.logs().await, + Command::Delete => self.delete().await, + Command::Auth(auth_args) => self.auth(auth_args).await, + Command::Login(login_args) => self.login(login_args).await, + Command::Run(run_args) => self.local_run(run_args).await, + } + } + + async fn init(&self, args: InitArgs) -> Result<()> { + // Interface with cargo to initialize new lib package for shuttle + let opts = NewOptions::new(None, false, true, args.path.clone(), None, None, None)?; + let cargo_config = cargo::util::config::Config::default()?; + let init_result = cargo::ops::init(&opts, &cargo_config)?; + // Mimick `cargo init` behavior and log status or error to shell + cargo_config + .shell() + .status("Created", format!("{} (shuttle) package", init_result))?; + + // Read Cargo.toml into a `Document` + let cargo_path = args.path.join("Cargo.toml"); + let mut cargo_doc = read_to_string(cargo_path.clone())?.parse::()?; + + // Remove empty dependencies table to re-insert after the lib table is inserted + cargo_doc.remove("dependencies"); + + // Insert `crate-type = ["cdylib"]` array into `[lib]` table + let crate_type_array = Array::from_iter(["cdylib"].into_iter()); + let mut lib_table = Table::new(); + lib_table["crate-type"] = Item::Value(Value::Array(crate_type_array)); + cargo_doc["lib"] = Item::Table(lib_table); + + // Fetch the latest shuttle-service version from crates.io + let manifest_path = find(Some(&args.path)).unwrap(); + let url = registry_url(manifest_path.as_path(), None).expect("Could not find registry URL"); + let latest_shuttle_service = + get_latest_dependency("shuttle-service", false, &manifest_path, Some(&url)) + .expect("Could not query the latest version of shuttle-service"); + let shuttle_version = latest_shuttle_service + .version() + .expect("No latest shuttle-service version available"); + + // Insert shuttle-service to `[dependencies]` table + let mut dep_table = Table::new(); + dep_table["shuttle-service"]["version"] = value(shuttle_version); + cargo_doc["dependencies"] = Item::Table(dep_table); + + // Truncate Cargo.toml and write the updated `Document` to it + let mut cargo_toml = File::create(cargo_path)?; + cargo_toml.write_all(cargo_doc.to_string().as_bytes())?; + + Ok(()) + } + + pub fn load_project(&mut self, project_args: &ProjectArgs) -> Result<()> { + trace!("loading project arguments: {project_args:?}"); + self.ctx.load_local(project_args) + } + + async fn login(&mut self, login_args: LoginArgs) -> Result<()> { + let api_key_str = login_args.api_key.unwrap_or_else(|| { + let url = "https://shuttle.rs/login"; + + let _ = webbrowser::open(url); + + println!("If your browser did not automatically open, go to {url}"); + print!("Enter Api Key: "); + + io::stdout().flush().unwrap(); + + let mut input = String::new(); + + io::stdin().read_line(&mut input).unwrap(); + + input + }); + + let api_key = api_key_str.trim().parse()?; + + self.ctx.set_api_key(api_key)?; + + Ok(()) + } + + async fn auth(&mut self, auth_args: AuthArgs) -> Result<()> { + let api_key = client::auth(self.ctx.api_url(), auth_args.username) + .await + .context("failed to retrieve api key")?; + self.ctx.set_api_key(api_key)?; + Ok(()) + } + + async fn delete(&self) -> Result<()> { + client::delete( + self.ctx.api_url(), + self.ctx.api_key()?, + self.ctx.project_name(), + ) + .await + .context("failed to delete deployment") + } + + async fn status(&self) -> Result<()> { + client::status( + self.ctx.api_url(), + self.ctx.api_key()?, + self.ctx.project_name(), + ) + .await + .context("failed to get status of deployment") + } + + async fn logs(&self) -> Result<()> { + client::logs( + self.ctx.api_url(), + self.ctx.api_key()?, + self.ctx.project_name(), + ) + .await + .context("failed to get logs of deployment") + } + + async fn local_run(&self, run_args: RunArgs) -> Result<()> { + trace!("starting a local run for a service: {run_args:?}"); + + let buf = Box::new(stdout()); + let working_directory = self.ctx.working_directory(); + + trace!("building project"); + println!( + "{:>12} {}", + "Building".bold().green(), + working_directory.display() + ); + let so_path = build_crate(working_directory, buf)?; + let loader = Loader::from_so_file(so_path)?; + + let mut factory = LocalFactory::new(self.ctx.project_name().clone())?; + let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), run_args.port); + let deployment_id = Uuid::new_v4(); + let (tx, mut rx) = mpsc::unbounded_channel(); + + trace!("loading project"); + println!( + "\n{:>12} {} on http://{}", + "Starting".bold().green(), + self.ctx.project_name(), + addr + ); + let (handle, so) = loader.load(&mut factory, addr, tx, deployment_id).await?; + + tokio::spawn(async move { + while let Some(log) = rx.recv().await { + print::log(log.datetime, log.item); + } + }); + + handle.await??; + + tokio::spawn(async move { + trace!("closing so file"); + so.close().unwrap(); + }); + + Ok(()) + } + + async fn deploy(&self, args: DeployArgs) -> Result<()> { + self.run_tests(args.no_test)?; + + let package_file = self + .run_cargo_package(args.allow_dirty) + .context("failed to package cargo project")?; + + let key = self.ctx.api_key()?; + + client::deploy( + package_file, + self.ctx.api_url(), + key, + self.ctx.project_name(), + ) + .and_then(|_| { + client::secrets( + self.ctx.api_url(), + key, + self.ctx.project_name(), + self.ctx.secrets(), + ) + }) + .await + .context("failed to deploy cargo project") + } + + async fn check_lib_version(&self, project_args: ProjectArgs) -> Result<()> { + let cargo_path = project_args.working_directory.join("Cargo.toml"); + let cargo_doc = read_to_string(cargo_path.clone())?.parse::()?; + let current_shuttle_version = &cargo_doc["dependencies"]["shuttle-service"]["version"]; + let service_semver = Version::parse(current_shuttle_version.as_str().unwrap())?; + let server_version = client::shuttle_version(self.ctx.api_url()).await?; + let server_version = Version::parse(&server_version)?; + + let version_required = format!("{}.{}", server_version.major, server_version.minor); + let server_semver = VersionReq::parse(&version_required)?; + + if server_semver.matches(&service_semver) { + Ok(()) + } else { + Err(anyhow!( + "Your shuttle_service version is outdated. Update your shuttle_service version to {} and try to deploy again", + &server_version, + )) + } + } + + // Packages the cargo project and returns a File to that file + fn run_cargo_package(&self, allow_dirty: bool) -> Result { + let config = cargo::util::config::Config::default()?; + + let working_directory = self.ctx.working_directory(); + let path = working_directory.join("Cargo.toml"); + + let ws = Workspace::new(&path, &config)?; + let opts = PackageOpts { + config: &config, + list: false, + check_metadata: true, + allow_dirty, + keep_going: false, + verify: false, + jobs: None, + to_package: Packages::Default, + targets: vec![], + cli_features: CliFeatures { + features: Rc::new(Default::default()), + all_features: false, + uses_default_features: true, + }, + }; + + let locks = cargo::ops::package(&ws, &opts)?.expect("unwrap ok here"); + let owned = locks.get(0).unwrap().file().try_clone()?; + Ok(owned) + } + + fn run_tests(&self, no_test: bool) -> Result<()> { + if no_test { + return Ok(()); + } + + let config = cargo::util::config::Config::default()?; + let working_directory = self.ctx.working_directory(); + let path = working_directory.join("Cargo.toml"); + + let compile_options = CompileOptions::new(&config, CompileMode::Test).unwrap(); + let ws = Workspace::new(&path, &config)?; + let opts = TestOptions { + compile_opts: compile_options, + no_run: false, + no_fail_fast: false, + }; + + let test_failures = cargo::ops::run_tests(&ws, &opts, &[])?; + match test_failures { + None => Ok(()), + Some(_) => Err(anyhow!( + "Some tests failed. To ignore all tests, pass the `--no-test` flag" + )), + } + } +} diff --git a/cargo-shuttle/src/main.rs b/cargo-shuttle/src/main.rs index 3d1d1dcaf..bf323a71c 100644 --- a/cargo-shuttle/src/main.rs +++ b/cargo-shuttle/src/main.rs @@ -1,182 +1,9 @@ -mod args; -mod client; -mod config; - -use std::fs::File; -use std::io; -use std::io::Write; -use std::rc::Rc; - -use anyhow::{Context, Result}; -use args::{LoginArgs, ProjectArgs}; -use cargo::core::resolver::CliFeatures; -use cargo::core::Workspace; -use cargo::ops::{PackageOpts, Packages}; -use futures::future::TryFutureExt; +use anyhow::Result; +use cargo_shuttle::{Args, Shuttle}; use structopt::StructOpt; -use crate::args::{Args, AuthArgs, Command, DeployArgs}; -use crate::config::RequestContext; - #[tokio::main] async fn main() -> Result<()> { + env_logger::init(); Shuttle::new().run(Args::from_args()).await } - -pub struct Shuttle { - ctx: RequestContext, -} - -impl Default for Shuttle { - fn default() -> Self { - Self::new() - } -} - -impl Shuttle { - pub fn new() -> Self { - let ctx = RequestContext::load_global().unwrap(); - Self { ctx } - } - - pub async fn run(mut self, args: Args) -> Result<()> { - if matches!( - args.cmd, - Command::Deploy(..) | Command::Delete | Command::Status | Command::Logs - ) { - self.load_project(&args.project_args)?; - } - - self.ctx.set_api_url(args.api_url); - - match args.cmd { - Command::Deploy(deploy_args) => self.deploy(deploy_args).await, - Command::Status => self.status().await, - Command::Logs => self.logs().await, - Command::Delete => self.delete().await, - Command::Auth(auth_args) => self.auth(auth_args).await, - Command::Login(login_args) => self.login(login_args).await, - } - } - - pub fn load_project(&mut self, project_args: &ProjectArgs) -> Result<()> { - self.ctx.load_local(project_args) - } - - async fn login(&mut self, login_args: LoginArgs) -> Result<()> { - let api_key_str = login_args.api_key.unwrap_or_else(|| { - let url = "https://shuttle.rs/login"; - - let _ = webbrowser::open(url); - - println!("If your browser did not automatically open, go to {url}"); - print!("Enter Api Key: "); - - io::stdout().flush().unwrap(); - - let mut input = String::new(); - - io::stdin().read_line(&mut input).unwrap(); - - input - }); - - let api_key = api_key_str.trim().parse()?; - - self.ctx.set_api_key(api_key)?; - - Ok(()) - } - - async fn auth(&mut self, auth_args: AuthArgs) -> Result<()> { - let api_key = client::auth(self.ctx.api_url(), auth_args.username) - .await - .context("failed to retrieve api key")?; - self.ctx.set_api_key(api_key)?; - Ok(()) - } - - async fn delete(&self) -> Result<()> { - client::delete( - self.ctx.api_url(), - self.ctx.api_key()?, - self.ctx.project_name(), - ) - .await - .context("failed to delete deployment") - } - - async fn status(&self) -> Result<()> { - client::status( - self.ctx.api_url(), - self.ctx.api_key()?, - self.ctx.project_name(), - ) - .await - .context("failed to get status of deployment") - } - - async fn logs(&self) -> Result<()> { - client::logs( - self.ctx.api_url(), - self.ctx.api_key()?, - self.ctx.project_name(), - ) - .await - .context("failed to get logs of deployment") - } - - async fn deploy(&self, args: DeployArgs) -> Result<()> { - let package_file = self - .run_cargo_package(args.allow_dirty) - .context("failed to package cargo project")?; - - let key = self.ctx.api_key()?; - - client::deploy( - package_file, - self.ctx.api_url(), - key, - self.ctx.project_name(), - ) - .and_then(|_| { - client::secrets( - self.ctx.api_url(), - key, - self.ctx.project_name(), - self.ctx.secrets(), - ) - }) - .await - .context("failed to deploy cargo project") - } - - // Packages the cargo project and returns a File to that file - fn run_cargo_package(&self, allow_dirty: bool) -> Result { - let config = cargo::util::config::Config::default()?; - - let working_directory = self.ctx.working_directory(); - let path = working_directory.join("Cargo.toml"); - - let ws = Workspace::new(&path, &config)?; - let opts = PackageOpts { - config: &config, - list: false, - check_metadata: true, - allow_dirty, - verify: false, - jobs: None, - to_package: Packages::Default, - targets: vec![], - cli_features: CliFeatures { - features: Rc::new(Default::default()), - all_features: false, - uses_default_features: true, - }, - }; - - let locks = cargo::ops::package(&ws, &opts)?.expect("unwrap ok here"); - let owned = locks.get(0).unwrap().file().try_clone()?; - Ok(owned) - } -} diff --git a/cargo-shuttle/src/print.rs b/cargo-shuttle/src/print.rs new file mode 100644 index 000000000..246dbf599 --- /dev/null +++ b/cargo-shuttle/src/print.rs @@ -0,0 +1,27 @@ +use chrono::{DateTime, Local, Utc}; +use colored::{ColoredString, Colorize}; +use log::Level; +use shuttle_common::LogItem; + +pub fn log(datetime: DateTime, log_item: LogItem) { + let datetime: DateTime = DateTime::from(datetime); + println!( + "{}{} {:<5} {}{} {}", + "[".bright_black(), + datetime.format("%Y-%m-%dT%H:%M:%SZ"), + get_colored_level(&log_item.level), + log_item.target, + "]".bright_black(), + log_item.body + ); +} + +fn get_colored_level(level: &Level) -> ColoredString { + match level { + Level::Trace => level.to_string().bright_black(), + Level::Debug => level.to_string().blue(), + Level::Info => level.to_string().green(), + Level::Warn => level.to_string().yellow(), + Level::Error => level.to_string().red(), + } +} diff --git a/cargo-shuttle/tests/.gitignore b/cargo-shuttle/tests/.gitignore new file mode 100644 index 000000000..3fec32c84 --- /dev/null +++ b/cargo-shuttle/tests/.gitignore @@ -0,0 +1 @@ +tmp/ diff --git a/cargo-shuttle/tests/integration/README.md b/cargo-shuttle/tests/integration/README.md index 546903892..933e4213f 100644 --- a/cargo-shuttle/tests/integration/README.md +++ b/cargo-shuttle/tests/integration/README.md @@ -1,5 +1,3 @@ # Integration Tests for `cargo-shuttle` Integration tests are organised following [matklad's recommedations](https://matklad.github.io/2021/02/27/delete-cargo-integration-tests.html). - -Initially, everything is tested via [assert_cmd](https://docs.rs/assert_cmd/latest/assert_cmd/cmd/struct.Command.html), but it might make sense to split `cargo-shuttle` into a bin+lib crate, to test the internals more easily. diff --git a/cargo-shuttle/tests/integration/deploy.rs b/cargo-shuttle/tests/integration/deploy.rs new file mode 100644 index 000000000..6cb13472a --- /dev/null +++ b/cargo-shuttle/tests/integration/deploy.rs @@ -0,0 +1,60 @@ +use std::path::Path; + +use cargo_shuttle::{Args, Command, DeployArgs, ProjectArgs, Shuttle}; +use futures::Future; +use reqwest::StatusCode; +use test_context::test_context; +use tokiotest_httpserver::{handler::HandlerBuilder, HttpTestContext}; + +/// creates a `cargo-shuttle` deploy instance with some reasonable defaults set. +fn cargo_shuttle_deploy(path: &str, api_url: String) -> impl Future> { + let working_directory = Path::new(path).to_path_buf(); + + Shuttle::new().run(Args { + api_url: Some(api_url), + project_args: ProjectArgs { + working_directory, + name: None, + }, + cmd: Command::Deploy(DeployArgs { + allow_dirty: false, + no_test: false, + }), + }) +} + +#[should_panic( + expected = "Your shuttle_service version is outdated. Update your shuttle_service version to 1.2.5 and try to deploy again" +)] +#[test_context(HttpTestContext)] +#[tokio::test] +async fn deploy_when_version_is_outdated(ctx: &mut HttpTestContext) { + ctx.add( + HandlerBuilder::new("/test/version") + .status_code(StatusCode::OK) + .response("1.2.5".into()) + .build(), + ); + let api_url = ctx.uri("/test").to_string(); + + cargo_shuttle_deploy("../examples/rocket/hello-world", api_url) + .await + .unwrap(); +} + +#[should_panic(expected = "not an absolute path")] +#[test_context(HttpTestContext)] +#[tokio::test] +async fn deploy_when_version_is_valid(ctx: &mut HttpTestContext) { + ctx.add( + HandlerBuilder::new("/test/version") + .status_code(StatusCode::OK) + .response(shuttle_service::VERSION.into()) + .build(), + ); + let api_url = ctx.uri("/test").to_string(); + + cargo_shuttle_deploy("../examples/rocket/hello-world", api_url) + .await + .unwrap(); +} diff --git a/cargo-shuttle/tests/integration/init.rs b/cargo-shuttle/tests/integration/init.rs new file mode 100644 index 000000000..2ae24a4dd --- /dev/null +++ b/cargo-shuttle/tests/integration/init.rs @@ -0,0 +1,35 @@ +use std::{ + fs::{read_to_string, remove_dir_all}, + path::Path, +}; + +use cargo_shuttle::{Args, Command, InitArgs, ProjectArgs, Shuttle}; +use futures::Future; + +/// creates a `cargo-shuttle` init instance with some reasonable defaults set. +fn cargo_shuttle_init(path: &str) -> impl Future> { + let _result = remove_dir_all(path); + + let working_directory = Path::new(".").to_path_buf(); + let path = Path::new(path).to_path_buf(); + + Shuttle::new().run(Args { + api_url: Some("network support is intentionally broken in tests".to_string()), + project_args: ProjectArgs { + working_directory, + name: None, + }, + cmd: Command::Init(InitArgs { path }), + }) +} + +#[tokio::test] +async fn basic_init() { + cargo_shuttle_init("tests/tmp/basic-init").await.unwrap(); + + let cargo_toml = read_to_string("tests/tmp/basic-init/Cargo.toml").unwrap(); + + assert!(cargo_toml.contains("name = \"basic-init\"")); + assert!(cargo_toml.contains("[lib]\ncrate-type = [\"cdylib\"]")); + assert!(cargo_toml.contains("shuttle-service = { version = ")); +} diff --git a/cargo-shuttle/tests/integration/main.rs b/cargo-shuttle/tests/integration/main.rs index cdd20c971..892149952 100644 --- a/cargo-shuttle/tests/integration/main.rs +++ b/cargo-shuttle/tests/integration/main.rs @@ -1,56 +1,43 @@ -use assert_cmd::Command; +mod deploy; +mod init; +mod run; -/// creates a `cargo-shuttle` Command instance with some reasonable defaults set. -fn cargo_shuttle_command() -> Command { - let mut cmd = Command::cargo_bin("cargo-shuttle").unwrap(); - cmd.env( - "SHUTTLE_API", - "network support is intentionally broken in tests", - ); - cmd -} +use cargo_shuttle::{Args, Command, ProjectArgs, Shuttle}; +use std::{future::Future, path::Path}; -#[test] -fn default_prints_usage_to_stderr() { - let mut cmd = cargo_shuttle_command(); - cmd.assert() - .stderr(predicates::str::is_match("^cargo-shuttle.*\n\nUSAGE:").unwrap()); -} +/// creates a `cargo-shuttle` run instance with some reasonable defaults set. +fn cargo_shuttle_command( + cmd: Command, + working_directory: &str, +) -> impl Future> { + let working_directory = Path::new(working_directory).to_path_buf(); -#[test] -fn help_prints_usage_to_stdout() { - let mut cmd = cargo_shuttle_command(); - cmd.arg("--help") - .assert() - .stdout(predicates::str::is_match("^cargo-shuttle.*\n\nUSAGE:").unwrap()); + Shuttle::new().run(Args { + api_url: Some("network support is intentionally broken in tests".to_string()), + project_args: ProjectArgs { + working_directory, + name: None, + }, + cmd, + }) } -#[test] -fn network_support_is_intentionally_broken_in_tests() { - let mut cmd = cargo_shuttle_command(); - cmd.arg("status").assert().stderr(predicates::str::contains( - "builder error: relative URL without a base", - )); +#[tokio::test] +#[should_panic(expected = "builder error: relative URL without a base")] +async fn network_support_is_intentionally_broken_in_tests() { + cargo_shuttle_command(Command::Status, ".").await.unwrap(); } -#[test] -fn fails_if_working_directory_does_not_exist() { - let mut cmd = cargo_shuttle_command(); - cmd.arg("status") - .arg("--working-directory=/path_that_does_not_exist") - .assert() - .stderr( - predicates::str::contains(r#"error: Invalid value for '--working-directory ': could not turn "/path_that_does_not_exist" into a real path: No such file or directory (os error 2)"#), - ); +#[tokio::test] +#[should_panic(expected = "No such file or directory")] +async fn fails_if_working_directory_does_not_exist() { + cargo_shuttle_command(Command::Status, "/path_that_does_not_exist") + .await + .unwrap(); } -#[test] -fn fails_if_working_directory_not_part_of_cargo_workspace() { - let mut cmd = cargo_shuttle_command(); - cmd.arg("status") - .arg("--working-directory=/") - .assert() - .stderr(predicates::str::contains( - r#"error: could not find `Cargo.toml` in `/` or any parent directory"#, - )); +#[tokio::test] +#[should_panic(expected = "error: could not find `Cargo.toml` in `/` or any parent directory")] +async fn fails_if_working_directory_not_part_of_cargo_workspace() { + cargo_shuttle_command(Command::Status, "/").await.unwrap(); } diff --git a/cargo-shuttle/tests/integration/run.rs b/cargo-shuttle/tests/integration/run.rs new file mode 100644 index 000000000..ef0a96dde --- /dev/null +++ b/cargo-shuttle/tests/integration/run.rs @@ -0,0 +1,199 @@ +use cargo_shuttle::{Args, Command, ProjectArgs, RunArgs, Shuttle}; +use portpicker::pick_unused_port; +use reqwest::StatusCode; +use std::{fs::canonicalize, process::exit, time::Duration}; +use tokio::time::sleep; + +/// creates a `cargo-shuttle` run instance with some reasonable defaults set. +async fn cargo_shuttle_run(working_directory: &str) -> u16 { + let _ = env_logger::builder() + .filter_module("cargo_shuttle", log::LevelFilter::Trace) + .is_test(true) + .try_init(); + let working_directory = canonicalize(working_directory).unwrap(); + let port = pick_unused_port().unwrap(); + let run_args = RunArgs { port }; + + let runner = Shuttle::new().run(Args { + api_url: Some("network support is intentionally broken in tests".to_string()), + project_args: ProjectArgs { + working_directory: working_directory.clone(), + name: None, + }, + cmd: Command::Run(run_args), + }); + + tokio::spawn(async move { + sleep(Duration::from_secs(180)).await; + + println!( + "run test for '{}' took too long. Did it fail to shutdown?", + working_directory.display() + ); + exit(1); + }); + + tokio::spawn(runner); + + // Wait for service to be responsive + while (reqwest::Client::new() + .get(format!("http://localhost:{port}")) + .send() + .await) + .is_err() + { + sleep(Duration::from_millis(350)).await; + } + + port +} + +#[tokio::test] +async fn rocket_hello_world() { + let port = cargo_shuttle_run("../examples/rocket/hello-world").await; + + let request_text = reqwest::Client::new() + .get(format!("http://localhost:{port}/hello")) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!(request_text, "Hello, world!"); +} + +// This example uses a shared Postgres. Thus local runs should create a docker container for it. +#[tokio::test] +async fn rocket_postgres() { + let port = cargo_shuttle_run("../examples/rocket/postgres").await; + let client = reqwest::Client::new(); + + let post_text = client + .post(format!("http://localhost:{port}/todo")) + .body("{\"note\": \"Deploy to shuttle\"}") + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!(post_text, "{\"id\":1,\"note\":\"Deploy to shuttle\"}"); + + let request_text = client + .get(format!("http://localhost:{port}/todo/1")) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!(request_text, "{\"id\":1,\"note\":\"Deploy to shuttle\"}"); +} + +#[tokio::test] +async fn rocket_authentication() { + let port = cargo_shuttle_run("../examples/rocket/authentication").await; + let client = reqwest::Client::new(); + + let public_text = client + .get(format!("http://localhost:{port}/public")) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!( + public_text, + "{\"message\":\"This endpoint is open to anyone\"}" + ); + + let private_status = client + .get(format!("http://localhost:{port}/private")) + .send() + .await + .unwrap() + .status(); + + assert_eq!(private_status, StatusCode::FORBIDDEN); + + let body = client + .post(format!("http://localhost:{port}/login")) + .body("{\"username\": \"username\", \"password\": \"password\"}") + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_str(&body).unwrap(); + let token = format!("Bearer {}", json["token"].as_str().unwrap()); + + let private_text = client + .get(format!("http://localhost:{port}/private")) + .header("Authorization", token) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!( + private_text, + "{\"message\":\"The `Claims` request guard ensures only valid JWTs can access this endpoint\",\"user\":\"username\"}" + ); +} + +#[tokio::test] +async fn axum_hello_world() { + let port = cargo_shuttle_run("../examples/axum/hello-world").await; + + let request_text = reqwest::Client::new() + .get(format!("http://localhost:{port}/hello")) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!(request_text, "Hello, world!"); +} + +#[tokio::test] +async fn tide_hello_world() { + let port = cargo_shuttle_run("../examples/tide/hello-world").await; + + let request_text = reqwest::Client::new() + .get(format!("http://localhost:{port}/hello")) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!(request_text, "Hello, world!"); +} + +#[tokio::test] +async fn tower_hello_world() { + let port = cargo_shuttle_run("../examples/tower/hello-world").await; + + let request_text = reqwest::Client::new() + .get(format!("http://localhost:{port}/hello")) + .send() + .await + .unwrap() + .text() + .await + .unwrap(); + + assert_eq!(request_text, "Hello, world!"); +} diff --git a/codegen/Cargo.toml b/codegen/Cargo.toml index 7858b1609..f4643ab72 100644 --- a/codegen/Cargo.toml +++ b/codegen/Cargo.toml @@ -1,19 +1,18 @@ [package] name = "shuttle-codegen" -version = "0.3.0" +version = "0.3.1" edition = "2021" license = "Apache-2.0" description = "Proc-macro code generator for the shuttle.rs service" - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] proc-macro = true [dependencies] -proc-macro2 = "1.0.36" -quote = "1.0.17" -syn = { version = "1.0.90", features = ["full"] } +proc-macro2 = "1.0.39" +quote = "1.0.18" +syn = { version = "1.0.96", features = ["full"] } [dev-dependencies] -pretty_assertions = "1.2" +pretty_assertions = "1.2.1" diff --git a/codegen/src/lib.rs b/codegen/src/lib.rs index dcf0471f6..b8036dc49 100644 --- a/codegen/src/lib.rs +++ b/codegen/src/lib.rs @@ -18,7 +18,7 @@ pub fn main(_attr: TokenStream, item: TokenStream) -> TokenStream { let constructor: for <'a> fn( &'a mut dyn shuttle_service::Factory, &'a shuttle_service::Runtime, - shuttle_service::Logger, + Box, ) -> std::pin::Pin< Box> + Send + 'a>, > = |factory, runtime, logger| Box::pin(__shuttle_wrapper(factory, runtime, logger)); @@ -87,12 +87,12 @@ impl ToTokens for Wrapper { async fn __shuttle_wrapper( #factory_ident: &mut dyn shuttle_service::Factory, runtime: &shuttle_service::Runtime, - logger: shuttle_service::Logger, + logger: Box, ) #fn_output { #extra_imports runtime.spawn_blocking(move || { - shuttle_service::log::set_boxed_logger(Box::new(logger)) + shuttle_service::log::set_boxed_logger(logger) .map(|()| shuttle_service::log::set_max_level(shuttle_service::log::LevelFilter::Info)) .expect("logger set should succeed"); }).await.unwrap(); @@ -143,10 +143,10 @@ mod tests { async fn __shuttle_wrapper( _factory: &mut dyn shuttle_service::Factory, runtime: &shuttle_service::Runtime, - logger: shuttle_service::Logger, + logger: Box, ) { runtime.spawn_blocking(move || { - shuttle_service::log::set_boxed_logger(Box::new(logger)) + shuttle_service::log::set_boxed_logger(logger) .map(|()| shuttle_service::log::set_max_level(shuttle_service::log::LevelFilter::Info)) .expect("logger set should succeed"); }).await.unwrap(); @@ -186,10 +186,10 @@ mod tests { async fn __shuttle_wrapper( _factory: &mut dyn shuttle_service::Factory, runtime: &shuttle_service::Runtime, - logger: shuttle_service::Logger, + logger: Box, ) -> Result<(), Box > { runtime.spawn_blocking(move || { - shuttle_service::log::set_boxed_logger(Box::new(logger)) + shuttle_service::log::set_boxed_logger(logger) .map(|()| shuttle_service::log::set_max_level(shuttle_service::log::LevelFilter::Info)) .expect("logger set should succeed"); }).await.unwrap(); @@ -230,12 +230,12 @@ mod tests { async fn __shuttle_wrapper( factory: &mut dyn shuttle_service::Factory, runtime: &shuttle_service::Runtime, - logger: shuttle_service::Logger, + logger: Box, ) -> Result<(), Box > { use shuttle_service::GetResource; runtime.spawn_blocking(move || { - shuttle_service::log::set_boxed_logger(Box::new(logger)) + shuttle_service::log::set_boxed_logger(logger) .map(|()| shuttle_service::log::set_max_level(shuttle_service::log::LevelFilter::Info)) .expect("logger set should succeed"); }).await.unwrap(); diff --git a/common/Cargo.toml b/common/Cargo.toml index 1003ab06d..0a4b04a26 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -1,18 +1,17 @@ [package] name = "shuttle-common" -version = "0.3.0" +version = "0.3.1" edition = "2021" license = "Apache-2.0" description = "Common library for the shuttle platform (https://www.shuttle.rs/)" - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -serde = "1.0.136" -serde_json = "1.0.79" -uuid = { version = "0.8.2", features = ["v4", "serde"] } -rocket = "0.5.0-rc.1" -anyhow = "1.0.55" +anyhow = "1.0.57" +chrono = { version = "0.4.19", features = ["serde"] } lazy_static = "1.4.0" -log = { version = "0.4", features = ["serde"] } -chrono = { version = "0.4", features = ["serde"] } +log = { version = "0.4.17", features = ["serde"] } +rocket = "0.5.0-rc.2" +serde = "1.0.137" +serde_json = "1.0.81" +uuid = { version = "1.1.1", features = ["v4", "serde"] } diff --git a/common/src/lib.rs b/common/src/lib.rs index c907b1f26..091ed3688 100644 --- a/common/src/lib.rs +++ b/common/src/lib.rs @@ -94,7 +94,7 @@ impl Display for DeploymentMeta { Project: {} Deployment Id: {} Deployment Status: {} - Host: {} + Host: https://{} Created At: {}{} "#, self.project, self.id, self.state, self.host, self.created_at, db @@ -107,20 +107,27 @@ pub struct DatabaseReadyInfo { pub role_name: String, pub role_password: String, pub database_name: String, + pub port: String, } impl DatabaseReadyInfo { - pub fn new(role_name: String, role_password: String, database_name: String) -> Self { + pub fn new( + role_name: String, + role_password: String, + database_name: String, + port: String, + ) -> Self { Self { role_name, role_password, database_name, + port, } } pub fn connection_string(&self, ip: &str) -> String { format!( - "postgres://{}:{}@{}/{}", - self.role_name, self.role_password, ip, self.database_name + "postgres://{}:{}@{}:{}/{}", + self.role_name, self.role_password, ip, self.port, self.database_name ) } } diff --git a/docker-compose.yml b/docker-compose.yml index c45cb9f28..d77f6fa6b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,7 +3,7 @@ services: app: build: context: . - dockerfile: Dockerfile.dev + dockerfile: api/Containerfile.dev ports: - "8001:8001" - "8000:8000" @@ -14,3 +14,10 @@ services: - SHUTTLE_USERS_TOML=/config/users.toml - SHUTTLE_ADMIN_SECRET=admin-pass - PROXY_FQDN=teste.rs + - PROVISIONER_ADDRESS=provisioner + provisioner: + build: + context: . + dockerfile: provisioner/Containerfile.dev + environment: + - PORT=5001 \ No newline at end of file diff --git a/e2e/Cargo.toml b/e2e/Cargo.toml index 3416d2f40..5b9131ae7 100644 --- a/e2e/Cargo.toml +++ b/e2e/Cargo.toml @@ -2,11 +2,10 @@ name = "e2e" version = "0.1.0" edition = "2021" - # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dev-dependencies] -reqwest = { version = "0.11.9", features = ["blocking", "json"] } +colored = "2.0.0" portpicker = "0.1.1" -colored = "2" -rand = "0.8" +rand = "0.8.5" +reqwest = { version = "0.11.10", features = ["blocking", "json"] } diff --git a/e2e/README.md b/e2e/README.md index 9969120dc..be8499f37 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -1,17 +1,17 @@ # Overview -This project is meant to run all the end-to-end tests for unveil. Here are some notes to help you in your testing +This project is meant to run all the end-to-end tests for shuttle. Here are some notes to help you in your testing journey. -## Making changes to unveil-service -The examples pull `unveil-service` from crates.io. Therefore, any changes made to `unveil-service` will not be detected +## Making changes to shuttle-service +The examples pull `shuttle-service` from crates.io. Therefore, any changes made to `shuttle-service` will not be detected until they are published to crates.io. A way around this is to use the [`[patch]`](https://doc.rust-lang.org/cargo/reference/overriding-dependencies.html#the-patch-section) section in -`Cargo.toml` to use the changed `unveil-service` instead. Create a `.cargo/config.toml` in your +`Cargo.toml` to use the changed `shuttle-service` instead. Create a `.cargo/config.toml` in your [config folder](https://doc.rust-lang.org/cargo/reference/config.html) with the following content. ``` toml [patch.crates-io] -unveil-service = { path = "[base]/unveil/service" } +shuttle-service = { path = "[base]/shuttle/service" } ``` -Now the tests will run against the changes made in `unveil-service`. \ No newline at end of file +Now the tests will run against the changes made in `shuttle-service`. diff --git a/e2e/tests/axum.rs b/e2e/tests/axum.rs index 931db0423..faa90a665 100644 --- a/e2e/tests/axum.rs +++ b/e2e/tests/axum.rs @@ -4,7 +4,7 @@ mod helpers; #[test] fn hello_world() { - let client = helpers::Api::new_docker("hello-world", Color::Cyan); + let client = helpers::Services::new_docker("hello-world", Color::Cyan); client.deploy("../examples/axum/hello-world"); let request_text = client diff --git a/e2e/tests/helpers/mod.rs b/e2e/tests/helpers/mod.rs index 7caf92998..2ed41c8a4 100644 --- a/e2e/tests/helpers/mod.rs +++ b/e2e/tests/helpers/mod.rs @@ -27,12 +27,14 @@ impl EnsureSuccess for io::Result { } } -pub struct Api { +pub struct Services { id: String, api_addr: SocketAddr, proxy_addr: SocketAddr, - image: Option, - container: Option, + api_image: Option, + api_container: Option, + provisioner_image: Option, + provisioner_container: Option, target: String, color: Color, } @@ -87,7 +89,7 @@ pub fn spawn_and_log>( child } -impl Api { +impl Services { fn new_free(target: D, color: C) -> Self where D: std::fmt::Display, @@ -113,8 +115,10 @@ impl Api { id, api_addr, proxy_addr, - image: None, - container: None, + api_image: None, + api_container: None, + provisioner_image: None, + provisioner_container: None, target: target.to_string(), color: color.into(), } @@ -128,27 +132,48 @@ impl Api { let mut api = Self::new_free(target, color); let users_toml_file = format!("{}/users.toml", env!("CARGO_MANIFEST_DIR")); - let api_target = format!(" {} api", api.target); - let image = format!("unveil_{}_{}", api.target, api.id); + // Make sure network is up + Command::new("docker") + .args(["network", "create", "--driver", "bridge", "shuttle-net"]) + .spawn() + .unwrap() + .wait() + .unwrap(); - let mut build = Command::new("docker"); + let provisioner_image = Self::build_image("provisioner", &api.target, &api.id); + api.provisioner_image = Some(provisioner_image.clone()); - build - .args(["build", "-f", "./Dockerfile.dev", "-t", &image, "."]) - .current_dir("../"); + let provisioner_target = format!("{} provisioner", api.target); + let provisioner_container = format!("shuttle_provisioner_{}_{}", api.target, api.id); + let mut run = Command::new("docker"); + run.args([ + "run", + "--name", + &provisioner_container, + "--network", + "shuttle-net", + "-e", + "PORT=5001", + &provisioner_image, + ]); + api.provisioner_container = Some(provisioner_container.clone()); - spawn_and_log(&mut build, api_target.as_str(), Color::White) - .wait() - .ensure_success("failed to build `api` image"); + spawn_and_log(&mut run, provisioner_target, api.color); + + let api_target = format!(" {} api", api.target); + let api_image = Self::build_image("api", &api.target, &api.id); + api.api_image = Some(api_image.clone()); File::create(&users_toml_file).unwrap(); - let container = format!("unveil_api_{}_{}", api.target, api.id); + let api_container = format!("shuttle_api_{}_{}", api.target, api.id); let mut run = Command::new("docker"); run.args([ "run", "--name", - &container, + &api_container, + "--network", + "shuttle-net", "-p", format!("{}:{}", api.proxy_addr.port(), 8000).as_str(), "-p", @@ -163,21 +188,38 @@ impl Api { "SHUTTLE_USERS_TOML=/config/users.toml", "-e", "SHUTTLE_INITIAL_KEY=ci-test", + "-e", + &format!("PROVISIONER_ADDRESS={provisioner_container}"), "-v", &format!("{}:/config/users.toml", users_toml_file), - &image, + &api_image, ]); + api.api_container = Some(api_container); spawn_and_log(&mut run, api_target, api.color); - api.image = Some(image); - api.container = Some(container); - api.wait_ready(Duration::from_secs(120)); api } + fn build_image(service: &str, target: &str, id: &str) -> String { + let image = format!("shuttle_{service}_{target}_{id}"); + let containerfile = format!("./{service}/Containerfile.dev"); + + let mut build = Command::new("docker"); + + build + .args(["build", "-f", &containerfile, "-t", &image, "."]) + .current_dir("../"); + + spawn_and_log(&mut build, target, Color::White) + .wait() + .ensure_success("failed to build `{service}` image"); + + image + } + pub fn wait_ready(&self, mut timeout: Duration) { let mut now = SystemTime::now(); while !timeout.is_zero() { @@ -197,7 +239,7 @@ impl Api { where I: IntoIterator, { - let client_target = format!("{} client", self.target); + let client_target = format!(" {} client", self.target); let mut build = Command::new("cargo"); build @@ -230,9 +272,27 @@ impl Api { } } -impl Drop for Api { +impl Drop for Services { fn drop(&mut self) { - if let Some(container) = &self.container { + if let Some(container) = &self.api_container { + Command::new("docker") + .args(["stop", container]) + .output() + .expect("failed to stop api container"); + Command::new("docker") + .args(["rm", container]) + .output() + .expect("failed to remove api container"); + } + + if let Some(image) = &self.api_image { + Command::new("docker") + .args(["rmi", image]) + .output() + .expect("failed to remove api image"); + } + + if let Some(container) = &self.provisioner_container { Command::new("docker") .args(["stop", container]) .output() @@ -243,7 +303,7 @@ impl Drop for Api { .expect("failed to remove api container"); } - if let Some(image) = &self.image { + if let Some(image) = &self.provisioner_image { Command::new("docker") .args(["rmi", image]) .output() diff --git a/e2e/tests/rocket.rs b/e2e/tests/rocket.rs index 860b864ae..d13762961 100644 --- a/e2e/tests/rocket.rs +++ b/e2e/tests/rocket.rs @@ -4,7 +4,7 @@ mod helpers; #[test] fn hello_world() { - let client = helpers::Api::new_docker("hello-world", Color::Green); + let client = helpers::Services::new_docker("hello-world", Color::Green); client.deploy("../examples/rocket/hello-world"); let request_text = client @@ -20,7 +20,7 @@ fn hello_world() { #[test] fn postgres() { - let client = helpers::Api::new_docker("postgres", Color::Blue); + let client = helpers::Services::new_docker("postgres", Color::Blue); client.deploy("../examples/rocket/postgres"); let add_response = client diff --git a/e2e/tests/tide.rs b/e2e/tests/tide.rs index c7cec1978..e03483d4c 100644 --- a/e2e/tests/tide.rs +++ b/e2e/tests/tide.rs @@ -4,7 +4,7 @@ mod helpers; #[test] fn hello_world() { - let client = helpers::Api::new_docker("hello-world", Color::Cyan); + let client = helpers::Services::new_docker("hello-world", Color::Cyan); client.deploy("../examples/tide/hello-world"); let request_text = client diff --git a/e2e/tests/tower.rs b/e2e/tests/tower.rs index ba9ea07cd..22fedb9fe 100644 --- a/e2e/tests/tower.rs +++ b/e2e/tests/tower.rs @@ -4,7 +4,7 @@ mod helpers; #[test] fn hello_world() { - let client = helpers::Api::new_docker("hello-world", Color::Cyan); + let client = helpers::Services::new_docker("hello-world", Color::Cyan); client.deploy("../examples/tower/hello-world"); let request_text = client diff --git a/examples/axum/hello-world/Cargo.toml b/examples/axum/hello-world/Cargo.toml index f5bfc66f3..66979945f 100644 --- a/examples/axum/hello-world/Cargo.toml +++ b/examples/axum/hello-world/Cargo.toml @@ -10,5 +10,5 @@ crate-type = ["cdylib"] [dependencies] axum = "0.5" -shuttle-service = { version = "0.3", features = ["web-axum"] } +shuttle-service = { version = "0.3.3", features = ["web-axum"] } sync_wrapper = "0.1" diff --git a/examples/axum/websocket/Cargo.toml b/examples/axum/websocket/Cargo.toml new file mode 100644 index 000000000..ee9ccbb71 --- /dev/null +++ b/examples/axum/websocket/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "websocket" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +axum = { version = "0.5", features = ["ws"] } +chrono = { version = "0.4", features = ["serde"] } +futures = "0.3" +hyper = { version = "0.14", features = ["client", "http2"] } +hyper-tls = "0.5" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +shuttle-service = { version = "0.3.3", features = ["web-axum"] } +sync_wrapper = "0.1" +tokio = { version = "1", features = ["full"] } diff --git a/examples/axum/websocket/Shuttle.toml b/examples/axum/websocket/Shuttle.toml new file mode 100644 index 000000000..a98513a3a --- /dev/null +++ b/examples/axum/websocket/Shuttle.toml @@ -0,0 +1 @@ +name = "websocket-axum-app" diff --git a/examples/axum/websocket/index.html b/examples/axum/websocket/index.html new file mode 100644 index 000000000..f7fbdf5ca --- /dev/null +++ b/examples/axum/websocket/index.html @@ -0,0 +1,75 @@ + + + + + + + Websocket status page + + + +
+
+ Current API status +
+
+
+
+
+ Last check time +
+
+
+
+
+ Clients watching +
+
+
+ + + + + + diff --git a/examples/axum/websocket/src/lib.rs b/examples/axum/websocket/src/lib.rs new file mode 100644 index 000000000..05b34923c --- /dev/null +++ b/examples/axum/websocket/src/lib.rs @@ -0,0 +1,131 @@ +use std::{sync::Arc, time::Duration}; + +use axum::{ + extract::{ + ws::{Message, WebSocket}, + WebSocketUpgrade, + }, + response::{Html, IntoResponse}, + routing::get, + Extension, Router, +}; +use chrono::{DateTime, Utc}; +use futures::{SinkExt, StreamExt}; +use hyper::{Client, Uri}; +use hyper_tls::HttpsConnector; +use serde::Serialize; +use sync_wrapper::SyncWrapper; +use tokio::{ + sync::{watch, Mutex}, + time::sleep, +}; + +struct State { + clients_count: usize, + rx: watch::Receiver, +} + +const PAUSE_SECS: u64 = 15; +const STATUS_URI: &str = "https://api.shuttle.rs/status"; + +#[derive(Serialize)] +struct Response { + clients_count: usize, + datetime: DateTime, + is_up: bool, +} + +#[shuttle_service::main] +async fn main() -> Result, shuttle_service::Error> { + let (tx, rx) = watch::channel(Message::Text("{}".to_string())); + + let state = Arc::new(Mutex::new(State { + clients_count: 0, + rx, + })); + + // Spawn a thread to continually check the status of the api + let state_send = state.clone(); + tokio::spawn(async move { + let duration = Duration::from_secs(PAUSE_SECS); + let https = HttpsConnector::new(); + let client = Client::builder().build::<_, hyper::Body>(https); + let uri: Uri = STATUS_URI.parse().unwrap(); + + loop { + let is_up = client.get(uri.clone()).await; + let is_up = is_up.is_ok(); + + let response = Response { + clients_count: state_send.lock().await.clients_count, + datetime: Utc::now(), + is_up, + }; + let msg = serde_json::to_string(&response).unwrap(); + + if tx.send(Message::Text(msg)).is_err() { + break; + } + + sleep(duration).await; + } + }); + + let router = Router::new() + .route("/", get(index)) + .route("/websocket", get(websocket_handler)) + .layer(Extension(state)); + + let sync_wrapper = SyncWrapper::new(router); + + Ok(sync_wrapper) +} + +async fn websocket_handler( + ws: WebSocketUpgrade, + Extension(state): Extension>>, +) -> impl IntoResponse { + ws.on_upgrade(|socket| websocket(socket, state)) +} + +async fn websocket(stream: WebSocket, state: Arc>) { + // By splitting we can send and receive at the same time. + let (mut sender, mut receiver) = stream.split(); + + let mut rx = { + let mut state = state.lock().await; + state.clients_count += 1; + state.rx.clone() + }; + + // This task will receive watch messages and forward it to this connected client. + let mut send_task = tokio::spawn(async move { + while let Ok(()) = rx.changed().await { + let msg = rx.borrow().clone(); + + if sender.send(msg).await.is_err() { + break; + } + } + }); + + // This task will receive messages from this client. + let mut recv_task = tokio::spawn(async move { + while let Some(Ok(Message::Text(text))) = receiver.next().await { + println!("this example does not read any messages, but got: {text}"); + } + }); + + // If any one of the tasks exit, abort the other. + tokio::select! { + _ = (&mut send_task) => recv_task.abort(), + _ = (&mut recv_task) => send_task.abort(), + }; + + // This client disconnected + state.lock().await.clients_count -= 1; +} + +async fn index() -> Html<&'static str> { + Html(include_str!("../index.html")) +} diff --git a/examples/rocket/authentication/Cargo.toml b/examples/rocket/authentication/Cargo.toml index 5076d4e95..23a5ccde5 100644 --- a/examples/rocket/authentication/Cargo.toml +++ b/examples/rocket/authentication/Cargo.toml @@ -14,4 +14,4 @@ jsonwebtoken = {version = "8", default-features = false } lazy_static = "1.4" rocket = {version = "0.5.0-rc.1", features = ["json"] } serde = {version = "1.0", features = ["derive"] } -shuttle-service = { version = "0.3", features = ["web-rocket"] } +shuttle-service = { version = "0.3.3", features = ["web-rocket"] } diff --git a/examples/rocket/authentication/README.md b/examples/rocket/authentication/README.md index 09db398e7..94f4dfd01 100644 --- a/examples/rocket/authentication/README.md +++ b/examples/rocket/authentication/README.md @@ -26,7 +26,9 @@ After logging into shuttle, use the following command to deploy this example: ```sh $ cargo shuttle deploy ``` -Make a note of the `Host` for the deploy to use in the examples below. Or just use `authentication-rocket-app.shuttleapp.rs` as the host below. + +Notice how this deploy fails since one of the test fails. See the `TODO` at the top of `src/claims.rs` on how to fix the test. Once the code is fixed, try to deploy again. +Now make a note of the `Host` for the deploy to use in the examples below. Or just use `authentication-rocket-app.shuttleapp.rs` as the host below. ### Seeing it in action First, we should be able to access the public endpoint without any authentication using: diff --git a/examples/rocket/authentication/src/claims.rs b/examples/rocket/authentication/src/claims.rs index ff5ed32ca..90a5b14d9 100644 --- a/examples/rocket/authentication/src/claims.rs +++ b/examples/rocket/authentication/src/claims.rs @@ -10,7 +10,10 @@ use rocket::{ }; use serde::{Deserialize, Serialize}; -const BEARER: &str = "Bearer "; +// TODO: this has an extra trailing space to cause the test to fail +// This is to demonstate shuttle will not deploy when a test fails. +// FIX: remove the extra space character and try deploying again +const BEARER: &str = "Bearer "; const AUTHORIZATION: &str = "Authorization"; /// Key used for symmetric token encoding @@ -22,7 +25,7 @@ lazy_static! { } // Used when decoding a token to `Claims` -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub(crate) enum AuthenticationError { Missing, Decoding(String), @@ -31,7 +34,7 @@ pub(crate) enum AuthenticationError { // Basic claim object. Only the `exp` claim (field) is required. Consult the `jsonwebtoken` documentation for other claims that can be validated. // The `name` is a custom claim for this API -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Debug)] pub(crate) struct Claims { pub(crate) name: String, exp: usize, @@ -63,7 +66,7 @@ impl Claims { /// Create a `Claims` from a 'Bearer ' value fn from_authorization(value: &str) -> Result { - let token = value.strip_prefix(BEARER); + let token = value.strip_prefix(BEARER).map(str::trim); if token.is_none() { return Err(AuthenticationError::Missing); @@ -108,3 +111,28 @@ impl Claims { Ok(token) } } + +#[cfg(test)] +mod tests { + use crate::claims::AuthenticationError; + + use super::Claims; + + #[test] + fn missing_bearer() { + let claim_err = Claims::from_authorization("no-Bearer-prefix").unwrap_err(); + + assert_eq!(claim_err, AuthenticationError::Missing); + } + + #[test] + fn to_token_and_back() { + let claim = Claims::from_name("test runner"); + let token = claim.to_token().unwrap(); + let token = format!("Bearer {token}"); + + let claim = Claims::from_authorization(&token).unwrap(); + + assert_eq!(claim.name, "test runner"); + } +} diff --git a/examples/rocket/hello-world/Cargo.toml b/examples/rocket/hello-world/Cargo.toml index 0d7aa5301..45f1f0c94 100644 --- a/examples/rocket/hello-world/Cargo.toml +++ b/examples/rocket/hello-world/Cargo.toml @@ -10,4 +10,4 @@ crate-type = ["cdylib"] [dependencies] rocket = "0.5.0-rc.1" -shuttle-service = { version = "0.3", features = ["web-rocket"] } +shuttle-service = { version = "0.3.3", features = ["web-rocket"] } diff --git a/examples/rocket/postgres/Cargo.toml b/examples/rocket/postgres/Cargo.toml index 09a347ba9..2cb36071e 100644 --- a/examples/rocket/postgres/Cargo.toml +++ b/examples/rocket/postgres/Cargo.toml @@ -12,4 +12,4 @@ crate-type = ["cdylib"] rocket = { version = "0.5.0-rc.1", features = ["json"] } serde = "1.0" sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres"] } -shuttle-service = { version = "0.3", features = ["sqlx-postgres", "secrets", "web-rocket"] } +shuttle-service = { version = "0.3.3", features = ["sqlx-postgres", "secrets", "web-rocket"] } diff --git a/examples/tide/hello-world/Cargo.toml b/examples/tide/hello-world/Cargo.toml index 573d5c231..7f3223eef 100644 --- a/examples/tide/hello-world/Cargo.toml +++ b/examples/tide/hello-world/Cargo.toml @@ -10,4 +10,4 @@ crate-type = ["cdylib"] [dependencies] tide = "0.16.0" -shuttle-service = { version = "0.3", features = ["web-tide"] } +shuttle-service = { version = "0.3.3", features = ["web-tide"] } diff --git a/examples/tower/hello-world/Cargo.toml b/examples/tower/hello-world/Cargo.toml index d2fbf3ff6..e886a8aa2 100644 --- a/examples/tower/hello-world/Cargo.toml +++ b/examples/tower/hello-world/Cargo.toml @@ -9,6 +9,6 @@ crate-type = ["cdylib"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -shuttle-service = { version = "0.3", features = ["web-tower"] } +shuttle-service = { version = "0.3.3", features = ["web-tower"] } tower = { version = "0.4", features = ["full"] } hyper = { version = "0.14", features = ["full"] } diff --git a/examples/url-shortener/Cargo.toml b/examples/url-shortener/Cargo.toml index 5b5352227..813dd4874 100644 --- a/examples/url-shortener/Cargo.toml +++ b/examples/url-shortener/Cargo.toml @@ -12,6 +12,6 @@ crate-type = ["cdylib"] rocket = { version = "0.5.0-rc.1", features = ["json"] } serde = "1.0" sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres"] } -shuttle-service = { version = "0.3", features = ["sqlx-postgres", "web-rocket"] } +shuttle-service = { version = "0.3.3", features = ["sqlx-postgres", "web-rocket"] } nanoid = "0.4" url ="2.2" diff --git a/proto/Cargo.toml b/proto/Cargo.toml new file mode 100644 index 000000000..571e2ed8d --- /dev/null +++ b/proto/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "proto" +version = "0.1.0" +edition = "2021" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +prost = "0.10.4" +tonic = "0.7.2" + +[dependencies.shuttle-common] +version = "0.3.1" +path = "../common" + +[build-dependencies] +tonic-build = "0.7.2" diff --git a/proto/build.rs b/proto/build.rs new file mode 100644 index 000000000..1502854e9 --- /dev/null +++ b/proto/build.rs @@ -0,0 +1,5 @@ +fn main() -> Result<(), Box> { + tonic_build::compile_protos("../proto/provisioner.proto")?; + + Ok(()) +} diff --git a/proto/provisioner.proto b/proto/provisioner.proto new file mode 100644 index 000000000..f93705325 --- /dev/null +++ b/proto/provisioner.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; +package provisioner; + +service Provisioner { + rpc ProvisionDatabase(DatabaseRequest) returns (DatabaseResponse); +} + +message DatabaseRequest { + string project_name = 1; +} + +message DatabaseResponse { + string username = 1; + string password = 2; + string database_name = 3; +} diff --git a/proto/src/lib.rs b/proto/src/lib.rs new file mode 100644 index 000000000..cb2bac417 --- /dev/null +++ b/proto/src/lib.rs @@ -0,0 +1,16 @@ +pub mod provisioner { + use shuttle_common::DatabaseReadyInfo; + + tonic::include_proto!("provisioner"); + + impl From for DatabaseReadyInfo { + fn from(response: DatabaseResponse) -> Self { + DatabaseReadyInfo::new( + response.username, + response.password, + response.database_name, + "5432".to_string(), + ) + } + } +} diff --git a/provisioner/Cargo.toml b/provisioner/Cargo.toml new file mode 100644 index 000000000..fe89731d3 --- /dev/null +++ b/provisioner/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "provisioner" +version = "0.1.0" +edition = "2021" +description = "Service responsible for provisioning and managing resources for services" +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +clap = { version = "3.1.18", features = ["derive", "env"] } +prost = "0.10.4" +rand = "0.8.5" +sqlx = { version = "0.5.13", features = ["postgres", "runtime-tokio-native-tls"] } +thiserror = "1.0.31" +tokio = { version = "1.19.2", features = ["macros", "rt-multi-thread"] } +tonic = "0.7.2" +tracing = "0.1.35" +tracing-subscriber = "0.3.11" + +[dependencies.proto] +version = "0.1.0" +path = "../proto" + +[dev-dependencies] +ctor = "0.1.22" +lazy_static = "1.4.0" +portpicker = "0.1.1" + +[build-dependencies] +tonic-build = "0.7.2" diff --git a/provisioner/Containerfile b/provisioner/Containerfile new file mode 100644 index 000000000..03cb7140f --- /dev/null +++ b/provisioner/Containerfile @@ -0,0 +1,28 @@ +FROM rust:buster as chef +RUN apt-get update &&\ + apt-get install -y protobuf-compiler +RUN cargo install cargo-chef +WORKDIR app + +FROM rust:buster AS runtime +RUN apt-get update &&\ + apt-get install -y curl postgresql supervisor +RUN pg_dropcluster $(pg_lsclusters -h | cut -d' ' -f-2 | head -n1) + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --recipe-path recipe.json +COPY . . +RUN cargo build --bin provisioner + +FROM runtime +COPY --from=builder /app/target/debug/provisioner /usr/local/bin/shuttle-provisioner + +COPY provisioner/docker/entrypoint.sh /bin/entrypoint.sh +COPY provisioner/docker/wait-for-pg-then /usr/bin/wait-for-pg-then +COPY provisioner/docker/supervisord.conf /usr/share/supervisord/supervisord.conf +ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/provisioner/Containerfile.dev b/provisioner/Containerfile.dev new file mode 100644 index 000000000..45953a4cc --- /dev/null +++ b/provisioner/Containerfile.dev @@ -0,0 +1,30 @@ +FROM rust:buster as runtime +RUN apt-get update &&\ + apt-get install -y curl postgresql supervisor +RUN pg_dropcluster $(pg_lsclusters -h | cut -d' ' -f-2 | head -n1) +RUN rustup component add rust-src + +FROM rust:buster AS chef +RUN apt-get update &&\ + apt-get install -y protobuf-compiler +WORKDIR app +RUN cargo install cargo-chef + +FROM chef AS planner +COPY . . +RUN cargo chef prepare --recipe-path recipe.json + +FROM chef AS builder +COPY --from=planner /app/recipe.json recipe.json +RUN cargo chef cook --recipe-path recipe.json +COPY . . +RUN cargo build --bin provisioner + +FROM runtime +WORKDIR / +COPY --from=builder /app/target/debug/provisioner /usr/local/bin/shuttle-provisioner + +COPY provisioner/docker/entrypoint.sh /bin/entrypoint.sh +COPY provisioner/docker/wait-for-pg-then /usr/bin/wait-for-pg-then +COPY provisioner/docker/supervisord.conf /usr/share/supervisord/supervisord.conf +ENTRYPOINT ["/bin/entrypoint.sh"] diff --git a/docker/entrypoint.sh b/provisioner/docker/entrypoint.sh similarity index 58% rename from docker/entrypoint.sh rename to provisioner/docker/entrypoint.sh index 84303f309..af1fec058 100755 --- a/docker/entrypoint.sh +++ b/provisioner/docker/entrypoint.sh @@ -1,11 +1,5 @@ #!/usr/bin/env bash -if [ -z $PROXY_FQDN ] -then - echo "The variable 'PROXY_FQDN' is missing" - exit 1 -fi - export PG_VERSION=${PG_VERSION:-11} export PG_CLUSTER_NAME=${PG_CLUSTER_NAME:-shuttle} @@ -31,6 +25,9 @@ if [[ "$(pg_lsclusters -h | wc -l)" -ne "1" ]]; then su postgres -c "psql -c \"ALTER USER postgres PASSWORD '${PG_PASSWORD}'\"" pg_ctlcluster $PG_VERSION $PG_CLUSTER_NAME stop set +e + + network=$(ip route | awk '/scope/{print $1}') + echo "host all all $network md5" >> /etc/postgresql/11/shuttle/pg_hba.conf fi export PG_LOG=$(pg_lsclusters -h | cut -d' ' -f7) @@ -39,25 +36,4 @@ export PG_HOST=localhost export PG_URI=postgres://postgres:${PG_PASSWORD}@localhost:${PG_PORT}/postgres -export CRATES_PATH=${CRATES_PATH:-/var/lib/shuttle/crates} - -mkdir -p $CRATES_PATH - -export PROXY_PORT=${PROXY_PORT:-8000} - -export API_PORT=${API_PORT:-8001} - -if [[ ! -z "${SHUTTLE_USERS_TOML}" && ! -s "${SHUTTLE_USERS_TOML}" ]] -then - if [[ -z "${SHUTTLE_INITIAL_KEY}" ]] - then - echo "\$SHUTTLE_INITIAL_KEY is not set to create initial user's key" - exit 1 - fi - - echo "Creating a first user with key '${SHUTTLE_INITIAL_KEY}' at '${SHUTTLE_USERS_TOML}'" - mkdir -p $(dirname "${SHUTTLE_USERS_TOML}") - echo -e "[$SHUTTLE_INITIAL_KEY]\nname = 'first-user'\nprojects = []" > "${SHUTTLE_USERS_TOML}" -fi - exec supervisord -n -c /usr/share/supervisord/supervisord.conf diff --git a/docker/supervisord.conf b/provisioner/docker/supervisord.conf similarity index 67% rename from docker/supervisord.conf rename to provisioner/docker/supervisord.conf index a65351ec2..d085dcf30 100644 --- a/docker/supervisord.conf +++ b/provisioner/docker/supervisord.conf @@ -7,8 +7,8 @@ startretries=8 startsecs=20 autorestart=true -[program:shuttle-api] -command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-backend --path %(ENV_CRATES_PATH)s --bind-addr 0.0.0.0 --api-port %(ENV_API_PORT)s --proxy-port %(ENV_PROXY_PORT)s --proxy-fqdn %(ENV_PROXY_FQDN)s +[program:shuttle-provisioner] +command=/usr/bin/wait-for-pg-then /usr/local/bin/shuttle-provisioner --ip 0.0.0.0 --port %(ENV_PORT)s --shared-pg-uri %(ENV_PG_URI)s redirect_stderr=true environment=RUST_BACKTRACE="1",RUST_LOG="debug" startretries=3 diff --git a/docker/wait-for-pg-then b/provisioner/docker/wait-for-pg-then similarity index 100% rename from docker/wait-for-pg-then rename to provisioner/docker/wait-for-pg-then diff --git a/provisioner/src/args.rs b/provisioner/src/args.rs new file mode 100644 index 000000000..3bf09c09e --- /dev/null +++ b/provisioner/src/args.rs @@ -0,0 +1,19 @@ +use std::net::{IpAddr, Ipv4Addr}; + +use clap::Parser; + +#[derive(Parser, Debug)] +#[clap(author, version, about, long_about = None)] +pub struct Args { + /// Address to bind provisioner on + #[clap(short, long, env = "PROVISIONER_IP", default_value_t = Ipv4Addr::LOCALHOST.into())] + pub ip: IpAddr, + + /// Port to start provisioner on + #[clap(short, long, env = "PROVISIONER_PORT", default_value_t = 5001)] + pub port: u16, + + /// URI to connect to Postgres for managing shared DB resources + #[clap(short, long, env = "PROVISIONER_PG_URI", hide_env_values = true)] + pub shared_pg_uri: String, +} diff --git a/provisioner/src/error.rs b/provisioner/src/error.rs new file mode 100644 index 000000000..6e91436d7 --- /dev/null +++ b/provisioner/src/error.rs @@ -0,0 +1,27 @@ +use thiserror::Error; +use tonic::Status; +use tracing::error; + +#[derive(Error, Debug)] +pub enum Error { + #[error("failed to create role")] + CreateRole(String), + + #[error("failed to update role")] + UpdateRole(String), + + #[error("failed to create DB")] + CreateDB(String), + + #[error("unexpected error")] + Unexpected(#[from] sqlx::Error), +} + +unsafe impl Send for Error {} + +impl From for Status { + fn from(err: Error) -> Self { + error!(error = &err as &dyn std::error::Error, "provision failed"); + Status::internal("failed to provision a database") + } +} diff --git a/provisioner/src/lib.rs b/provisioner/src/lib.rs new file mode 100644 index 000000000..47ca163c5 --- /dev/null +++ b/provisioner/src/lib.rs @@ -0,0 +1,123 @@ +use std::time::Duration; + +pub use args::Args; +pub use error::Error; +use proto::provisioner::provisioner_server::Provisioner; +pub use proto::provisioner::provisioner_server::ProvisionerServer; +use proto::provisioner::{DatabaseRequest, DatabaseResponse}; +use rand::Rng; +use sqlx::{postgres::PgPoolOptions, PgPool}; +use tonic::{Request, Response, Status}; +use tracing::info; + +mod args; +mod error; + +pub struct MyProvisioner { + pool: PgPool, +} + +impl MyProvisioner { + pub fn new(uri: &str) -> sqlx::Result { + Ok(Self { + pool: PgPoolOptions::new() + .min_connections(4) + .max_connections(12) + .connect_timeout(Duration::from_secs(60)) + .connect_lazy(uri)?, + }) + } + + pub async fn request_shared_db(&self, project_name: &str) -> Result { + let (username, password) = self.shared_role(project_name).await?; + let database_name = self.shared_db(project_name, &username).await?; + + Ok(DatabaseResponse { + username, + password, + database_name, + }) + } + + async fn shared_role(&self, project_name: &str) -> Result<(String, String), Error> { + let username = format!("user-{project_name}"); + let password = generate_password(); + + let matching_user = sqlx::query("SELECT rolname FROM pg_roles WHERE rolname = $1") + .bind(&username) + .fetch_optional(&self.pool) + .await?; + + if matching_user.is_none() { + info!("creating new user"); + + // Binding does not work for identifiers + // https://stackoverflow.com/questions/63723236/sql-statement-to-create-role-fails-on-postgres-12-using-dapper + let create_role_query = + format!("CREATE ROLE \"{username}\" WITH LOGIN PASSWORD '{password}'"); + sqlx::query(&create_role_query) + .execute(&self.pool) + .await + .map_err(|e| Error::CreateRole(e.to_string()))?; + } else { + info!("cycling password of user"); + + // Binding does not work for identifiers + // https://stackoverflow.com/questions/63723236/sql-statement-to-create-role-fails-on-postgres-12-using-dapper + let update_role_query = + format!("ALTER ROLE \"{username}\" WITH LOGIN PASSWORD '{password}'"); + sqlx::query(&update_role_query) + .execute(&self.pool) + .await + .map_err(|e| Error::UpdateRole(e.to_string()))?; + } + + Ok((username, password)) + } + + async fn shared_db(&self, project_name: &str, username: &str) -> Result { + let database_name = format!("db-{project_name}"); + + let matching_db = sqlx::query("SELECT datname FROM pg_database WHERE datname = $1") + .bind(&database_name) + .fetch_optional(&self.pool) + .await?; + + if matching_db.is_none() { + info!("creating database"); + + // Binding does not work for identifiers + // https://stackoverflow.com/questions/63723236/sql-statement-to-create-role-fails-on-postgres-12-using-dapper + let create_db_query = format!("CREATE DATABASE \"{database_name}\" OWNER '{username}'"); + sqlx::query(&create_db_query) + .execute(&self.pool) + .await + .map_err(|e| Error::CreateDB(e.to_string()))?; + } + + Ok(database_name) + } +} + +#[tonic::async_trait] +impl Provisioner for MyProvisioner { + #[tracing::instrument(skip(self))] + async fn provision_database( + &self, + request: Request, + ) -> Result, Status> { + let reply = self + .request_shared_db(&request.into_inner().project_name) + .await?; + + Ok(Response::new(reply)) + } +} + +fn generate_password() -> String { + rand::thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(12) + .map(char::from) + .collect() +} diff --git a/provisioner/src/main.rs b/provisioner/src/main.rs new file mode 100644 index 000000000..37a196adb --- /dev/null +++ b/provisioner/src/main.rs @@ -0,0 +1,27 @@ +use std::net::SocketAddr; + +use clap::Parser; +use provisioner::{Args, MyProvisioner, ProvisionerServer}; +use tonic::transport::Server; + +#[tokio::main] +async fn main() -> Result<(), Box> { + tracing_subscriber::fmt::init(); + + let Args { + ip, + port, + shared_pg_uri, + } = Args::parse(); + let addr = SocketAddr::new(ip, port); + + let provisioner = MyProvisioner::new(&shared_pg_uri).unwrap(); + + println!("starting provisioner on {}", addr); + Server::builder() + .add_service(ProvisionerServer::new(provisioner)) + .serve(addr) + .await?; + + Ok(()) +} diff --git a/provisioner/tests/provisioner.rs b/provisioner/tests/provisioner.rs new file mode 100644 index 000000000..cb04fe8da --- /dev/null +++ b/provisioner/tests/provisioner.rs @@ -0,0 +1,194 @@ +use portpicker::pick_unused_port; +use std::{ + process::Command, + thread::sleep, + time::{Duration, SystemTime}, +}; + +use ctor::dtor; +use lazy_static::lazy_static; +use provisioner::MyProvisioner; + +lazy_static! { + static ref PG: DockerPG = DockerPG::new(); +} + +#[dtor] +fn cleanup() { + PG.cleanup(); +} + +struct DockerPG { + container_name: String, + uri: String, +} + +impl DockerPG { + fn new() -> Self { + let container_name = "shuttle_provisioner_it"; + let port = pick_unused_port().unwrap(); + + Command::new("docker") + .args([ + "run", + "--rm", + "--name", + container_name, + "-e", + "POSTGRES_PASSWORD=password", + "-p", + &format!("{port}:5432"), + "postgres:11", + ]) + .spawn() + .unwrap(); + + Self::wait_ready(container_name, Duration::from_secs(120)); + + Self { + container_name: container_name.to_string(), + uri: format!("postgres://postgres:password@localhost:{port}"), + } + } + + fn wait_ready(container_name: &str, mut timeout: Duration) { + let mut now = SystemTime::now(); + while !timeout.is_zero() { + let status = Command::new("docker") + .args(["exec", container_name, "pg_isready"]) + .output() + .unwrap() + .status; + + if status.success() { + return; + } + + sleep(Duration::from_millis(350)); + + timeout = timeout + .checked_sub(now.elapsed().unwrap()) + .unwrap_or_default(); + now = SystemTime::now(); + } + panic!("timed out while waiting for provisioner DB to come up"); + } + + fn cleanup(&self) { + Command::new("docker") + .args(["stop", &self.container_name]) + .output() + .expect("failed to stop provisioner test DB container"); + Command::new("docker") + .args(["rm", &self.container_name]) + .output() + .expect("failed to remove provisioner test DB container"); + } +} + +fn exec(query: &str) -> String { + let output = Command::new("docker") + .args([ + "exec", + &PG.container_name, + "psql", + "--username", + "postgres", + "--tuples-only", + "--no-align", + "--field-separator", + ",", + "--command", + query, + ]) + .output() + .unwrap() + .stdout; + + String::from_utf8(output).unwrap().trim().to_string() +} + +#[tokio::test] +async fn shared_db_role_does_not_exist() { + let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + + assert_eq!( + exec("SELECT rolname FROM pg_roles WHERE rolname = 'user-not_exist'"), + "" + ); + + provisioner.request_shared_db("not_exist").await.unwrap(); + + assert_eq!( + exec("SELECT rolname FROM pg_roles WHERE rolname = 'user-not_exist'"), + "user-not_exist" + ); +} + +#[tokio::test] +async fn shared_db_role_does_exist() { + let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + + exec("CREATE ROLE \"user-exist\" WITH LOGIN PASSWORD 'temp'"); + assert_eq!( + exec("SELECT passwd FROM pg_shadow WHERE usename = 'user-exist'"), + "md5d44ae85dd21bda2a4f9946217adea2cc" + ); + + provisioner.request_shared_db("exist").await.unwrap(); + + // Make sure password got cycled + assert_ne!( + exec("SELECT passwd FROM pg_shadow WHERE usename = 'user-exist'"), + "md5d44ae85dd21bda2a4f9946217adea2cc" + ); +} + +#[tokio::test] +#[should_panic( + expected = "CreateRole(\"error returned from database: cannot insert multiple commands into a prepared statement\"" +)] +async fn injection_safe() { + let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + + provisioner + .request_shared_db("new\"; CREATE ROLE \"injected") + .await + .unwrap(); +} + +#[tokio::test] +async fn shared_db_missing() { + let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + + assert_eq!( + exec("SELECT datname FROM pg_database WHERE datname = 'db-missing'"), + "" + ); + + provisioner.request_shared_db("missing").await.unwrap(); + + assert_eq!( + exec("SELECT datname FROM pg_database WHERE datname = 'db-missing'"), + "db-missing" + ); +} + +#[tokio::test] +async fn shared_db_filled() { + let provisioner = MyProvisioner::new(&PG.uri).unwrap(); + + exec("CREATE ROLE \"user-filled\" WITH LOGIN PASSWORD 'temp'"); + exec("CREATE DATABASE \"db-filled\" OWNER 'user-filled'"); + assert_eq!( + exec("SELECT datname FROM pg_database WHERE datname = 'db-filled'"), + "db-filled" + ); + + provisioner.request_shared_db("filled").await.unwrap(); + + assert_eq!( + exec("SELECT datname FROM pg_database WHERE datname = 'db-filled'"), + "db-filled" + ); +} diff --git a/service/Cargo.toml b/service/Cargo.toml index 40628ad2a..3b5373882 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "shuttle-service" -version = "0.3.1" +version = "0.3.3" edition = "2021" license = "Apache-2.0" description = "Service traits and macros to deploy on the shuttle platform (https://www.shuttle.rs/)" @@ -9,34 +9,42 @@ description = "Service traits and macros to deploy on the shuttle platform (http doctest = false [dependencies] -anyhow = "1.0" -async-trait = "0.1" -chrono = "0.4" +anyhow = "1.0.57" +async-trait = "0.1.56" +axum = { version = "0.5.7", optional = true } +cargo = { version = "0.62.0", optional = true } +chrono = "0.4.19" +futures = { version = "0.3.21", features = ["std"] } +hyper = { version = "0.14.19", features = ["server", "tcp", "http1"], optional = true } +lazy_static = "1.4.0" libloading = { version = "0.7.3", optional = true } -log = "0.4" -sync_wrapper = { version = "0.1", optional = true } -axum = { version = "0.5", optional = true } -rocket = { version = "0.5.0-rc.1", optional = true } +log = "0.4.17" +regex = "1.5.6" +rocket = { version = "0.5.0-rc.2", optional = true } +sqlx = { version = "0.5.13", optional = true } +sync_wrapper = { version = "0.1.1", optional = true } +thiserror = "1.0.31" tide = { version = "0.16.0", optional = true } -tower = { version = "0.4", features = ["make"], optional = true } -hyper = { version = "0.14", features = ["server", "tcp", "http1"], optional = true } -sqlx = { version = "0.5", optional = true } -tokio = { version = "1.0", features = ["rt", "rt-multi-thread"] } -thiserror = "1.0" -lazy_static = "1.4" -regex = "1.5" +tokio = { version = "1.19.2", features = ["rt", "rt-multi-thread"] } +tower = { version = "0.4.12", features = ["make"], optional = true } + +[dependencies.shuttle-codegen] +version = "0.3.1" +path = "../codegen" +optional = true -shuttle-codegen = { version = "0.3.0", path = "../codegen", optional = true } -shuttle-common = { version = "0.3.0", path = "../common" } +[dependencies.shuttle-common] +version = "0.3.1" +path = "../common" [dev-dependencies] portpicker = "0.1.1" -uuid = "0.8.2" +uuid = "1.1.1" [features] default = ["codegen"] codegen = ["shuttle-codegen"] -loader = ["libloading"] +loader = ["cargo", "libloading"] sqlx-integration = ["sqlx/runtime-tokio-native-tls"] sqlx-postgres = ["sqlx-integration", "sqlx/postgres"] diff --git a/service/src/error.rs b/service/src/error.rs index 11385d51b..4e3982087 100644 --- a/service/src/error.rs +++ b/service/src/error.rs @@ -8,6 +8,10 @@ pub enum Error { Io(#[from] std::io::Error), #[error("Database error: {0}")] Database(String), + #[error("Panic occurred in `Service::build`: {0}")] + BuildPanic(String), + #[error("Panic occurred in `Service::bind`: {0}")] + BindPanic(String), #[error("Custom error: {0}")] Custom(#[from] CustomError), } diff --git a/service/src/lib.rs b/service/src/lib.rs index feccd7c7d..c1df2944c 100644 --- a/service/src/lib.rs +++ b/service/src/lib.rs @@ -12,21 +12,36 @@ //! Shuttle is an open-source app platform that uses traits and annotations to configure your backend deployments. //! //! ## Usage +//! Start by installing the [`cargo shuttle`](https://docs.rs/crate/cargo-shuttle/latest) subcommand by runnning the following in a terminal: //! -//! Depend on `shuttle-service` in `Cargo.toml`: +//! ```bash +//! $ cargo install cargo-shuttle +//! ``` //! -//! ```toml -//! shuttle-service = { version = "0.3", features = ["web-rocket"] } +//! Now that shuttle is installed, you can create your first project using: +//! +//! ```bash +//! cargo shuttle init my-rocket-app //! ``` //! -//! and make sure your crate has a `cdylib` output target: +//! By looking at the `Cargo.toml` file of the created project you will see the crate has a `cdylib` type. +//! This is because all shuttle projects are loaded by shuttle during runtime as dynamic libraries. +//! Thus, you can convert any library crate to a shuttle project by adding these lines to `Cargo.toml`. //! //! ```toml //! [lib] //! crate-type = ["cdylib"] //! ``` //! -//! See the [shuttle_service::main][main] macro for more information on supported services - like Axum. Here's a simple example using [rocket](https://docs.rs/rocket) to get you started: +//! Another piece needed for a shuttle project is the `shuttle-service` dependency. +//! Go ahead and update the `shuttle-service` dependency inside `Cargo.toml` to prepare this crate as a rocket project +//! by adding the `web-rocket` feature on the `shuttle-service` dependency. +//! +//! ```toml +//! shuttle-service = { version = "0.3.3", features = ["web-rocket"] } +//! ``` +//! +//! Now replace `src/lib.rs` with the following content. //! //! ```rust,no_run //! #[macro_use] @@ -47,17 +62,28 @@ //! } //! ``` //! -//! Complete examples can be found [in the repository](https://github.com/getsynth/shuttle/tree/main/examples/rocket). +//! See the [shuttle_service::main][main] macro for more information on supported services - like Axum. +//! Or look at more complete examples [in the repository](https://github.com/getsynth/shuttle/tree/main/examples), but +//! take note that the examples may update before official releases. //! -//! ## Deploying +//! ## Running locally +//! To test your app locally before deploying, use: +//! +//! ```bash +//! $ cargo shuttle run +//! ``` //! -//! You can deploy your service with the [`cargo shuttle`](https://docs.rs/crate/cargo-shuttle/latest) subcommand. To install run: +//! You should see your app build and start on the default port 8000. You can test this using; //! //! ```bash -//! $ cargo install cargo-shuttle +//! $ curl http://localhost:8000/hello +//! Hello, world! //! ``` //! -//! in a terminal. Once installed, run: +//! ## Deploying +//! +//! You can deploy your service with the [`cargo shuttle`](https://docs.rs/crate/cargo-shuttle/latest) subcommand too. +//! But, you will need to authenticate with the shuttle service first using: //! //! ```bash //! $ cargo shuttle login @@ -74,7 +100,7 @@ //! Your service will immediately be available at `{crate_name}.shuttleapp.rs`. For example: //! //! ```bash -//! $ curl https://hello-world-rocket-app.shuttleapp.rs +//! $ curl https://my-rocket-app.shuttleapp.rs/hello //! Hello, world! //! ``` //! @@ -82,12 +108,14 @@ //! //! Here is a quick example to deploy a service which uses a postgres database and [sqlx](http://docs.rs/sqlx): //! -//! Depend on `shuttle-service` in `Cargo.toml`: +//! Add the `sqlx-postgres` feature to the `shuttle-service` dependency inside `Cargo.toml`: //! //! ```toml -//! shuttle-service = { version = "0.3", features = ["web-rocket", "sqlx-postgres"] } +//! shuttle-service = { version = "0.3.3", features = ["web-rocket", "sqlx-postgres"] } //! ``` //! +//! Now update the `#[shuttle_service::main]` function to take in a `PgPool`: +//! //! ```rust,no_run //! #[macro_use] //! extern crate rocket; @@ -113,22 +141,16 @@ //! } //! ``` //! +//! For a local run, shuttle will automatically provision a Postgres instance inside a [Docker](https://www.docker.com/) container on your machine and connect it to the `PgPool`. +//! +//! For deploys, shuttle will provision a database for your application and connect it to the `PgPool` on your behalf. +//! //! To learn more about shuttle managed services, see [shuttle_service::main][main#getting-shuttle-managed-services]. //! //! ## Configuration //! //! The `cargo shuttle` command can be customised by creating a `Shuttle.toml` in the same location as your `Cargo.toml`. //! -//! ## Getting API keys -//! -//! After you've installed the [cargo-shuttle](https://docs.rs/crate/cargo-shuttle/latest) command, run: -//! -//! ```bash -//! $ cargo shuttle login -//! ``` -//! -//! this will open a browser window and prompt you to connect using your GitHub account. -//! //! ##### Change the name of your service //! //! To have your service deployed with a different name, add a `name` entry in the `Shuttle.toml`: @@ -145,6 +167,34 @@ //! cargo shuttle deploy --name=$PROJECT_NAME //! ``` //! +//! ##### Using Podman instead of Docker +//! If you are using [Podman](https://podman.io/) instead of Docker, then `cargo shuttle run` will give +//! `got unexpected error while inspecting docker container: error trying to connect: No such file or directory` error. +//! +//! To fix this error you will need to expose a rootless socket for Podman first. This can be done using: +//! +//! ```bash +//! podman system service --time=0 unix:///tmp/podman.sock +//! ``` +//! +//! Now set the `DOCKER_HOST` environment variable to point to this socket using: +//! +//! ```bash +//! export DOCKER_HOST=unix:///tmp/podman.sock +//! ``` +//! +//! Now all `cargo shuttle run` commands will work against Podman. +//! +//! ## Getting API keys +//! +//! After you've installed the [cargo-shuttle](https://docs.rs/crate/cargo-shuttle/latest) command, run: +//! +//! ```bash +//! $ cargo shuttle login +//! ``` +//! +//! this will open a browser window and prompt you to connect using your GitHub account. +//! //! ## We're in alpha 🤗 //! //! Thanks for using shuttle! We're very happy to have you with us! @@ -170,7 +220,6 @@ use async_trait::async_trait; // Pub uses by `codegen` pub use log; -pub use logger::Logger; pub use tokio::runtime::Runtime; pub mod error; @@ -203,17 +252,18 @@ extern crate shuttle_codegen; /// ``` /// /// ## shuttle supported services -/// The following type can take the place of the `Ok` type and enjoy first class service support in shuttle. Be sure to also enable the feature on +/// The following types can be returned from a `#[shuttle_service::main]` function and enjoy first class service support in shuttle. Be sure to also enable the correct feature on /// `shuttle-service` in `Cargo.toml` for the type to be recognized. /// -/// | Ok type | Feature flag | Service | Version | Example | -/// | ------------------------------------------------------------------------------ | ------------ | ------------------------------------------- | ---------- | ----------------------------------------------------------------------------------- | -/// | [`Rocket`](https://docs.rs/rocket/0.5.0-rc.1/rocket/struct.Rocket.html) | web-rocket | [rocket](https://docs.rs/rocket/0.5.0-rc.1) | 0.5.0-rc.1 | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/rocket/hello-world) | -/// | [`SyncWrapper`](https://docs.rs/axum/0.5/axum/struct.Router.html) | web-axum | [axum](https://docs.rs/axum/0.5) | 0.5 | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/axum/hello-world) | -/// | [`Server`](https://docs.rs/tide/latest/tide/struct.Server.html) | web-tide | [tide](https://docs.rs/tide/0.16.0) | 0.16.0 | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/tide/hello-world) | +/// | Return type | Feature flag | Service | Version | Example | +/// | ------------------------------------- | ------------ | ------------------------------------------- | ---------- | ----------------------------------------------------------------------------------- | +/// | `ShuttleRocket` | web-rocket | [rocket](https://docs.rs/rocket/0.5.0-rc.2) | 0.5.0-rc.2 | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/rocket/hello-world) | +/// | `ShuttleAxum` | web-axum | [axum](https://docs.rs/axum/0.5) | 0.5 | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/axum/hello-world) | +/// | `ShuttleTide` | web-tide | [tide](https://docs.rs/tide/0.16.0) | 0.16.0 | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/tide/hello-world) | +/// | `Result` | web-tower | [tower](https://docs.rs/tower/0.4.12) | 0.14.12 | [GitHub](https://github.com/getsynth/shuttle/tree/main/examples/tower/hello-world) | /// /// # Getting shuttle managed services -/// The shuttle is able to manage service dependencies for you. These services are passed in as inputs to your main function: +/// Shuttle is able to manage service dependencies for you. These are passed in as inputs to your `#[shuttle_service::main]` function: /// ```rust,no_run /// use sqlx::PgPool; /// use shuttle_service::ShuttleRocket; @@ -304,7 +354,11 @@ pub trait Service: Send + Sync { /// And the logger is for logging all runtime events /// /// The default is a noop that returns `Ok(())`. - async fn build(&mut self, _: &mut dyn Factory, _logger: Logger) -> Result<(), Error> { + async fn build( + &mut self, + _: &mut dyn Factory, + _logger: Box, + ) -> Result<(), Error> { Ok(()) } @@ -327,7 +381,7 @@ pub type StateBuilder = for<'a> fn( &'a mut dyn Factory, &'a Runtime, - Logger, + Box, ) -> Pin> + Send + 'a>>; /// A wrapper that takes a user's future, gives the future a factory, and takes the returned service from the future @@ -342,7 +396,7 @@ impl IntoService for for<'a> fn( &'a mut dyn Factory, &'a Runtime, - Logger, + Box, ) -> Pin> + Send + 'a>> where SimpleService: Service, @@ -361,10 +415,13 @@ where #[cfg(feature = "web-rocket")] #[async_trait] impl Service for SimpleService> { - async fn build(&mut self, factory: &mut dyn Factory, logger: Logger) -> Result<(), Error> { + async fn build( + &mut self, + factory: &mut dyn Factory, + logger: Box, + ) -> Result<(), Error> { if let Some(builder) = self.builder.take() { let rocket = builder(factory, &self.runtime, logger).await?; - self.service = Some(rocket); } @@ -373,11 +430,16 @@ impl Service for SimpleService> { fn bind(&mut self, addr: SocketAddr) -> Result { let rocket = self.service.take().expect("service has already been bound"); + let shutdown = rocket::config::Shutdown { + ctrlc: false, + ..rocket::config::Shutdown::default() + }; let config = rocket::Config { address: addr.ip(), port: addr.port(), log_level: rocket::config::LogLevel::Off, + shutdown, ..Default::default() }; let launched = rocket.configure(config).launch(); @@ -397,10 +459,13 @@ pub type ShuttleRocket = Result, Error>; #[cfg(feature = "web-axum")] #[async_trait] impl Service for SimpleService> { - async fn build(&mut self, factory: &mut dyn Factory, logger: Logger) -> Result<(), Error> { + async fn build( + &mut self, + factory: &mut dyn Factory, + logger: Box, + ) -> Result<(), Error> { if let Some(builder) = self.builder.take() { let axum = builder(factory, &self.runtime, logger).await?; - self.service = Some(axum); } @@ -435,10 +500,13 @@ impl Service for SimpleService> where T: Clone + Send + Sync + 'static, { - async fn build(&mut self, factory: &mut dyn Factory, logger: Logger) -> Result<(), Error> { + async fn build( + &mut self, + factory: &mut dyn Factory, + logger: Box, + ) -> Result<(), Error> { if let Some(builder) = self.builder.take() { let tide = builder(factory, &self.runtime, logger).await?; - self.service = Some(tide); } @@ -475,11 +543,10 @@ where async fn build( &mut self, factory: &mut dyn Factory, - logger: logger::Logger, + logger: Box, ) -> Result<(), Error> { if let Some(builder) = self.builder.take() { let tower = builder(factory, &self.runtime, logger).await?; - self.service = Some(tower); } @@ -503,6 +570,8 @@ where } } +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + /// Helper macro that generates the entrypoint required of any service. /// /// Can be used in one of two ways: diff --git a/service/src/loader.rs b/service/src/loader.rs index 736173a70..d9000a168 100644 --- a/service/src/loader.rs +++ b/service/src/loader.rs @@ -1,9 +1,21 @@ +use std::any::Any; +use std::ffi::OsStr; use std::net::SocketAddr; -use std::{ffi::OsStr, sync::mpsc::SyncSender}; - +use std::panic::AssertUnwindSafe; +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, Context}; +use cargo::core::compiler::CompileMode; +use cargo::core::{Shell, Verbosity, Workspace}; +use cargo::ops::{compile, CompileOptions}; +use cargo::util::homedir; +use cargo::Config; use libloading::{Library, Symbol}; use shuttle_common::DeploymentId; use thiserror::Error as ThisError; +use tokio::sync::mpsc::UnboundedSender; + +use futures::FutureExt; use crate::{ logger::{Log, Logger}, @@ -52,19 +64,86 @@ impl Loader { self, factory: &mut dyn Factory, addr: SocketAddr, - tx: SyncSender, + tx: UnboundedSender, deployment_id: DeploymentId, ) -> Result<(ServeHandle, Library), Error> { let mut service = self.service; - let logger = Logger::new(tx, deployment_id); + let logger = Box::new(Logger::new(tx, deployment_id)); + + AssertUnwindSafe(service.build(factory, logger)) + .catch_unwind() + .await + .map_err(|e| Error::BuildPanic(map_any_to_panic_string(&*e)))??; - service.build(factory, logger).await?; + // channel used by task spawned below to indicate whether or not panic + // occurred in `service.bind` call + let (send, recv) = tokio::sync::oneshot::channel(); // Start service on this side of the FFI - let handle = tokio::task::spawn(async move { service.bind(addr)?.await? }); + let handle = tokio::spawn(async move { + let bound = AssertUnwindSafe(async { service.bind(addr) }) + .catch_unwind() + .await; + + let payload = if let Err(e) = &bound { + Err(Error::BindPanic(map_any_to_panic_string(&**e))) + } else { + Ok(()) + }; + send.send(payload).unwrap(); + + if let Ok(b) = bound { + b?.await? + } else { + Err(anyhow!("panic in `Service::bound`")) + } + }); + + recv.await.unwrap().map(|_| (handle, self.so)) + } +} + +/// Given a project directory path, builds the crate +pub fn build_crate(project_path: &Path, buf: Box) -> anyhow::Result { + let mut shell = Shell::from_write(buf); + shell.set_verbosity(Verbosity::Normal); + + let cwd = std::env::current_dir() + .with_context(|| "couldn't get the current directory of the process")?; + let homedir = homedir(&cwd).ok_or_else(|| { + anyhow!( + "Cargo couldn't find your home directory. \ + This probably means that $HOME was not set." + ) + })?; + + let config = Config::new(shell, cwd, homedir); + let manifest_path = project_path.join("Cargo.toml"); + + let ws = Workspace::new(&manifest_path, &config)?; + + if let Some(profiles) = ws.profiles() { + for profile in profiles.get_all().values() { + if profile.panic.as_deref() == Some("abort") { + return Err(anyhow!("a Shuttle project cannot have panics that abort. Please ensure your Cargo.toml does not contain `panic = \"abort\"` for any profiles")); + } + } + } + + let opts = CompileOptions::new(&config, CompileMode::Build)?; + let compilation = compile(&ws, &opts)?; - Ok((handle, self.so)) + if compilation.cdylibs.is_empty() { + return Err(anyhow!("a cdylib was not created. Try adding the following to the Cargo.toml of the service:\n[lib]\ncrate-type = [\"cdylib\"]\n")); } + + Ok(compilation.cdylibs[0].path.clone()) +} + +fn map_any_to_panic_string(a: &dyn Any) -> String { + a.downcast_ref::<&str>() + .map(|x| x.to_string()) + .unwrap_or_else(|| "".to_string()) } #[cfg(test)] diff --git a/service/src/logger.rs b/service/src/logger.rs index d42b3ff02..40009e62e 100644 --- a/service/src/logger.rs +++ b/service/src/logger.rs @@ -1,7 +1,7 @@ use chrono::{DateTime, Utc}; use log::{Level, Metadata, Record}; use shuttle_common::{DeploymentId, LogItem}; -use std::sync::mpsc::SyncSender; +use tokio::sync::mpsc::UnboundedSender; #[derive(Debug)] pub struct Log { @@ -12,11 +12,11 @@ pub struct Log { pub struct Logger { deployment_id: DeploymentId, - tx: SyncSender, + tx: UnboundedSender, } impl Logger { - pub fn new(tx: SyncSender, deployment_id: DeploymentId) -> Self { + pub fn new(tx: UnboundedSender, deployment_id: DeploymentId) -> Self { Self { tx, deployment_id } } } diff --git a/service/tests/build_crate.rs b/service/tests/build_crate.rs new file mode 100644 index 000000000..45a0991b9 --- /dev/null +++ b/service/tests/build_crate.rs @@ -0,0 +1,52 @@ +use std::{io::Write, path::Path}; + +use shuttle_service::loader::build_crate; + +struct DummyWriter {} + +impl Write for DummyWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[test] +fn not_shuttle() { + let buf = Box::new(DummyWriter {}); + let project_path = format!("{}/tests/resources/not-shuttle", env!("CARGO_MANIFEST_DIR")); + let so_path = build_crate(Path::new(&project_path), buf).unwrap(); + + assert!( + so_path + .display() + .to_string() + .ends_with("tests/resources/not-shuttle/target/debug/libnot_shuttle.so"), + "did not get expected so_path: {}", + so_path.display() + ); +} + +#[test] +#[should_panic( + expected = "a cdylib was not created. Try adding the following to the Cargo.toml of the service:\n[lib]\ncrate-type = [\"cdylib\"]\n" +)] +fn not_cdylib() { + let buf = Box::new(DummyWriter {}); + let project_path = format!("{}/tests/resources/not-cdylib", env!("CARGO_MANIFEST_DIR")); + build_crate(Path::new(&project_path), buf).unwrap(); +} + +#[test] +#[should_panic(expected = "failed to read")] +fn not_found() { + let buf = Box::new(DummyWriter {}); + let project_path = format!( + "{}/tests/resources/non-existing", + env!("CARGO_MANIFEST_DIR") + ); + build_crate(Path::new(&project_path), buf).unwrap(); +} diff --git a/service/tests/helpers/mod.rs b/service/tests/helpers/mod.rs index 261e68ed3..13c6a289f 100644 --- a/service/tests/helpers/mod.rs +++ b/service/tests/helpers/mod.rs @@ -1,4 +1,5 @@ use std::future::Future; +use std::path::PathBuf; use std::process::Command; use std::thread::sleep; use std::time::Duration; @@ -6,6 +7,8 @@ use std::time::Duration; use portpicker::pick_unused_port; use sqlx::Connection; +use shuttle_service::loader::{Loader, LoaderError}; + pub struct PostgresInstance { port: u16, container: String, @@ -31,7 +34,7 @@ impl PostgresInstance { &format!("POSTGRES_PASSWORD={}", password), "-p", &format!("{}:5432", port), - "postgres:11", // Our Dockerfile image is based on buster which has postgres version 11 + "postgres:11", // Our Containerfile image is based on buster which has postgres version 11 ]) .spawn() .expect("failed to start a postgres instance"); @@ -126,3 +129,27 @@ impl Drop for PostgresInstance { .expect("postgres container remove failed"); } } + +pub fn build_so_create_loader(resources: &str, crate_name: &str) -> Result { + let crate_dir: PathBuf = [resources, crate_name].iter().collect(); + + Command::new("cargo") + .args(["build", "--release"]) + .current_dir(&crate_dir) + .spawn() + .unwrap() + .wait() + .unwrap(); + + let dashes_replaced = crate_name.replace('-', "_"); + + let lib_name = if cfg!(target_os = "windows") { + format!("{}.dll", dashes_replaced) + } else { + format!("lib{}.so", dashes_replaced) + }; + + let so_path = crate_dir.join("target/release").join(lib_name); + + Loader::from_so_file(&so_path) +} diff --git a/service/tests/loader.rs b/service/tests/loader.rs index 7043f2b06..f92716451 100644 --- a/service/tests/loader.rs +++ b/service/tests/loader.rs @@ -1,16 +1,20 @@ +mod helpers; + +use helpers::{build_so_create_loader, PostgresInstance}; + +use shuttle_service::loader::LoaderError; +use shuttle_service::{Error, Factory}; + use std::net::{Ipv4Addr, SocketAddr}; -use std::process::{exit, Command}; -use std::sync::mpsc; +use std::process::exit; use std::time::Duration; -mod helpers; - use async_trait::async_trait; -use helpers::PostgresInstance; -use shuttle_service::loader::{Loader, LoaderError}; -use shuttle_service::{Error, Factory}; +use tokio::sync::mpsc; use uuid::Uuid; +const RESOURCES_PATH: &str = "tests/resources"; + struct DummyFactory { postgres_instance: Option, } @@ -43,38 +47,18 @@ impl Factory for DummyFactory { #[test] fn not_shuttle() { - Command::new("cargo") - .args(["build", "--release"]) - .current_dir("tests/resources/not-shuttle") - .spawn() - .unwrap() - .wait() - .unwrap(); - - let result = - Loader::from_so_file("tests/resources/not-shuttle/target/release/libnot_shuttle.so"); - + let result = build_so_create_loader(RESOURCES_PATH, "not-shuttle"); assert!(matches!(result, Err(LoaderError::GetEntrypoint(_)))); } #[tokio::test] async fn sleep_async() { - Command::new("cargo") - .args(["build", "--release"]) - .current_dir("tests/resources/sleep-async") - .spawn() - .unwrap() - .wait() - .unwrap(); - - let loader = - Loader::from_so_file("tests/resources/sleep-async/target/release/libsleep_async.so") - .unwrap(); + let loader = build_so_create_loader(RESOURCES_PATH, "sleep-async").unwrap(); let mut factory = DummyFactory::new(); let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8001); let deployment_id = Uuid::new_v4(); - let (tx, _rx) = mpsc::sync_channel(1); + let (tx, _rx) = mpsc::unbounded_channel(); let (handler, _) = loader .load(&mut factory, addr, tx, deployment_id) .await @@ -96,20 +80,12 @@ async fn sleep_async() { #[tokio::test] async fn sleep() { - Command::new("cargo") - .args(["build", "--release"]) - .current_dir("tests/resources/sleep") - .spawn() - .unwrap() - .wait() - .unwrap(); - - let loader = Loader::from_so_file("tests/resources/sleep/target/release/libsleep.so").unwrap(); + let loader = build_so_create_loader(RESOURCES_PATH, "sleep").unwrap(); let mut factory = DummyFactory::new(); let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8001); let deployment_id = Uuid::new_v4(); - let (tx, _rx) = mpsc::sync_channel(1); + let (tx, _rx) = mpsc::unbounded_channel(); let (handler, _) = loader .load(&mut factory, addr, tx, deployment_id) .await @@ -131,16 +107,7 @@ async fn sleep() { #[tokio::test] async fn sqlx_pool() { - Command::new("cargo") - .args(["build", "--release"]) - .current_dir("tests/resources/sqlx-pool") - .spawn() - .unwrap() - .wait() - .unwrap(); - - let loader = - Loader::from_so_file("tests/resources/sqlx-pool/target/release/libsqlx_pool.so").unwrap(); + let loader = build_so_create_loader(RESOURCES_PATH, "sqlx-pool").unwrap(); // Don't initialize a pre-existing PostgresInstance here because the `PostgresInstance::wait_for_connectable()` // code has `awaits` and we want to make sure they do not block inside `Service::build()`. @@ -150,7 +117,7 @@ async fn sqlx_pool() { let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8001); let deployment_id = Uuid::new_v4(); - let (tx, rx) = mpsc::sync_channel(32); + let (tx, mut rx) = mpsc::unbounded_channel(); let (handler, _) = loader .load(&mut factory, addr, tx, deployment_id) .await @@ -158,7 +125,7 @@ async fn sqlx_pool() { handler.await.unwrap().unwrap(); - let log = rx.recv().unwrap(); + let log = rx.recv().await.unwrap(); assert_eq!(log.deployment_id, deployment_id); assert!( log.item.body.starts_with("/* SQLx ping */"), @@ -168,3 +135,35 @@ async fn sqlx_pool() { assert_eq!(log.item.target, "sqlx::query"); assert_eq!(log.item.level, log::Level::Info); } + +#[tokio::test] +async fn build_panic() { + let loader = build_so_create_loader(RESOURCES_PATH, "build-panic").unwrap(); + + let mut factory = DummyFactory::new(); + let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8001); + let deployment_id = Uuid::new_v4(); + let (tx, _) = mpsc::unbounded_channel(); + + if let Err(Error::BuildPanic(msg)) = loader.load(&mut factory, addr, tx, deployment_id).await { + assert_eq!(&msg, "panic in build"); + } else { + panic!("expected `Err(Error::BuildPanic(_))`"); + } +} + +#[tokio::test] +async fn bind_panic() { + let loader = build_so_create_loader(RESOURCES_PATH, "bind-panic").unwrap(); + + let mut factory = DummyFactory::new(); + let addr = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8001); + let deployment_id = Uuid::new_v4(); + let (tx, _) = mpsc::unbounded_channel(); + + if let Err(Error::BindPanic(msg)) = loader.load(&mut factory, addr, tx, deployment_id).await { + assert_eq!(&msg, "panic in bind"); + } else { + panic!("expected `Err(Error::BindPanic(_))`"); + } +} diff --git a/service/tests/resources/bind-panic/Cargo.toml b/service/tests/resources/bind-panic/Cargo.toml new file mode 100644 index 000000000..36ee66653 --- /dev/null +++ b/service/tests/resources/bind-panic/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "bind-panic" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[workspace] + +[dependencies] +shuttle-service = { path = "../../../" } +async-trait = "0.1.56" diff --git a/service/tests/resources/bind-panic/src/lib.rs b/service/tests/resources/bind-panic/src/lib.rs new file mode 100644 index 000000000..69c9cb4e4 --- /dev/null +++ b/service/tests/resources/bind-panic/src/lib.rs @@ -0,0 +1,31 @@ +use async_trait::async_trait; + +use shuttle_service::{IntoService, ServeHandle, Service}; + +#[macro_use] +extern crate shuttle_service; + +#[derive(Default)] +struct Builder; + +impl IntoService for Builder { + type Service = MyService; + + fn into_service(self) -> Self::Service { + MyService + } +} + +struct MyService; + +#[async_trait] +impl Service for MyService { + fn bind( + &mut self, + _: std::net::SocketAddr, + ) -> Result { + panic!("panic in bind"); + } +} + +declare_service!(Builder, Builder::default); diff --git a/service/tests/resources/build-panic/Cargo.toml b/service/tests/resources/build-panic/Cargo.toml new file mode 100644 index 000000000..55feae7ab --- /dev/null +++ b/service/tests/resources/build-panic/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "build-panic" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[workspace] + +[dependencies] +shuttle-service = { path = "../../../" } +async-trait = "0.1.56" +log = "0.4.17" diff --git a/service/tests/resources/build-panic/src/lib.rs b/service/tests/resources/build-panic/src/lib.rs new file mode 100644 index 000000000..e6f89e0c8 --- /dev/null +++ b/service/tests/resources/build-panic/src/lib.rs @@ -0,0 +1,45 @@ +use async_trait::async_trait; + +use shuttle_service::{Error, Factory, IntoService, Runtime, ServeHandle, Service}; + +#[macro_use] +extern crate shuttle_service; + +#[derive(Default)] +struct Builder; + +impl IntoService for Builder { + type Service = MyService; + + fn into_service(self) -> Self::Service { + MyService { + runtime: Runtime::new().unwrap(), + } + } +} + +struct MyService { + runtime: Runtime, +} + +#[async_trait] +impl Service for MyService { + async fn build( + &mut self, + _factory: &mut dyn Factory, + _logger: Box, + ) -> Result<(), Error> { + panic!("panic in build"); + } + + fn bind( + &mut self, + _: std::net::SocketAddr, + ) -> Result { + let handle = self.runtime.spawn(async move { Ok(()) }); + + Ok(handle) + } +} + +declare_service!(Builder, Builder::default); diff --git a/service/tests/resources/not-cdylib/Cargo.toml b/service/tests/resources/not-cdylib/Cargo.toml new file mode 100644 index 000000000..2bfd617f7 --- /dev/null +++ b/service/tests/resources/not-cdylib/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "not-cdynlib" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[workspace] + +[dependencies] diff --git a/service/tests/resources/not-cdylib/src/lib.rs b/service/tests/resources/not-cdylib/src/lib.rs new file mode 100644 index 000000000..212b945d0 --- /dev/null +++ b/service/tests/resources/not-cdylib/src/lib.rs @@ -0,0 +1,4 @@ +// This will fail to compile since it is missing the following section in its Cargo.toml +// +// [lib] +// crate-type = ["cdylib"] diff --git a/service/tests/resources/sqlx-pool/src/lib.rs b/service/tests/resources/sqlx-pool/src/lib.rs index 1c6d8b5b1..18adbf674 100644 --- a/service/tests/resources/sqlx-pool/src/lib.rs +++ b/service/tests/resources/sqlx-pool/src/lib.rs @@ -1,5 +1,5 @@ use shuttle_service::error::CustomError; -use shuttle_service::{GetResource, IntoService, Logger, Runtime, ServeHandle, Service}; +use shuttle_service::{log, GetResource, IntoService, Runtime, ServeHandle, Service}; use sqlx::PgPool; #[macro_use] @@ -53,11 +53,11 @@ impl Service for PoolService { async fn build( &mut self, factory: &mut dyn shuttle_service::Factory, - logger: Logger, + logger: Box, ) -> Result<(), shuttle_service::Error> { self.runtime .spawn_blocking(move || { - shuttle_service::log::set_boxed_logger(Box::new(logger)) + shuttle_service::log::set_boxed_logger(logger) .map(|()| { shuttle_service::log::set_max_level(shuttle_service::log::LevelFilter::Info) }) diff --git a/shell.nix b/shell.nix index 6c4d5ea98..04b6c118f 100644 --- a/shell.nix +++ b/shell.nix @@ -1,6 +1,7 @@ let moz_overlay = import (builtins.fetchTarball https://github.com/mozilla/nixpkgs-mozilla/archive/master.tar.gz); - nixpkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/e7d63bd0d50df412f5a1d8acfa3caae75522e347.tar.gz") { overlays = [ moz_overlay ]; }; + # Pin to stable from https://status.nixos.org/ + nixpkgs = import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/9bc0e974545d5bc4c24e1ed047be0dc4e30e494b.tar.gz") { overlays = [ moz_overlay ]; }; in with nixpkgs; stdenv.mkDerivation { @@ -15,5 +16,13 @@ in cargo-watch terraform awscli2 + websocat + protobuf + grpcurl + gh + docker-compose ]; + + PROTOC = "${protobuf}/bin/protoc"; + PROTOC_INCLUDE="${protobuf}/include"; } diff --git a/terraform/modules/shuttle/locals.tf b/terraform/modules/shuttle/locals.tf index a36d29e4f..e026037b9 100644 --- a/terraform/modules/shuttle/locals.tf +++ b/terraform/modules/shuttle/locals.tf @@ -1,6 +1,7 @@ locals { - data_dir = "/opt/shuttle" - docker_image = "public.ecr.aws/shuttle/backend" + data_dir = "/opt/shuttle" + docker_backend_image = "public.ecr.aws/shuttle/backend" + docker_provisioner_image = "public.ecr.aws/shuttle/provisioner" } resource "random_string" "initial_key" { diff --git a/terraform/modules/shuttle/misc/cloud-config.yaml b/terraform/modules/shuttle/misc/cloud-config.yaml index 2ca3fd2cb..7d45d9d56 100644 --- a/terraform/modules/shuttle/misc/cloud-config.yaml +++ b/terraform/modules/shuttle/misc/cloud-config.yaml @@ -14,6 +14,35 @@ system_info: default_user: groups: [docker] +# Make sure files are written every time +cloud_final_modules: + - package-update-upgrade-install + - [runcmd, always] + - [write-files, always] + - fan + - landscape + - lxd + - ubuntu-drivers + - write-files-deferred + - puppet + - chef + - mcollective + - salt-minion + - reset_rmc + - refresh_rmc_and_interface + - rightscale_userdata + - scripts-vendor + - scripts-per-once + - scripts-per-boot + - scripts-per-instance + - [scripts-user, always] + - ssh-authkey-fingerprints + - keys-to-console + - install-hotplug + - phone-home + - final-message + - power-state-change + # Create our systemd files write_files: - encoding: b64 @@ -26,11 +55,19 @@ write_files: path: /lib/systemd/system/shuttle-backend.service owner: root:root permissions: "0644" + - encoding: b64 + content: ${shuttle_provisioner_content} + path: /lib/systemd/system/shuttle-provisioner.service + owner: root:root + permissions: "0644" power_state: mode: reboot # Up services on every boot runcmd: + - docker network inspect shuttle-net || docker network create --driver bridge shuttle-net + - [systemctl, daemon-reload] - [systemctl, enable, "opt-shuttle.mount"] - [systemctl, enable, "shuttle-backend.service"] + - [systemctl, enable, "shuttle-provisioner.service"] diff --git a/terraform/modules/shuttle/service.tf b/terraform/modules/shuttle/service.tf index b511f939f..6921f5154 100644 --- a/terraform/modules/shuttle/service.tf +++ b/terraform/modules/shuttle/service.tf @@ -62,7 +62,7 @@ data "aws_ami" "ubuntu" { filter { name = "name" - values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-*"] + values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-arm64-server-20220511"] } filter { @@ -111,13 +111,20 @@ locals { "${path.module}/systemd/system/shuttle-backend.service.tftpl", { data_dir = local.data_dir, - docker_image = local.docker_image, - pg_password = var.postgres_password, + docker_image = local.docker_backend_image, shuttle_admin_secret = var.shuttle_admin_secret, proxy_fqdn = var.proxy_fqdn, shuttle_initial_key = random_string.initial_key.result } ) + shuttle_provisioner_content = templatefile( + "${path.module}/systemd/system/shuttle-provisioner.service.tftpl", + { + data_dir = local.data_dir, + docker_image = local.docker_provisioner_image, + pg_password = var.postgres_password, + } + ) } data "cloudinit_config" "backend" { @@ -129,8 +136,9 @@ data "cloudinit_config" "backend" { content = templatefile( "${path.module}/misc/cloud-config.yaml", { - opt_shuttle_content = base64encode(local.opt_shuttle_content), - shuttle_backend_content = base64encode(local.shuttle_backend_content) + opt_shuttle_content = base64encode(local.opt_shuttle_content), + shuttle_backend_content = base64encode(local.shuttle_backend_content) + shuttle_provisioner_content = base64encode(local.shuttle_provisioner_content) } ) filename = "cloud-config.yaml" diff --git a/terraform/modules/shuttle/systemd/system/shuttle-backend.service.tftpl b/terraform/modules/shuttle/systemd/system/shuttle-backend.service.tftpl index 0d4e85a6f..a199e7fd3 100644 --- a/terraform/modules/shuttle/systemd/system/shuttle-backend.service.tftpl +++ b/terraform/modules/shuttle/systemd/system/shuttle-backend.service.tftpl @@ -2,6 +2,7 @@ Description=maintains the shuttle Backend API After=docker.socket After=opt-shuttle.mount +After=shuttle-provisioner.service [Service] Type=simple @@ -10,21 +11,20 @@ RestartSec=30 Restart=always ExecStartPre=/usr/bin/docker pull ${docker_image}:latest ExecStart=/usr/bin/docker run --rm \ + --network shuttle-net \ + --name backend \ -e CRATES_PATH=/opt/shuttle/crates \ -e PROXY_PORT=8000 \ -p 8000:8000 \ -e API_PORT=8001 \ -p 8001:8001 \ - -e PG_PORT=5432 \ - -p 5432:5432 \ -e PROXY_FQDN=${proxy_fqdn} \ -e SHUTTLE_USERS_TOML=/opt/shuttle/users/users.toml \ - -e PG_PASSWORD=${pg_password} \ -e SHUTTLE_ADMIN_SECRET=${shuttle_admin_secret} \ -e SHUTTLE_INITIAL_KEY=${shuttle_initial_key} \ - -e PG_DATA=/opt/shuttle/postgres \ - -v ${data_dir}/user-data:/opt/shuttle:rw \ - -v ${data_dir}/conf/postgres:/etc/postgresql/11/shuttle:rw \ + -e PROVISIONER_ADDRESS=provisioner \ + -v ${data_dir}/user-data/crates:/opt/shuttle/crates:rw \ + -v ${data_dir}/user-data/users:/opt/shuttle/users:rw \ ${docker_image}:latest [Install] diff --git a/terraform/modules/shuttle/systemd/system/shuttle-provisioner.service.tftpl b/terraform/modules/shuttle/systemd/system/shuttle-provisioner.service.tftpl new file mode 100644 index 000000000..bd212375d --- /dev/null +++ b/terraform/modules/shuttle/systemd/system/shuttle-provisioner.service.tftpl @@ -0,0 +1,25 @@ +[Unit] +Description=maintains the shuttle provisioner which creates and manages resources +After=docker.socket +After=opt-shuttle.mount + +[Service] +Type=simple +User=ubuntu +RestartSec=30 +Restart=always +ExecStartPre=/usr/bin/docker pull ${docker_image}:latest +ExecStart=/usr/bin/docker run --rm \ + --network shuttle-net \ + --name provisioner \ + -e PG_PORT=5432 \ + -p 5432:5432 \ + -e PORT=5001 \ + -e PG_PASSWORD=${pg_password} \ + -e PG_DATA=/opt/shuttle/postgres \ + -v ${data_dir}/conf/postgres:/etc/postgresql/11/shuttle:rw \ + -v ${data_dir}/user-data/postgres:/opt/shuttle/postgres:rw \ + ${docker_image}:latest + +[Install] +WantedBy=multi-user.target diff --git a/www/_blog/2022-04-27-dev-log-1.mdx b/www/_blog/2022-04-27-dev-log-1.mdx index eff4f7bc5..b294fdce5 100644 --- a/www/_blog/2022-04-27-dev-log-1.mdx +++ b/www/_blog/2022-04-27-dev-log-1.mdx @@ -14,7 +14,12 @@ date: "2022-04-27T15:00:00" IFC uses application code as the source of truth for provisioning infrastructure. No longer are your applications and servers decoupled, the two go hand in hand. shuttle does this by doing static analysis of user code and generating the corresponding infrastructure in real time. A bit like this:

- +

In the [previous DevLog](https://www.shuttle.rs/blog/2022/04/22/dev-log-0) we started the journey of building the shuttle MVP. We went over the design and implementation of the `cargo` subcommand which deploys cargo projects to shuttle. This has been a race against the clock, so corners were cut and tradeoffs were made. A similar theme emerges in this DevLog which covers the **deployment state machine**. We're going to think about compiling and deploying user code, while also covering one of my favourite design patterns in Rust. @@ -26,15 +31,16 @@ shuttle exposes an HTTP endpoint under `POST /deploy`. This endpoint receives a The aim of the game, is to convert that series of bytes into a deployed web service - how do we go about doing that? The deployment process is broken into 4 stages: -1. `Queued` - the cargo project is received and waiting to be compiled -2. `Built` - the cargo project is compiled successfully -3. `Loaded` - the output of the compilation is loaded as a dynamically-linked library + +1. `Queued` - the cargo project is received and waiting to be compiled +2. `Built` - the cargo project is compiled successfully +3. `Loaded` - the output of the compilation is loaded as a dynamically-linked library 4. `Deployed` - the app inside the DLL is running and listening for connections Then life happens so you need a couple more states: -5. `Error` - there was an issue anywhere in the build process -6. `Deleted` - user-initiated deletion of the deploymentThis endpoint +5. `Error` - there was an issue anywhere in the build process +6. `Deleted` - user-initiated deletion of the deploymentThis endpoint Which corresponds to: @@ -63,7 +69,7 @@ struct QueuedState { } ``` -When a deployment is queued, the shuttle build system writes the `crate_bytes` (just a tarball of a cargo project) to the file system. It then extracts the tarball and starts the compilation process by running `cargo::ops::compile`. +When a deployment is queued, the shuttle build system writes the `crate_bytes` (just a tarball of a cargo project) to the file system. It then extracts the tarball and starts the compilation process by running `cargo::ops::compile`. The output of the build process is an `.so` file which is held in the next stage - the `BuildState`: @@ -75,14 +81,14 @@ struct BuiltState { So far so good. At this point we have a pointer to a compiled shared object file - next we need to load it into memory. - `shuttle` uses the [`libloading`](https://crates.io/crates/libloading) crate to dynamically load from a `.so` file a value of a type implementing the [`Service`](https://docs.rs/shuttle-service/0.2.6/shuttle_service/trait.Service.html) trait. The `Service` trait is code-generated for the user via the `#[shuttle_service::main]` annotation and it's how shuttle interfaces with client apps. +`shuttle` uses the [`libloading`](https://crates.io/crates/libloading) crate to dynamically load from a `.so` file a value of a type implementing the [`Service`](https://docs.rs/shuttle-service/0.2.6/shuttle_service/trait.Service.html) trait. The `Service` trait is code-generated for the user via the `#[shuttle_service::main]` annotation and it's how shuttle interfaces with client apps. - ```rust - pub struct LoadedState { - service: Box, - so: Library, - } - ``` +```rust +pub struct LoadedState { + service: Box, + so: Library, +} +``` We keep the `Library` struct around since `Box` is just a pointer to data loaded and managed by `Library`. Library going out of scope deallocates that data; meaning service will be pointing to deallocated memory hence we get a `segfault`. So it's important to keep `Library` around for the lifetime of the deployment. diff --git a/www/_blog/2022-05-09-ifc.mdx b/www/_blog/2022-05-09-ifc.mdx index c8cde8819..25940bd07 100644 --- a/www/_blog/2022-05-09-ifc.mdx +++ b/www/_blog/2022-05-09-ifc.mdx @@ -14,14 +14,12 @@ In the early days of Facebook (back when it was still called `thefacebook.com`), Heroku became part of the cloud-native lore as the first incredibly successful attempt at tackling this complexity. They led the first crusade to rid software developers of the infrastructure complexity dragon. People loved it. Heroku pioneered the wildly popular container-based approach to deployment that abstracted away the burden of managing virtual machines. By being opinionated with the use of containers, Heroku was able to appeal to a broad set of customers looking to quickly build apps. Containers are mutually isolated processes, wired together by third-party configuration which does not belong in the application’s code base - this design choice results in a lack of elasticity and granular control of your system. This results in a conservative outlook of dealing with infrastructure, constantly over-provisioning and hence overpaying to account for potential future load. - -Furthermore, infrastructure is still treated separately from code - the two worlds live separately and don’t really know much about each other. There is much less wiring to do than with AWS for example, but what is left to do - and there’s a lot of it - you still have to do yourself. Heroku trades off AWS’s elasticity for ready-made building block components that are statically wired up together through a combination of CLI commands and dashboard operations. Of course, Heroku is limited by its founding principle: static containers as building blocks of applications. With Heroku, it is true you do not have to think about infrastructure - but only in the beginning. Once your application scales, your bills stack up and you’re left without a choice: go back to AWS. +Furthermore, infrastructure is still treated separately from code - the two worlds live separately and don’t really know much about each other. There is much less wiring to do than with AWS for example, but what is left to do - and there’s a lot of it - you still have to do yourself. Heroku trades off AWS’s elasticity for ready-made building block components that are statically wired up together through a combination of CLI commands and dashboard operations. Of course, Heroku is limited by its founding principle: static containers as building blocks of applications. With Heroku, it is true you do not have to think about infrastructure - but only in the beginning. Once your application scales, your bills stack up and you’re left without a choice: go back to AWS. ### The Serverless Conundrum We need to talk about serverless. Serverless (think AWS Lambda) was a new cloud computing execution model where machine allocation happens on-demand and the user is primarily abstracted away from the underlying servers. With it came a familiar promise - developers not needing to think about infrastructure at all. Despite its somewhat counterintuitive name (because, of course, there are always servers running somewhere), serverless sounds like a great ideal to strive towards. This is simple, developers want to spend as much time as possible on delivering business value by writing code, while companies would like to avoid spending fortunes on DevOps. This seems to be the holy grail, but there’s a catch. You might ask, “if you say serverless is so great, why have we all not switched yet”? - Well, serverless forces you to write application business logic as functions, rather than the more traditional idiom of stateful processes. To reap the benefits of serverless, you have to build your application as a multitude of stateless request or event handlers, often requiring a bottoms-up redesign of your system. For some use-cases the serverless paradigm works, but in many cases breaking things into discrete, decoupled functions may not be optimal or even feasible. The next question is, can we have our cake and eat it too? Can we maintain the paradigm of stateful processes and abstract away the underlying infrastructure and orchestration? ### Infrastructure from Code @@ -49,7 +47,7 @@ This setup should also break the boundaries that keep containers isolated from e When looking back at Heroku’s success, it becomes apparent that focusing on one language, Ruby, which was becoming quite popular at the time - was a remarkable strategy. It enabled their team to focus acutely and produce an unparalleled experience for their users. -At shuttle we are convinced Rust is the best language to start this journey with. It’s been [the most loved](https://www.cantorsparadise.com/the-most-loved-programming-language-in-the-world-5220475fcc22) language by developers for many years in a row (as well as one of the fastest-growing languages). If you want to create the best developer experience - it makes sense to start with the most loved language. Indeed, Rust is the first language packed with such a powerful set of tools for static analysis and code generation, that are required to create the best developer experience when it comes to *Infrastructure ~~as~~ from Code*. +At shuttle we are convinced Rust is the best language to start this journey with. It’s been [the most loved](https://www.cantorsparadise.com/the-most-loved-programming-language-in-the-world-5220475fcc22) language by developers for many years in a row (as well as one of the fastest-growing languages). If you want to create the best developer experience - it makes sense to start with the most loved language. Indeed, Rust is the first language packed with such a powerful set of tools for static analysis and code generation, that are required to create the best developer experience when it comes to _Infrastructure ~~as~~ from Code_. Removing the burden of dealing with DevOps from developers, many of whom find it daunting and stressful, not only do we stand to make development more enjoyable and efficient, but also enable far more people to write and ship applications. @@ -59,4 +57,4 @@ Our community is just as important to us, as our vision is, so if any of this re Or check out our [jobs board](https://www.workatastartup.com/companies/shuttle). -Also, if you’re curious to learn more about *how* we are building this - [check out our GitHub](https://github.com/getsynth/shuttle). +Also, if you’re curious to learn more about _how_ we are building this - [check out our GitHub](https://github.com/getsynth/shuttle). diff --git a/www/_blog/2022-06-01-hyper-vs-rocket.mdx b/www/_blog/2022-06-01-hyper-vs-rocket.mdx new file mode 100644 index 000000000..386549f84 --- /dev/null +++ b/www/_blog/2022-06-01-hyper-vs-rocket.mdx @@ -0,0 +1,251 @@ +--- +title: Hyper vs Rocket - Low Level vs Batteries included +description: A comparison of using the low-level HTTP framework 'hyper' vs a batteries included framework like 'Rocket' +author: ben +tags: [rust, rocket, hyper, comparison] +thumb: hyper-vs-rocket.png +cover: hyper-vs-rocket.png +date: "2022-05-09T15:00:00" +--- + +In this post we're going to be comparing two popular Rust libraries used for building web applications. We'll be writing an example in each and compare their ergonomics and how they perform. + +The first library [Hyper](https://github.com/hyperium/hyper) is a low level HTTP library which contains the primitives for building server applications. The second library [Rocket](https://rocket.rs/) comes with more "batteries included" and provides a more declarative approach to building web applications. + +## The Demo + +We're going to build a simple site to showcase how each libraries implements: + +### Routing + +Routing decides what to respond for a given URL. Some paths are fixed, in our example we will have a fixed route `/` which returns `Hello World`. Some paths are dynamic and can have parameters. In the example we will have `/hello/*name*` which will response `Hello *name*` which will have *name* substituted in each response. + +### Shared state + +We want to have a central state for the application. + +In this demo we will have central site visitor counter which counts the number of requests. This number can be viewed as JSON on the `/counter.json` route. In this example we will be storing the counter in application memory. However if you were storing it in a database the shared state would be a database client. + +The are lots of other functionality necessary for a site such as handling HTTP methods, receiving data, rendering templates and error handling. But for the scope of this post and example we will only be comparing these two features. + +### The rules + +The rules for this demonstration is to only use the specific library and any of its re-exported dependencies. So no additional libraries (except in the hyper example we need a `tokio::main`). + +## Hyper + +Hyper's readme describes hyper as a "A fast and correct HTTP implementation for Rust with client and server APIs". For this demo we will be using the server side of the library. It has **9.7k** stars on GitHub and **48M** crates downloads. It is used as a often a dependency and many other libraries such as [reqwest and tonic](https://crates.io/crates/hyper/reverse_dependencies) build on top of it. + +In this example we see how far we can get with just using the library. This demo uses Hyper 0.14[^hyper-deps]. Below is the full code for the site: + +```rust +use hyper::server::conn::AddrStream; +use hyper::service::{make_service_fn, service_fn}; +use hyper::{Body, Request, Response, Server}; +use std::convert::Infallible; +use std::sync::{atomic::AtomicUsize, Arc}; + +#[derive(Clone)] +struct AppContext { + pub counter: Arc, +} + +async fn handle(context: AppContext, req: Request) -> Result, Infallible> { + // Increment the visit count + let new_count = context + .counter + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + + if req.method().as_str() != "GET" { + return Ok(Response::builder().status(406).body(Body::empty()).unwrap()); + } + + let path = req.uri().path(); + let response = if path == "/" { + Response::new(Body::from("Hello World")) + } else if path == "/counter.json" { + let data = format!("{{\"counter\":{}}}", new_count); + Response::builder() + .header("Content-Type", "application/json") + .body(Body::from(data)) + .unwrap() + } else if let Some(name) = path.strip_prefix("/hello/") { + Response::new(Body::from(format!("Hello, {}!", name))) + } else { + Response::builder().status(404).body(Body::empty()).unwrap() + }; + Ok(response) +} + +#[tokio::main] +async fn main() { + let context = AppContext { + counter: Arc::new(AtomicUsize::new(0)), + }; + + let make_service = make_service_fn(move |_conn: &AddrStream| { + let context = context.clone(); + let service = service_fn(move |req| handle(context.clone(), req)); + async move { Ok::<_, Infallible>(service) } + }); + + let server = Server::bind(&"127.0.0.1:3000".parse().unwrap()) + .serve(make_service) + .await; + + if let Err(e) = server { + eprintln!("server error: {}", e); + } +} +``` + +At the top we define a `handle` function which processes all the requests. + +Routing is done through the chain of ifs and elses in the `handle` function. First the path of the request (e.g `/` for the index) is extracted using `req.uri().path()`. Fixed routes are easy to branch on using string comparison like `path == "/"`. For routes which match multiple paths such as the `/hello/` route it uses [`str::strip_prefix`](https://doc.rust-lang.org/std/primitive.str.html#method.strip_prefix) which returns a `None` if the path doesn't +start with the prefix or `Some` if the path starts with the prefix along with a slice that proceeds the prefix. + +```rust +"/".strip_prefix("/hello/") == None +"/test".strip_prefix("/hello/") == None +"/hello/jack".strip_prefix("/hello/") == Some("jack") +``` + +The function has a early return for requests with a method other than GET because there are no POST routes or others for this example. If the site accepted different requests types and had to add additional guards then we could additional clauses to the if statement. Although you could see how expand on the if chain would get more complex and verbose. + +To return a response, Hyper re-exports [`Response`](https://docs.rs/hyper/0.14.19/hyper/struct.Response.html) (from the [http crate](https://docs.rs/http/latest/http/)). It has a nice simple builder pattern for building the responses. The serialization code is hand written using `format!`. Of course we could import serde but that's against the rules. + +The counter is done by creating a struct in the initializing code and cloning it on every request to send to the handler function. Without going into the details it uses `Arc` instead of a `usize` as the atomic variant has special properties for when multiple handlers are using and mutating it. The code increments the visitor counter before anything else in the handler function so that a visit is recorded for all requests. + +### Hyper Verdict + +In terms of development (on a low end machine we used for profiling[^profile-machine]), a debug build (without any of the build artifacts) takes **79.0s**. After the initial compilation, incremental compilation takes only **1.9s**. For building a `release` build with further optimizations (on top of the debug build artifacts) it takes **32.5s**. + +The initialization code was take from [Hyper's server docs](https://docs.rs/hyper/latest/hyper/server/index.html) and is quite verbose and out of the box for Hyper there are no logs or server information. + +In terms of runtime performance over three 30 second connections Hyper responded to on average **74,563** requests per second on the index route on the above code. Which is incredible quick! + +## Rocket + +Rocket is a "web framework for Rust with a focus on ease-of-use, expressibility, and speed". It has **17.4k** github stars and **1.7M** crates downloads. Rocket internally uses Hyper. + +For this demo we are using the `0.5.0-rc2` version of Rocket[^rocket-deps] which builds on Rust stable. + +```rust +use rocket::{ + fairing::{Fairing, Info, Kind}, + get, launch, routes, + serde::{json::Json, Serialize}, + Config, Data, Request, State, +}; +use std::sync::atomic::AtomicUsize; + +#[derive(Serialize, Default)] +#[serde(crate = "rocket::serde")] +struct AppContext { + pub counter: AtomicUsize, +} + +#[launch] +fn rocket() -> _ { + let config = Config { + port: 3000, + ..Config::debug_default() + }; + + rocket::custom(&config) + .attach(CounterFairing) + .manage(AppContext::default()) + .mount("/", routes![hello1, hello2, counter]) +} + +struct CounterFairing; + +#[rocket::async_trait] +impl Fairing for CounterFairing { + fn info(&self) -> Info { + Info { + name: "Request Counter", + kind: Kind::Request, + } + } + + async fn on_request(&self, request: &mut Request<'_>, _: &mut Data<'_>) { + request + .rocket() + .state::() + .unwrap() + .counter + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } +} + +#[get("/")] +fn hello1() -> &'static str { + "Hello World" +} + +#[get("/hello/")] +fn hello2(name: &str) -> String { + format!("Hello, {}!", name) +} + +#[get("/counter.json")] +fn counter(state: &State) -> Json<&AppContext> { + Json(state.inner()) +} +``` + +In Rocket we describe each endpoint using a function. The `get` macro attribute handles path routing and http method constraint. No need to add early returns for methods and dealing with raw string slices. It takes the declarative approach, `#[get("/hello/")]` is more descriptive and less verbose than `if let Some(name) = path.strip_prefix("/hello/")`. The functions are registered using `.mount("/", routes![hello1, hello2, counter])`. + +The application has a state defined here: + +```rust +#[derive(Serialize, Default)] +#[serde(crate = "rocket::serde")] +struct AppContext { + pub counter: AtomicUsize, +} +``` + +And it is created and registered using `.manage(AppContext::default())`. Rocket re-exports the serialization library serde so we can use `#[derive(Serialize)]` to generate serialization logic for the counter state, so no hand writing the serialization code unlike first method. + +In Rocket endpoint functions can just return `String` or `str` slices and Rocket handles it automatically. Rocket also comes with a `Json` return type and reusing the fact that `AppContext` implements `Serialize` we can freely build a `Json` response from it. The `Json` structure handles setting the `Content-Type` header automatically for us. + +Rocket has a middleware implementation which it calls ["fairings"](https://rocket.rs/v0.5-rc/guide/fairings/#fairings). In the example it defines a `CounterFairing` which on every request modifies the counter state. The initialization code is really slim, it sets up a config and a Rocket structure is created using a builder pattern. Annotating the main function with `#[launch]` helps Rocket find the entry point and abstracts how the server is span up. Rocket also has really nice built in logs which are great for development. + +

+ +

+ +### Rocket Verdict + +Since Rocket has more dependencies and requires more macro expansion it takes a bit longer build taking **141.9s** (2m 21.9s) on a cold start to compile. A release builds on top of debug artefact takes **147.0s** (2m 27.0s) to compile. Incremental builds are still fast taking **3.3s** to compile after a small change to the response of a endpoint. + +Using the same benchmark as Hyper, on average Rocket returned **43,899** requests per second in a release build with the logging disabled - roughly **60%** of Hyper's throughput. + +## Conclusion + +Writing both of theses examples were fun to build and there weren't any frustrations or problems using them. Both are plenty fast for performance to be a concern. + +Rockets documentation is very good and explanatory. All of Hyper's api is well documented on its [docs.rs page](https://docs.rs/hyper/latest/hyper/). Both libraries are actively developed with many commits and pull requests made in the last month. + +Do you prefer the control and speed of Hyper or prefer the expressiveness of Rocket? + +## [Shuttle](https://www.shuttle.rs/): Stateful Serverless for Rust + +Deploying and managing your Rust web apps can be an expensive, anxious and time consuming process. + +If you want a batteries included and ops-free experience, [try out Shuttle](https://docs.rs/shuttle-service/latest/shuttle_service/). + +
+ +[^profile-machine]: The build and request profile machine is a vm with 2 cores and 7 GB RAM. Take the numbers with a grain of salt + +[^hyper-deps]: The dependencies for building the project `hyper = { version = "0.14", features = ["server", "tcp", "http1"] }` and `tokio = { version = "1.18.2", features = ["rt", "macros", "rt-multi-thread"] }` + +[^rocket-deps]: The dependencies for building the project `rocket = { version = "0.5.0-rc.2", features = ["json"] }` diff --git a/www/_blog/2022-06-09-the-builder-pattern.mdx b/www/_blog/2022-06-09-the-builder-pattern.mdx new file mode 100644 index 000000000..590e327f3 --- /dev/null +++ b/www/_blog/2022-06-09-the-builder-pattern.mdx @@ -0,0 +1,495 @@ +--- +title: Builders in Rust +description: In this post we do a deep dive into the builder pattern - an easy way to write cleaner and more readable code. +author: ben +tags: [rust, tutorial] +thumb: crab-builder.png +cover: crab-builder.png +date: "2022-06-09T15:00:00" +--- +This blog post is powered by shuttle! The serverless platform built for Rust. + +In this post, we'll be going over the "builder pattern". The builder pattern is an API design pattern for constructing instances of Rust structures. We'll be going over where it makes sense to use it and some of the benefits of applying it to your structs. + +## Examples + +Here are some examples of the builder pattern in common Rust crates: + +[`Command`](https://doc.rust-lang.org/std/process/struct.Command.html) from the Rust standard library +```rust +Command::new("cmd") + .args(["/C", "echo hello"]) + .output() +``` + +[`Rocket`](https://api.rocket.rs/v0.5-rc/rocket/struct.Rocket.html) in Rocket +```rust +rocket::build() + .mount("/hello", routes![world]) + .launch() +``` + +[`Response`](https://docs.rs/http/latest/http/response/struct.Response.html#method.builder) in the HTTP crate +```rust +Response::builder() + .status(200) + .header("X-Custom-Foo", "Bar") + .header("Set-Cookie", "key=2") + .body(()) + .unwrap(); +``` + +[Cargo uses the pattern internally for tests](https://github.com/rust-lang/cargo/blob/c6745a3d7fcea3a949c3e13e682b8ddcbd213add/tests/testsuite/build.rs#L74-L91) + +Ok - so let's dive into *what* the builder pattern actually is. + +## What is the builder pattern + +Given the following struct representation: +```rust +struct Message { + from: String, + content: String, + attachment: Option +} +``` +Using struct initialization syntax: +```rust +Message { + from: "John Smith".into(), + content: "Hello!".into(), + attachment: None +} +``` +Using a builder pattern: +```rust +Message::builder() + .from("John Smith".into()) + .content("Hello!".into()) + .build() +``` + +The builder pattern consists of: +- A function that generates a *intermediate builder structure* (`Message::builder()`) +- A chain of methods which set values on the builder: (`.from("John Smith".into()).content("Hello!".into())`) +- A final method which builds the final value from the intermediate structure `.build()` + +The structure of the builder pattern follows the functional programming design and has likeness of building iterators. + +The setting methods take a mutable reference to the builder and return the same reference (thus for chaining to work). The handy part about working with mutable references is that it can be shared around between functions and if statements: + +```rust +fn build_message_from_console_input( + builder: &mut MessageBuilder +) -> Result<(), Box> { + let mut buffer = String::new(); + let mut stdin = std::io::stdin(); + stdin.read_line(&mut buffer).unwrap(); + + let split = buffer.rsplit_once("with attachment: "); + if let Some((message, attachment_path)) = split { + let attachment = + std::fs::read_to_string(attachment_path).unwrap(); + builder + .content(message.into()); + .attachment(attachment); + } else { + builder.text_filter(buffer); + } +} +``` + +Next we'll explore some places where the builder pattern can offer a lot of benefits. + +#### Constraints and computed data + +Given the following struct which represents running a certain function at a certain time: + +```rust +struct FutureRequest { + at: chrono::DateTime, + func: T +} +``` + +We don't want the program to be able to create a `FutureRequest` for a time in the past. + +With regular struct initialisation and public fields there isn't a good way to constrain the values being given to the struct[^type_constraints] + +```rust +let fq = FutureRequest { + at: chrono::DateTime::from_utc( + chrono::NaiveDate::from_ymd(-112, 2, 18) + .and_hms(11, 5, 6), + Utc + ), + func: || println!("𓅥𓃶𓀫"), +} +``` + +However with the builder pattern and a method for setting the time we can validate the value before it is assigned + +```rust +#[derive(Debug)] +struct SchedulingInPastError; + +impl ()> FutureRequestBuilder { + fn at( + &mut self, + date_time: chrono::DateTime + ) -> Result<&mut Self, SchedulingInPastError> { + if date_time < Utc::now() { + Err(SchedulingInPastError) + } else { + self.at = date_time; + Ok(self) + } + } +} +``` + +Maybe we don't even want an absolute time - but a relative time at some point in the future. + +```rust +impl ()> FutureRequestBuilder { + fn after(&mut self, duration: std::time::Duration) -> &mut Self { + self.at = Utc::now() + chrono::Duration::from_std(duration).unwrap(); + self + } +} +``` + +#### Encapsulating data + +Sometimes - we want to keep some fields hidden from the user: + +```rust +struct Query { + pub on_database: String, + // ... +} + +fn foo(query: &mut Query) { + // You want mutable access to call mutable methods on the query + // but want to prevent against: + query.on_database.drain(..); +} +``` + +So you could make the fields private and create a function which constructs the value (known as a constructor): + +```rust +impl Query { + fn new( + fields: Vec, + text_filter: String, + database: String, + table: String, + fixed_amount: Option, + descending: bool, + ) -> Self { + unimplemented!() + } +} + +let query = Query::new( + vec!["title".into()], + "Morbius 2".into(), + "imdb".into(), + "films".into(), + None, + false +); +``` +But this causes confusion at the call site. Its not clear whether "imdb" is the database, the table or the text_filter? [^vscode-inlay-hints]. + +The builder pattern makes it much easier to read and understand what's happening during initialisation: + +```rust +let query = Query::builder() + .fields(vec!["title".into()]), + .text_filter("Morbius 2".into()), + .database("imdb".into()), + .table("films".into()), + .fixed_amount(None), + .descending(false) + .build(); +``` + +#### Enums and nested data + +So far we've just discussed structs - let's talk about enums: + +```rust +enum HTMLNode { + Text(String), + Comment(String), + Element(HTMLElement) +} + +struct HTMLElement { + tag_name: String, + attributes: HashMap>, + children: Vec +} +``` + +Here there is builder associated with each variant: + +```rust +HTMLNode::text_builder() + .text("Some text".into()) + .build() + +// vs + +HTMLNode::Text("Some text".into()) + +// -- + +HTMLNode::element_builder() + .tag_name("p".into()) + .attribute("class".into(), "big quote".into()) + .attribute("tabindex".into(), "5".into()) + .content("Some text") + +// vs + +HTMLNode::Element(HTMLElement { + tag_name: "p".into(), + attributes: [ + ("class".into(), "big quote".into()), + ("tabindex".into(), "5".into()) + ].into_iter(), + children: vec![HTMLNode::Text("Some text".into())] +}) +``` + +## Building our own builder pattern + +Now let's build our own builders (no pun intended). In this example we have some users: + +```rust +#[derive(Debug)] +struct User { + username: String, + birthday: NaiveDate, +} + +struct UserBuilder { + username: Option, + birthday: Option, +} + +#[derive(Debug)] +struct InvalidUsername; + +#[derive(Debug)] +enum IncompleteUserBuild { + NoUsername, + NoCreatedOn, +} + +impl UserBuilder { + fn new() -> Self { + Self { + username: None, + birthday: None, + } + } + + fn set_username(&mut self, username: String) -> Result<&mut Self, InvalidUsername> { + // true if every character is number of lowercase letter in English alphabet + let valid = username + .chars() + .all(|chr| matches!(chr, 'a'..='z' | '0'..='9')); + + if valid { + self.username = Some(username); + Ok(self) + } else { + Err(InvalidUsername) + } + } + + fn set_birthday(&mut self, date: NaiveDate) -> &mut Self { + self.birthday = Some(date); + self + } + + fn build(&self) -> Result { + if let Some(username) = self.username.clone() { + if let Some(birthday) = self.birthday.clone() { + Ok(User { username, birthday }) + } else { + Err(IncompleteUserBuild::NoCreatedOn) + } + } else { + Err(IncompleteUserBuild::NoUsername) + } + } +} +``` + +Some things to look out for: +- Every set method must take a mutable reference in order to add the data to the backer +- The method must then return the mutable reference it has to allow for them to be chained. + +There are clones in the `build` method but if that method is only called once then it is optimized out by Rust. + +## Automatic approaches + +Similar to how Clone and Debug work, crates can create there own derive macros. [There are a lot of crates which can help with generating the builder pattern](https://lib.rs/keywords/builder). Let's take a look at a few: + +### [derive_builder](https://lib.rs/crates/derive_builder) + +```rust +#[derive(Debug, derive_builder::Builder)] +#[builder(build_fn(validate = "Self::validate"))] +struct Query { + fields: Vec, + text_filter: String, + database: String, + table: String, + fixed_amount: Option, + descending: bool, +} + +// Usage same as described patterns: +let query = Query::builder() + .table("...".into()) + // ... + .build() + .unwrap(); +``` + +This derive macro generates a new struct named the same as the original structure but postfixed with `Builder` (in this case `QueryBuilder`). + +Derive builder has the downside of a whole object validation rather than per field. As well as the error variant of construction being a `String`, which makes it harder to match on the error or return error data compared to a error enum: + +```rust +impl Query { + fn validate(&self) -> Result<(), String> { + let valid = self + .database + .as_ref() + .map(|value| value == "pg_roles") + .unwrap_or_default(); + + if valid { + Ok(()) + } else { + Err("Cannot construct Query on 'pg_roles'".into()) + } + } +} +``` + +### [typed-builder](https://lib.rs/crates/typed-builder) + +Typed-builder solves two problems with `derive_builder`: + +With `derive_builder` you can set a field twice (or more) +```rust +Query::builder() + .database("imdb".into()) + // ... + .database("fishbase".into()) +``` + +Which takes the value of the last set field which is likely a mistake. Although Rust can optimize out a write without a read it is very difficult to have a linter error for this mistake. `derive_builder` also delegates the check to whether all the required fields have been set to runtime. + +With `typed-builder` it has a very similar implementation but has a different output which Rust can reason about and check that they are no duplicate sets and the build is well formed (all the required fields have been set). + +The downside here is that it takes longer to expand the macros as there is more to generate. The added complexity also makes it more complicated to pass the builder around. + +### [Buildstructor](https://lib.rs/crates/buildstructor) + +Buildstructor is a annotation for an existing impl block. Rather than using the fields on a structure (as seen in the previous two) to generate code it builds wrappers around existing constructor functions: + +```rust +struct MyStruct { + sum: usize +} + +#[buildstructor::buildstructor] +impl MyStruct { + #[builder] + fn new(a: usize, b: usize) -> MyStruct { + Self { sum: a + b } + } +} + +MyStruct::builder().a(1).b(2).build(); +``` + +Similar to `typed-builder` it generates intermediate staging structs for building which has the benefits of compile time checking that all the fields exist. However that comes again with the drawback of slower compile time and less flexibility when passing it around. + +Typed builder looks to be more compatible with the Rust language which allows it to support async builders! It's definitely the more interesting one of the bunch and I will be looking to play with with it future projects. + +### Alternative patterns + +If you just want to build a struct which has a large amount of default fields, using `..` (base syntax) with the [Default](https://doc.rust-lang.org/std/default/trait.Default.html) trait (whether a custom implementation or the default one with `#[derive(Default)]`) will do: + +```rust +#[derive(Default)] +struct X { + a: u32, + b: i32, + c: bool, +} + +X { a: 10, ..Default::default() } +``` + +If you want computation, constraints, encapsulation and named fields you could create a intermediate struct which can be passed to a constructor: + +```rust +struct Report { + title: String, + on: chrono::DateTime + // ... +} + +struct ReportArguments { + title: String, + on: Option + // ... +} + +impl Report { + fn new_from_arguments(ReportArguments { title, on }: ReportArguments) -> Result { + if title. + .chars() + .all(|chr| matches!(chr, 'a'..='z' | '0'..='9')) + { + Ok(Self { + title, + on: chrono.unwrap_or_else(|| todo!()) + }) + } else { + Err("Invalid report name") + } + } +} +``` + +However both of these don't the use the nice chaining syntax. + +## Conclusion + +The builder pattern can help you write cleaner, more readable APIs, and it turn help the consumers of your APIs write better code. We can apply constraints to make sure that our structs are intialised correctly with a clean API enforcing the contract. + +One thing to remember is that code is read *much* more than it's written - so it's worth going out of our way to make our code just that little bit more pleasant to read. + +## [Shuttle](https://www.shuttle.rs/): Stateful Serverless for Rust + +Deploying and managing your Rust web apps can be an expensive, anxious and time consuming process. + +If you want a batteries included and ops-free experience, [try out Shuttle](https://docs.rs/shuttle-service/latest/shuttle_service/). + +
+ +[^type_constraints]: I partially agree with this, there are ways to design your types to be constrained. Here we could create a `struct FutureEvent(chrono::DateTime)` structure where the constraint is constructing the `FutureEvent` type rather than leaving the constraint to the field. But there are lots of scenarios where that isn't the case. + +[^vscode-inlay-hints]: With vscode and rust analyzer there is a feature called [inlay hints](https://rust-analyzer.github.io/manual.html#inlay-hints) which shows the names of parameters in the editor. While this is great this is a feature specific to vscode at the moment. You won't see the hints on GitHub diffs and in other text editors. diff --git a/www/components/ExternalLink.tsx b/www/components/ExternalLink.tsx index 35a1d0bf9..b244f681a 100644 --- a/www/components/ExternalLink.tsx +++ b/www/components/ExternalLink.tsx @@ -1,17 +1,22 @@ import mixpanel from "mixpanel-browser"; +interface Props { + readonly mixpanelEvent?: string; +} + export default function ExternalLink({ ref, href, target, rel, + mixpanelEvent, ...props -}: JSX.IntrinsicElements["a"]): JSX.Element { +}: JSX.IntrinsicElements["a"] & Props): JSX.Element { return ( { - el && mixpanel.track_links(el, `Clicked Link`); + el && mixpanel.track_links(el, mixpanelEvent ?? `Clicked Link`); }} target={target ?? "_blank"} rel={rel ?? "noopener noreferrer"} diff --git a/www/components/Footer.tsx b/www/components/Footer.tsx index aed55c9fb..f2019fc3a 100644 --- a/www/components/Footer.tsx +++ b/www/components/Footer.tsx @@ -114,6 +114,7 @@ export default function Footer() { key={index} href={community.href} className="inline-block rounded border border-current py-3 px-5 text-base font-medium text-slate-600 hover:text-slate-900 dark:text-gray-200 hover:dark:text-white" + mixpanelEvent={community.name} >