From a4d43c0ff8cf4db84a3c0f42cfb935fe5a137190 Mon Sep 17 00:00:00 2001 From: Rob Ede Date: Sun, 6 Feb 2022 08:19:35 +0000 Subject: [PATCH] fix db/basic --- .github/workflows/linux.yml | 9 ++- Cargo.lock | 102 +++++++++++++----------- basics/json-validation/src/main.rs | 2 +- basics/todo/src/main.rs | 2 +- database_interactions/basic/.gitignore | 2 + database_interactions/basic/Cargo.toml | 20 ++--- database_interactions/basic/src/db.rs | 31 ++++--- database_interactions/basic/src/main.rs | 41 +++++----- other/http-proxy/src/main.rs | 2 +- other/run-in-thread/src/main.rs | 2 +- security/awc_https/src/main.rs | 2 +- weather.db | 0 12 files changed, 112 insertions(+), 103 deletions(-) create mode 100644 database_interactions/basic/.gitignore create mode 100644 weather.db diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index fa6a584..817e988 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -39,10 +39,11 @@ jobs: - name: create test db for sqlx run: | sudo apt-get update && sudo apt-get install sqlite3 - cd database_interactions/sqlx_todo - cp .env.example .env - cat schema.sql | sqlite3 test.db - chmod a+rwx test.db + cargo install sqlx-cli --no-default-features --features=rustls,sqlite + cd basics/todo + sqlx database create + chmod a+rwx todo.db + sqlx migrate run - name: cargo check uses: actions-rs/cargo@v1 diff --git a/Cargo.lock b/Cargo.lock index d7ffba8..79252ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1258,9 +1258,8 @@ version = "1.0.0" dependencies = [ "actix-web 4.0.0-rc.2", "env_logger 0.9.0", - "failure", - "futures", - "num_cpus", + "futures-util", + "log", "r2d2", "r2d2_sqlite", "rusqlite", @@ -2256,7 +2255,7 @@ dependencies = [ "byteorder", "chrono", "diesel_derives", - "libsqlite3-sys", + "libsqlite3-sys 0.9.1", "pq-sys", "r2d2", "uuid 0.8.2", @@ -2371,6 +2370,9 @@ name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +dependencies = [ + "serde 1.0.136", +] [[package]] name = "encoding_rs" @@ -2503,6 +2505,18 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d04dafd11240188e146b6f6476a898004cace3be31d4ec5e08e216bf4947ac0" +dependencies = [ + "futures-core", + "futures-sink", + "pin-project 1.0.10", + "spin 0.9.2", +] + [[package]] name = "fnv" version = "1.0.7" @@ -3462,9 +3476,19 @@ checksum = "565dbd88872dbe4cc8a46e527f26483c1d1f7afa6b884a3bd6cd893d4f98da74" [[package]] name = "libsqlite3-sys" -version = "0.22.2" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290b64917f8b0cb885d9de0f9959fe1f775d7fa12f1da2db9001c1c8ab60f89d" +checksum = "0e9eb7b8e152b6a01be6a4a2917248381875758250dc3df5d46caf9250341dda" +dependencies = [ + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libsqlite3-sys" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2cafc7c74096c336d9d27145f7ebd4f4b6f95ba16aa5a282387267e6925cb58" dependencies = [ "cc", "pkg-config", @@ -4622,9 +4646,9 @@ dependencies = [ [[package]] name = "r2d2_sqlite" -version = "0.18.0" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d24607049214c5e42d3df53ac1d8a23c34cc6a5eefe3122acb2c72174719959" +checksum = "54ca3c9468a76fc2ad724c486a59682fc362efeac7b18d1c012958bc19f34800" dependencies = [ "r2d2", "rusqlite", @@ -5033,15 +5057,15 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.25.4" +version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c4b1eaf239b47034fb450ee9cdedd7d0226571689d8823030c4b6c2cb407152" +checksum = "4ba4d3462c8b2e4d7f4fcfcf2b296dc6b65404fbbc7b63daa37fd485c149daf7" dependencies = [ "bitflags", "fallible-iterator", "fallible-streaming-iterator", "hashlink", - "libsqlite3-sys", + "libsqlite3-sys 0.23.2", "memchr", "smallvec", ] @@ -5595,6 +5619,9 @@ name = "spin" version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5" +dependencies = [ + "lock_api", +] [[package]] name = "sqlformat" @@ -5609,9 +5636,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7911b0031a0247af40095838002999c7a52fba29d9739e93326e71a5a1bc9d43" +checksum = "692749de69603d81e016212199d73a2e14ee20e2def7d7914919e8db5d4d48b9" dependencies = [ "sqlx-core", "sqlx-macros", @@ -5619,9 +5646,9 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aec89bfaca8f7737439bad16d52b07f1ccd0730520d3bf6ae9d069fe4b641fb1" +checksum = "518be6f6fff5ca76f985d434f9c37f3662af279642acf730388f271dff7b9016" dependencies = [ "ahash", "atoi", @@ -5633,22 +5660,25 @@ dependencies = [ "crossbeam-queue", "crossbeam-utils 0.8.6", "either", + "flume", "futures-channel", "futures-core", + "futures-executor", "futures-intrusive", "futures-util", "hashlink", "hex", "indexmap", - "itoa 0.4.8", + "itoa 1.0.1", "libc", - "libsqlite3-sys", + "libsqlite3-sys 0.23.2", "log", "memchr", "once_cell", "parking_lot", "percent-encoding", "rustls 0.19.1", + "serde 1.0.136", "sha2 0.9.9", "smallvec", "sqlformat", @@ -5659,21 +5689,23 @@ dependencies = [ "url", "webpki 0.21.4", "webpki-roots 0.21.1", - "whoami", ] [[package]] name = "sqlx-macros" -version = "0.5.9" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "584866c833511b1a152e87a7ee20dee2739746f60c858b3c5209150bc4b466f5" +checksum = "38e45140529cf1f90a5e1c2e561500ca345821a1c513652c8f486bbf07407cc8" dependencies = [ "dotenv", "either", "heck 0.3.3", + "hex", "once_cell", "proc-macro2", "quote", + "serde 1.0.136", + "serde_json", "sha2 0.9.9", "sqlx-core", "sqlx-rt", @@ -5687,27 +5719,11 @@ version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8061cbaa91ee75041514f67a09398c65a64efed72c90151ecd47593bad53da99" dependencies = [ - "actix-rt 2.6.0", "once_cell", "tokio 1.16.1", "tokio-rustls 0.22.0", ] -[[package]] -name = "sqlx_todo" -version = "1.0.0" -dependencies = [ - "actix-web 4.0.0-rc.2", - "anyhow", - "dotenv", - "env_logger 0.9.0", - "futures", - "log", - "serde 1.0.136", - "serde_json", - "sqlx", -] - [[package]] name = "standback" version = "0.2.17" @@ -6124,13 +6140,13 @@ dependencies = [ "actix-files 0.6.0-beta.16", "actix-session 0.5.0-beta.7", "actix-web 4.0.0-rc.2", - "diesel", "dotenv", "env_logger 0.9.0", - "futures", + "futures-util", "log", "serde 1.0.136", "serde_json", + "sqlx", "tera", ] @@ -7070,16 +7086,6 @@ dependencies = [ "tokio-util 0.3.1", ] -[[package]] -name = "whoami" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524b58fa5a20a2fb3014dd6358b70e6579692a56ef6fce928834e488f42f65e8" -dependencies = [ - "wasm-bindgen", - "web-sys", -] - [[package]] name = "widestring" version = "0.4.3" diff --git a/basics/json-validation/src/main.rs b/basics/json-validation/src/main.rs index fe75c48..b7253b9 100644 --- a/basics/json-validation/src/main.rs +++ b/basics/json-validation/src/main.rs @@ -81,7 +81,7 @@ async fn create_something( async fn main() -> io::Result<()> { env_logger::init_from_env(env_logger::Env::new().default_filter_or("info")); - log::info!("starting HTTP serer at http://localhost:8080"); + log::info!("starting HTTP server at http://localhost:8080"); HttpServer::new(|| { App::new() diff --git a/basics/todo/src/main.rs b/basics/todo/src/main.rs index b216e3d..fd8857b 100644 --- a/basics/todo/src/main.rs +++ b/basics/todo/src/main.rs @@ -27,7 +27,7 @@ async fn main() -> io::Result<()> { .await .expect("Failed to create pool"); - log::info!("starting HTTP serer at http://localhost:8080"); + log::info!("starting HTTP server at http://localhost:8080"); HttpServer::new(move || { log::debug!("Constructing the App"); diff --git a/database_interactions/basic/.gitignore b/database_interactions/basic/.gitignore new file mode 100644 index 0000000..83cb498 --- /dev/null +++ b/database_interactions/basic/.gitignore @@ -0,0 +1,2 @@ +weather.db +weather.db-* diff --git a/database_interactions/basic/Cargo.toml b/database_interactions/basic/Cargo.toml index 2a664c5..10eadc3 100644 --- a/database_interactions/basic/Cargo.toml +++ b/database_interactions/basic/Cargo.toml @@ -4,13 +4,13 @@ version = "1.0.0" edition = "2021" [dependencies] -actix-web = "4.0.0-rc.1" -env_logger = "0.9.0" -failure = "0.1.7" -futures = "0.3.1" -num_cpus = "1.13" -r2d2 = "0.8.2" -r2d2_sqlite = "0.18.0" -rusqlite = "0.25.4" -serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +actix-web = "4.0.0-rc.2" + +env_logger = "0.9" +futures-util = { version = "0.3", default-features = false, features = ["std"] } +log = "0.4" +r2d2 = "0.8" +r2d2_sqlite = "0.19" +rusqlite = "0.26" +serde = { version = "1", features = ["derive"] } +serde_json = "1" diff --git a/database_interactions/basic/src/db.rs b/database_interactions/basic/src/db.rs index d8515bc..d21213d 100644 --- a/database_interactions/basic/src/db.rs +++ b/database_interactions/basic/src/db.rs @@ -1,5 +1,4 @@ -use actix_web::{error::InternalError, http::StatusCode, web}; -use failure::Error; +use actix_web::{error, web, Error}; use rusqlite::Statement; use serde::{Deserialize, Serialize}; use std::{thread::sleep, time::Duration}; @@ -22,26 +21,26 @@ pub enum Queries { GetTopTenColdestMonths, } -pub async fn execute( - pool: &Pool, - query: Queries, -) -> Result, InternalError> { +pub async fn execute(pool: &Pool, query: Queries) -> Result, Error> { let pool = pool.clone(); + + let conn = web::block(move || pool.get()) + .await? + .map_err(error::ErrorInternalServerError)?; + web::block(move || { // simulate an expensive query, see comments at top of main.rs sleep(Duration::from_secs(2)); - let result = match query { - Queries::GetTopTenHottestYears => get_hottest_years(pool.get()?), - Queries::GetTopTenColdestYears => get_coldest_years(pool.get()?), - Queries::GetTopTenHottestMonths => get_hottest_months(pool.get()?), - Queries::GetTopTenColdestMonths => get_coldest_months(pool.get()?), - }; - result.map_err(Error::from) + match query { + Queries::GetTopTenHottestYears => get_hottest_years(conn), + Queries::GetTopTenColdestYears => get_coldest_years(conn), + Queries::GetTopTenHottestMonths => get_hottest_months(conn), + Queries::GetTopTenColdestMonths => get_coldest_months(conn), + } }) - .await - .unwrap() - .map_err(|e| InternalError::new(e, StatusCode::INTERNAL_SERVER_ERROR)) + .await? + .map_err(error::ErrorInternalServerError) } fn get_hottest_years(conn: Connection) -> WeatherAggResult { diff --git a/database_interactions/basic/src/main.rs b/database_interactions/basic/src/main.rs index 5c29502..906692f 100644 --- a/database_interactions/basic/src/main.rs +++ b/database_interactions/basic/src/main.rs @@ -1,28 +1,27 @@ -/* Actix-Web Asynchronous Database Example +//! Actix Web Asynchronous Database Example +//! +//! This project illustrates expensive and blocking database requests that runs +//! in a thread-pool using `web::block` with two examples: +//! +//! 1. An asynchronous handler that executes 4 queries in *sequential order*, +//! collecting the results and returning them as a single serialized json object +//! +//! 2. An asynchronous handler that executes 4 queries in *parallel*, +//! collecting the results and returning them as a single serialized json object +//! +//! Note: The use of sleep(Duration::from_secs(2)); in db.rs is to make performance +//! improvement with parallelism more obvious. -This project illustrates expensive and blocking database requests that runs -in a thread-pool using `web::block` with two examples: - - 1. An asynchronous handler that executes 4 queries in *sequential order*, - collecting the results and returning them as a single serialized json object - - 2. An asynchronous handler that executes 4 queries in *parallel*, - collecting the results and returning them as a single serialized json object - - Note: The use of sleep(Duration::from_secs(2)); in db.rs is to make performance - improvement with parallelism more obvious. - */ use std::io; use actix_web::{middleware, web, App, Error as AWError, HttpResponse, HttpServer}; -use futures::future::join_all; +use futures_util::future::join_all; use r2d2_sqlite::{self, SqliteConnectionManager}; mod db; use db::{Pool, Queries}; /// Version 1: Calls 4 queries in sequential order, as an asynchronous handler -#[allow(clippy::eval_order_dependence)] // it's FP? async fn asyncio_weather(db: web::Data) -> Result { let result = vec![ db::execute(&db, Queries::GetTopTenHottestYears).await?, @@ -50,14 +49,15 @@ async fn parallel_weather(db: web::Data) -> Result #[actix_web::main] async fn main() -> io::Result<()> { - std::env::set_var("RUST_LOG", "actix_web=info"); - env_logger::init(); + env_logger::init_from_env(env_logger::Env::new().default_filter_or("info")); - // Start N db executor actors (N = number of cores avail) + // connect to SQLite DB let manager = SqliteConnectionManager::file("weather.db"); let pool = Pool::new(manager).unwrap(); - // Start http server + log::info!("starting HTTP server at http://localhost:8080"); + + // start HTTP server HttpServer::new(move || { App::new() // store db pool as Data object @@ -71,7 +71,8 @@ async fn main() -> io::Result<()> { .route(web::get().to(parallel_weather)), ) }) - .bind("127.0.0.1:8080")? + .bind(("127.0.0.1", 8080))? + .workers(2) .run() .await } diff --git a/other/http-proxy/src/main.rs b/other/http-proxy/src/main.rs index 8d2a134..63d2a63 100644 --- a/other/http-proxy/src/main.rs +++ b/other/http-proxy/src/main.rs @@ -69,7 +69,7 @@ async fn main() -> std::io::Result<()> { let forward_url = Url::parse(&forward_url).unwrap(); log::info!( - "starting HTTP serer at http://{}:{}", + "starting HTTP server at http://{}:{}", &args.listen_addr, args.listen_port ); diff --git a/other/run-in-thread/src/main.rs b/other/run-in-thread/src/main.rs index a00a7db..088b7f8 100644 --- a/other/run-in-thread/src/main.rs +++ b/other/run-in-thread/src/main.rs @@ -8,7 +8,7 @@ async fn index(req: HttpRequest) -> &'static str { } async fn run_app(tx: mpsc::Sender) -> std::io::Result<()> { - log::info!("starting HTTP serer at http://localhost:8080"); + log::info!("starting HTTP server at http://localhost:8080"); // srv is server controller type, `dev::Server` let server = HttpServer::new(|| { diff --git a/security/awc_https/src/main.rs b/security/awc_https/src/main.rs index 986783d..4d2f851 100644 --- a/security/awc_https/src/main.rs +++ b/security/awc_https/src/main.rs @@ -41,7 +41,7 @@ async fn main() -> std::io::Result<()> { let client_tls_config = Arc::new(rustls_config()); - log::info!("starting HTTP serer at http://localhost:8080"); + log::info!("starting HTTP server at http://localhost:8080"); HttpServer::new(move || { // create client _inside_ `HttpServer::new` closure to have one per worker thread diff --git a/weather.db b/weather.db new file mode 100644 index 0000000..e69de29