From 78a753a06a2f19aacb07855538614235c3511e47 Mon Sep 17 00:00:00 2001 From: Arve Seljebu Date: Thu, 30 Jan 2020 07:52:10 +0100 Subject: [PATCH] update rusqlite and r2d2_sqlite in async_db and r2d2 (#148) * upgrade rusqlite to 0.18 * add note about async_db blocking and running in thread pool * idiomatic async_db - DRY - Iterator::collect + return type - note about why we are sleeping * remove r2d2/test.db --- async_db/Cargo.toml | 4 +- async_db/src/db.rs | 138 +++++++++++++++++++------------------------ async_db/src/main.rs | 3 +- r2d2/Cargo.toml | 4 +- r2d2/test.db | Bin 20480 -> 0 bytes 5 files changed, 66 insertions(+), 83 deletions(-) delete mode 100644 r2d2/test.db diff --git a/async_db/Cargo.toml b/async_db/Cargo.toml index b994daae..4d8e44c6 100644 --- a/async_db/Cargo.toml +++ b/async_db/Cargo.toml @@ -14,7 +14,7 @@ failure = "0.1.1" futures = "0.3.1" num_cpus = "1.10.0" r2d2 = "0.8.2" -r2d2_sqlite = "0.8.0" -rusqlite = "0.16" +r2d2_sqlite = "0.10" +rusqlite = "0.18" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/async_db/src/db.rs b/async_db/src/db.rs index 65329601..252f55cb 100644 --- a/async_db/src/db.rs +++ b/async_db/src/db.rs @@ -3,12 +3,13 @@ use failure::Error; use futures::{Future, TryFutureExt}; use r2d2; use r2d2_sqlite; -use rusqlite::NO_PARAMS; +use rusqlite::{Statement, NO_PARAMS}; use serde::{Deserialize, Serialize}; use std::{thread::sleep, time::Duration}; pub type Pool = r2d2::Pool; pub type Connection = r2d2::PooledConnection; +type WeatherAggResult = Result, rusqlite::Error>; #[derive(Debug, Serialize, Deserialize)] pub enum WeatherAgg { @@ -28,115 +29,96 @@ pub fn execute( query: Queries, ) -> impl Future, AWError>> { let pool = pool.clone(); - web::block(move || match query { - Queries::GetTopTenHottestYears => get_hottest_years(pool.get()?), - Queries::GetTopTenColdestYears => get_coldest_years(pool.get()?), - Queries::GetTopTenHottestMonths => get_hottest_months(pool.get()?), - Queries::GetTopTenColdestMonths => get_coldest_months(pool.get()?), + web::block(move || { + // simulate an expensive query, see comments at top of main.rs + sleep(Duration::from_secs(2)); + + let result = match query { + Queries::GetTopTenHottestYears => get_hottest_years(pool.get()?), + Queries::GetTopTenColdestYears => get_coldest_years(pool.get()?), + Queries::GetTopTenHottestMonths => get_hottest_months(pool.get()?), + Queries::GetTopTenColdestMonths => get_coldest_months(pool.get()?), + }; + result.map_err(Error::from) }) .map_err(AWError::from) } -fn get_hottest_years(conn: Connection) -> Result, Error> { - let stmt = " +fn get_hottest_years(conn: Connection) -> WeatherAggResult { + let stmt = conn.prepare( + " SELECT cast(strftime('%Y', date) as int) as theyear, sum(tmax) as total FROM nyc_weather WHERE tmax <> 'TMAX' GROUP BY theyear - ORDER BY total DESC LIMIT 10;"; + ORDER BY total DESC LIMIT 10", + )?; - let mut prep_stmt = conn.prepare(stmt)?; - let annuals = prep_stmt - .query_map(NO_PARAMS, |row| WeatherAgg::AnnualAgg { - year: row.get(0), - total: row.get(1), - }) - .and_then(|mapped_rows| { - Ok(mapped_rows - .map(|row| row.unwrap()) - .collect::>()) - })?; - - sleep(Duration::from_secs(2)); //see comments at top of main.rs - - Ok(annuals) + get_rows_as_annual_agg(stmt) } -fn get_coldest_years(conn: Connection) -> Result, Error> { - let stmt = " +fn get_coldest_years(conn: Connection) -> WeatherAggResult { + let stmt = conn.prepare( + " SELECT cast(strftime('%Y', date) as int) as theyear, sum(tmax) as total FROM nyc_weather WHERE tmax <> 'TMAX' GROUP BY theyear - ORDER BY total ASC LIMIT 10;"; + ORDER BY total ASC LIMIT 10", + )?; - let mut prep_stmt = conn.prepare(stmt)?; - let annuals = prep_stmt - .query_map(NO_PARAMS, |row| WeatherAgg::AnnualAgg { - year: row.get(0), - total: row.get(1), - }) - .and_then(|mapped_rows| { - Ok(mapped_rows - .map(|row| row.unwrap()) - .collect::>()) - })?; - - sleep(Duration::from_secs(2)); //see comments at top of main.rs - - Ok(annuals) + get_rows_as_annual_agg(stmt) } -fn get_hottest_months(conn: Connection) -> Result, Error> { - let stmt = "SELECT cast(strftime('%Y', date) as int) as theyear, +fn get_rows_as_annual_agg(mut statement: Statement) -> WeatherAggResult { + statement + .query_map(NO_PARAMS, |row| { + Ok(WeatherAgg::AnnualAgg { + year: row.get(0)?, + total: row.get(1)?, + }) + }) + .and_then(Iterator::collect) +} + +fn get_hottest_months(conn: Connection) -> WeatherAggResult { + let stmt = conn.prepare( + "SELECT cast(strftime('%Y', date) as int) as theyear, cast(strftime('%m', date) as int) as themonth, sum(tmax) as total FROM nyc_weather WHERE tmax <> 'TMAX' GROUP BY theyear, themonth - ORDER BY total DESC LIMIT 10;"; + ORDER BY total DESC LIMIT 10", + )?; - let mut prep_stmt = conn.prepare(stmt)?; - let annuals = prep_stmt - .query_map(NO_PARAMS, |row| WeatherAgg::MonthAgg { - year: row.get(0), - month: row.get(1), - total: row.get(2), - }) - .and_then(|mapped_rows| { - Ok(mapped_rows - .map(|row| row.unwrap()) - .collect::>()) - })?; - - sleep(Duration::from_secs(2)); //see comments at top of main.rs - Ok(annuals) + get_rows_as_month_agg(stmt) } -fn get_coldest_months(conn: Connection) -> Result, Error> { - let stmt = "SELECT cast(strftime('%Y', date) as int) as theyear, +fn get_coldest_months(conn: Connection) -> WeatherAggResult { + let stmt = conn.prepare( + "SELECT cast(strftime('%Y', date) as int) as theyear, cast(strftime('%m', date) as int) as themonth, sum(tmax) as total FROM nyc_weather WHERE tmax <> 'TMAX' GROUP BY theyear, themonth - ORDER BY total ASC LIMIT 10;"; + ORDER BY total ASC LIMIT 10", + )?; - let mut prep_stmt = conn.prepare(stmt)?; - let annuals = prep_stmt - .query_map(NO_PARAMS, |row| WeatherAgg::MonthAgg { - year: row.get(0), - month: row.get(1), - total: row.get(2), - }) - .and_then(|mapped_rows| { - Ok(mapped_rows - .map(|row| row.unwrap()) - .collect::>()) - })?; - - sleep(Duration::from_secs(2)); //see comments at top of main.rs - Ok(annuals) + get_rows_as_month_agg(stmt) +} + +fn get_rows_as_month_agg(mut statement: Statement) -> WeatherAggResult { + statement + .query_map(NO_PARAMS, |row| { + Ok(WeatherAgg::MonthAgg { + year: row.get(0)?, + month: row.get(1)?, + total: row.get(2)?, + }) + }) + .and_then(Iterator::collect) } diff --git a/async_db/src/main.rs b/async_db/src/main.rs index 93443ad5..90a59562 100644 --- a/async_db/src/main.rs +++ b/async_db/src/main.rs @@ -1,6 +1,7 @@ /* Actix-Web Asynchronous Database Example -This project illustrates two examples: +This project illustrates expensive and blocking database requests that runs +in a thread-pool using `web::block` with two examples: 1. An asynchronous handler that executes 4 queries in *sequential order*, collecting the results and returning them as a single serialized json object diff --git a/r2d2/Cargo.toml b/r2d2/Cargo.toml index b3444c25..220d8fa8 100644 --- a/r2d2/Cargo.toml +++ b/r2d2/Cargo.toml @@ -13,5 +13,5 @@ env_logger = "0.6" uuid = { version = "0.8", features = ["v4"] } r2d2 = "0.8" -r2d2_sqlite = "0.8" -rusqlite = "0.16" +r2d2_sqlite = "0.10" +rusqlite = "0.18" diff --git a/r2d2/test.db b/r2d2/test.db deleted file mode 100644 index 3ea0c83d772f543f8555d66d0e2ddeeddaf4a252..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20480 zcmeI%PjAyO7zXg9=?ZM3fdi@#9DG1x)iA_~^A8D$uoal-26Qdb4k&UQhgI4-(RLHp zao~&W4j+aK@FC!e#04kPCJ-v>k)g+`n>cxz+Q0UaH(SrU!>J<0{&-JJiDz6gOw+he zh+!D#bzji^q}B9J{bZoG<}drRcF!BPFa5Y^e>dvQHKY02K5nix-_Hu;I0PU70SG_< z0uX=z1Rwwb2>jauJJz{Yt7RVDnTl*z9Zb}CQoYqzd!3};A^qe*w?nE!WOt4$=hNE1)nY`ZBx2~x; znC@5OwEBjWRhxyQN9MU!l9+F=Rua6Nc-eQ_zpm(XnYYFVqg3bm>l>Y|ezNiG^bA?+JWK}NK3N^~dY#RFdKzhO z)f%l=$*e`so>t0cpR@Eos=U89F6wUDUkv?1g8&2|009U<00Izz00bZa0SG|g%n95u zt+wYnEOeO5tL~N~%3R7~;y8C5_pZYh^}0;^tD#?L5P$##AOHafKmY;|fB*y_009X6 z7Xp{9_J!X|0_F4nM)R9tf3%P7Lwnu6t_NrkfB*y_009U<00Izz00bZafwL{J(y#_s zoFWvV&mt-V&!IdIGn#Q#&``LA!~9%k9-peobY-CoM_v%fJk&W^Q10t+>|}x#Tsfie zxZs7X!u2xjXJHh@ib@{oPXsEaan4=pdxFK8U&KMks<5TQv5aCj@;NPvIHFuAPNO*T zsFa1D@jNITsmk!}nl-qV`!bLGoY7bbmvYHCjRhCf<1A3VE4@HEVzfWqQR5X;UtF1Q>%jf?Md&#h0+T32U-`OARPiOll#_S;g0SG_< z0uX=z1Rwwb2tWV=e-&7^2If4afkqXaqb#sc1?MOU)XU&J