From 83da336a3ff40bd5e026f84d882b3cf07934ece0 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Sun, 2 Jul 2023 15:16:47 -0700 Subject: [PATCH 1/6] Make it easier to add watches from a CLI tool. --- src/lib.rs | 10 ++-- src/util.rs | 16 +++++- src/watches/handlers.rs | 111 +++++++++++++++++++++++++--------------- src/watches/mod.rs | 15 ++++++ 4 files changed, 104 insertions(+), 48 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 5a9185a..d658431 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,9 @@ pub use db::get_db_pool; pub use db_id::DbId; pub mod import_utils; +pub use users::User; +pub use watches::{ShowKind, Watch, WatchQuest}; + // everything else is private to the crate mod db; mod db_id; @@ -24,10 +27,7 @@ mod watches; // things we want in the crate namespace use optional_optional_user::OptionalOptionalUser; use templates::*; -use users::User; -use watches::{templates::*, ShowKind, Watch}; - -use crate::watches::handlers::get_watch; +use watches::templates::*; type AuthContext = axum_login::extractors::AuthContext>; @@ -42,7 +42,7 @@ pub async fn app(db_pool: sqlx::SqlitePool, session_secret: &[u8]) -> axum::Rout use login::{get_login, get_logout, post_login, post_logout}; use signup::{get_create_user, get_signup_success, post_create_user}; use watches::handlers::{ - get_add_new_watch, get_search_watch, get_watches, post_add_existing_watch, + get_add_new_watch, get_search_watch, get_watch, get_watches, post_add_existing_watch, post_add_new_watch, }; diff --git a/src/util.rs b/src/util.rs index dd16117..375a8f1 100644 --- a/src/util.rs +++ b/src/util.rs @@ -20,7 +20,7 @@ pub fn validate_optional_length( } } -/// Serde deserialization decorator to map empty Strings to None, +/// Serde deserialization decorator to map empty Strings to None pub fn empty_string_as_none<'de, D, T>(de: D) -> Result, D::Error> where D: serde::Deserializer<'de>, @@ -35,3 +35,17 @@ where .map(Some), } } + +/// Convert a stringy number like "1999" to a 64-bit signed unix epoch-based +/// timestamp +pub fn year_to_epoch(year: Option<&str>) -> Option { + year? + .trim() + .parse::() + .map(|year| { + let years = (year - 1970) as f32; + let days = (years * 365.2425) as i64; + days * 24 * 60 * 60 + }) + .ok() +} diff --git a/src/watches/handlers.rs b/src/watches/handlers.rs index beb101f..f1fb141 100644 --- a/src/watches/handlers.rs +++ b/src/watches/handlers.rs @@ -7,7 +7,11 @@ use serde::Deserialize; use sqlx::{query, query_as, SqlitePool}; use super::templates::{AddNewWatchPage, GetWatchPage, SearchWatchesPage}; -use crate::{db_id::DbId, util::empty_string_as_none, AuthContext, MyWatchesPage, ShowKind, Watch}; +use crate::{ + db_id::DbId, + util::{empty_string_as_none, year_to_epoch}, + AuthContext, MyWatchesPage, ShowKind, Watch, WatchQuest, +}; //-************************************************************************ // Constants @@ -116,48 +120,26 @@ pub async fn post_add_new_watch( { let watch_id = DbId::new(); let witch_watch_id = DbId::new(); - let release_date = form.year.map(|year| match year.trim().parse::() { - Ok(year) => { - let years = (year - 1970) as i64; - let days = (years as f32 * 365.2425) as i64; - Some(days * 24 * 60 * 60) - } - Err(_) => None, - }); - let mut tx = pool - .begin() - .await - .map_err(|_| WatchAddErrorKind::UnknownDBError)?; - query(ADD_WATCH_QUERY) - .bind(watch_id) - .bind(&form.title) - .bind(form.kind) - .bind(release_date) - .bind(form.metadata_url) - .bind(user.id) - .execute(&mut tx) - .await - .map_err(|err| { - tracing::error!("Got error: {err}"); - WatchAddErrorKind::UnknownDBError - })?; + let release_date = year_to_epoch(form.year.as_deref()); + let watch = Watch { + id: watch_id, + title: form.title, + kind: form.kind, + metadata_url: form.metadata_url, + length: None, + release_date, + added_by: user.id, + }; + let quest = WatchQuest { + id: witch_watch_id, + user: user.id, + watch: watch_id, + is_public: !form.private, + already_watched: form.watched_already, + }; + + add_new_watch_impl(&pool, &watch, Some(quest)).await?; - query(ADD_WITCH_WATCH_QUERY) - .bind(witch_watch_id) - .bind(user.id) - .bind(watch_id) - .bind(!form.private) - .bind(form.watched_already) - .execute(&mut tx) - .await - .map_err(|err| { - tracing::error!("Got error: {err}"); - WatchAddErrorKind::UnknownDBError - })?; - tx.commit().await.map_err(|err| { - tracing::error!("Got error: {err}"); - WatchAddErrorKind::UnknownDBError - })?; let location = format!("/watch/{watch_id}"); Ok(Redirect::to(&location)) } @@ -166,6 +148,51 @@ pub async fn post_add_new_watch( } } +pub(crate) async fn add_new_watch_impl( + db_pool: &SqlitePool, + watch: &Watch, + quest: Option, +) -> Result<(), WatchAddError> { + let mut tx = db_pool + .begin() + .await + .map_err(|_| WatchAddErrorKind::UnknownDBError)?; + query(ADD_WATCH_QUERY) + .bind(watch.id) + .bind(&watch.title) + .bind(watch.kind) + .bind(watch.release_date) + .bind(&watch.metadata_url) + .bind(watch.added_by) + .execute(&mut tx) + .await + .map_err(|err| { + tracing::error!("Got error: {err}"); + WatchAddErrorKind::UnknownDBError + })?; + + if let Some(quest) = quest { + query(ADD_WITCH_WATCH_QUERY) + .bind(quest.id) + .bind(quest.user) + .bind(quest.watch) + .bind(quest.is_public) + .bind(quest.already_watched) + .execute(&mut tx) + .await + .map_err(|err| { + tracing::error!("Got error: {err}"); + WatchAddErrorKind::UnknownDBError + })?; + } + tx.commit().await.map_err(|err| { + tracing::error!("Got error: {err}"); + WatchAddErrorKind::UnknownDBError + })?; + + Ok(()) +} + /// Add a Watch to your watchlist by selecting it with a checkbox pub async fn post_add_existing_watch( _auth: AuthContext, diff --git a/src/watches/mod.rs b/src/watches/mod.rs index e170da3..c4a2a9a 100644 --- a/src/watches/mod.rs +++ b/src/watches/mod.rs @@ -52,6 +52,9 @@ impl From for ShowKind { } } +//-************************************************************************ +/// Something able to be watched. +//-************************************************************************ #[derive( Debug, Default, @@ -86,3 +89,15 @@ impl Watch { } } } + +//-************************************************************************ +/// Something a user wants to watch +//-************************************************************************ +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct WatchQuest { + pub id: DbId, + pub user: DbId, + pub watch: DbId, + pub is_public: bool, + pub already_watched: bool, +} From 825db8803970629b26acf35f9d80efc697933af8 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Mon, 3 Jul 2023 15:20:19 -0700 Subject: [PATCH 2/6] imports 10k random movies --- Cargo.lock | 246 +++++++++++++++++++++++++++++++++++++--- Cargo.toml | 3 + src/bin/import_omega.rs | 40 +++++++ src/import_utils.rs | 43 ++++--- 4 files changed, 299 insertions(+), 33 deletions(-) create mode 100644 src/bin/import_omega.rs diff --git a/Cargo.lock b/Cargo.lock index 8c47268..75687f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -45,6 +45,55 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is-terminal", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "anyhow" version = "1.0.71" @@ -192,7 +241,7 @@ dependencies = [ "axum-core", "axum-macros", "bitflags 1.3.2", - "bytes", + "bytes 1.4.0", "futures-util", "headers", "http", @@ -223,7 +272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes", + "bytes 1.4.0", "futures-util", "http", "http-body", @@ -241,7 +290,7 @@ checksum = "febf23ab04509bd7672e6abe76bd8277af31b679e89fa5ffc6087dc289a448a3" dependencies = [ "axum", "axum-core", - "bytes", + "bytes 1.4.0", "cookie", "futures-util", "http", @@ -268,7 +317,7 @@ dependencies = [ "base64 0.13.1", "dyn-clone", "eyre", - "futures", + "futures 0.3.28", "ring", "secrecy", "serde", @@ -301,7 +350,7 @@ dependencies = [ "async-session", "axum", "axum-extra", - "futures", + "futures 0.3.28", "http-body", "tokio", "tower", @@ -369,9 +418,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.3.2" +version = "2.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded" +checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42" [[package]] name = "blake2" @@ -427,6 +476,16 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + [[package]] name = "bytes" version = "1.4.0" @@ -464,6 +523,55 @@ dependencies = [ "winapi", ] +[[package]] +name = "clap" +version = "4.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", + "unicase", + "unicode-width", +] + +[[package]] +name = "clap_derive" +version = "4.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.18", +] + +[[package]] +name = "clap_lex" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "constant_time_eq" version = "0.1.5" @@ -603,6 +711,27 @@ version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +[[package]] +name = "errno" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +dependencies = [ + "errno-dragonfly", + "libc", + "windows-sys", +] + +[[package]] +name = "errno-dragonfly" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "event-listener" version = "2.5.3" @@ -646,6 +775,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" + [[package]] name = "futures" version = "0.3.28" @@ -744,6 +879,7 @@ dependencies = [ "pin-project-lite", "pin-utils", "slab", + "tokio-io", ] [[package]] @@ -800,7 +936,7 @@ checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ "base64 0.13.1", "bitflags 1.3.2", - "bytes", + "bytes 1.4.0", "headers-core", "http", "httpdate", @@ -835,6 +971,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -866,7 +1008,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes", + "bytes 1.4.0", "fnv", "itoa", ] @@ -877,7 +1019,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes", + "bytes 1.4.0", "http", "pin-project-lite", ] @@ -915,7 +1057,7 @@ version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ - "bytes", + "bytes 1.4.0", "futures-channel", "futures-core", "futures-util", @@ -990,6 +1132,26 @@ dependencies = [ "cfg-if 1.0.0", ] +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "is-terminal" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb" +dependencies = [ + "hermit-abi 0.3.1", + "rustix", + "windows-sys", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1054,6 +1216,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0" + [[package]] name = "lock_api" version = "0.4.10" @@ -1159,7 +1327,7 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" dependencies = [ - "hermit-abi", + "hermit-abi 0.2.6", "libc", ] @@ -1424,6 +1592,19 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustix" +version = "0.38.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4" +dependencies = [ + "bitflags 2.3.3", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + [[package]] name = "rustls" version = "0.20.8" @@ -1667,7 +1848,7 @@ dependencies = [ "atoi", "bitflags 1.3.2", "byteorder", - "bytes", + "bytes 1.4.0", "chrono", "crc", "crossbeam-queue", @@ -1745,6 +1926,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "subtle" version = "2.4.1" @@ -1858,7 +2045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", - "bytes", + "bytes 1.4.0", "libc", "mio", "num_cpus", @@ -1871,6 +2058,17 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes 0.4.12", + "futures 0.1.31", + "log", +] + [[package]] name = "tokio-macros" version = "2.1.0" @@ -1928,7 +2126,7 @@ checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ "base64 0.13.1", "bitflags 1.3.2", - "bytes", + "bytes 1.4.0", "futures-core", "futures-util", "http", @@ -1945,8 +2143,8 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" dependencies = [ - "bitflags 2.3.2", - "bytes", + "bitflags 2.3.3", + "bytes 1.4.0", "futures-core", "futures-util", "http", @@ -2090,6 +2288,12 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" +[[package]] +name = "unicode-width" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" + [[package]] name = "unicode_categories" version = "0.1.1" @@ -2113,6 +2317,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "valuable" version = "0.1.0" @@ -2339,6 +2549,8 @@ dependencies = [ "axum-macros", "axum-test", "chrono", + "clap", + "futures-util", "justerror", "optional_optional_user", "password-hash", diff --git a/Cargo.toml b/Cargo.toml index 8c0b7d3..c2518bc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "witch_watch" version = "0.0.1" edition = "2021" +default-run = "witch_watch" [dependencies] axum = { version = "0.6", features = ["macros", "headers"] } @@ -28,6 +29,8 @@ ulid = { version = "1", features = ["rand"] } # proc macros: optional_optional_user = {path = "optional_optional_user"} chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } +clap = { version = "4.3.10", features = ["derive", "env", "unicode", "suggestions", "usage"] } +futures-util = { version = "0.3.28", features = ["tokio-io"] } [dev-dependencies] axum-test = "9.0.0" diff --git a/src/bin/import_omega.rs b/src/bin/import_omega.rs new file mode 100644 index 0000000..8f92447 --- /dev/null +++ b/src/bin/import_omega.rs @@ -0,0 +1,40 @@ +use std::{ffi::OsString, time::Duration}; + +use clap::Parser; +use futures_util::stream::TryStreamExt; +use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; +use witch_watch::{ + get_db_pool, + import_utils::{add_watch_omega, ensure_omega}, +}; + +const MOVIE_QUERY: &str = "select * from movies order by random() limit 10000"; + +#[derive(Debug, Parser)] +struct Cli { + #[clap(long, short)] + pub db_path: OsString, +} + +#[tokio::main] +async fn main() { + let cli = Cli::parse(); + let path = cli.db_path; + + let opts = SqliteConnectOptions::new().filename(&path).read_only(true); + let movie_db = SqlitePoolOptions::new() + .idle_timeout(Duration::from_secs(90)) + .connect_with(opts) + .await + .expect("could not open movies db"); + + let ww_db = get_db_pool().await; + + let mut movies = sqlx::query_as(MOVIE_QUERY).fetch(&movie_db); + + ensure_omega(&ww_db).await; + + while let Ok(Some(movie)) = movies.try_next().await { + add_watch_omega(&ww_db, movie).await; + } +} diff --git a/src/import_utils.rs b/src/import_utils.rs index 82ea9a5..3bd958f 100644 --- a/src/import_utils.rs +++ b/src/import_utils.rs @@ -1,28 +1,40 @@ use sqlx::{query, query_scalar, SqlitePool}; -use crate::{db_id::DbId, Watch}; +use crate::{ + db_id::DbId, util::year_to_epoch, watches::handlers::add_new_watch_impl, ShowKind, Watch, +}; const USER_EXISTS_QUERY: &str = "select count(*) from witches where id = $1"; -const ADD_WATCH_QUERY: &str = "insert into watches (id, title, kind, metadata_url, length, release_date, added_by) values ($1, $2, $3, $4, $5, $6, $7)"; const OMEGA_ID: u128 = u128::MAX; +#[derive(Debug, sqlx::FromRow)] +pub struct ImportMovieOmega { + pub title: String, + pub year: Option, + pub length: Option, +} + +impl From for Watch { + fn from(value: ImportMovieOmega) -> Self { + Watch { + title: value.title, + release_date: year_to_epoch(value.year.as_deref()), + length: value.length.and_then(|v| v.parse::().ok()), + id: DbId::new(), + kind: ShowKind::Movie, + metadata_url: None, + added_by: OMEGA_ID.into(), + } + } +} + //-************************************************************************ // utility functions for building CLI tools, currently just for benchmarking //-************************************************************************ -pub async fn add_watch(db_pool: &SqlitePool, watch: &Watch) { - if query(ADD_WATCH_QUERY) - .bind(watch.id) - .bind(&watch.title) - .bind(watch.kind) - .bind(&watch.metadata_url) - .bind(watch.length) - .bind(watch.release_date) - .bind(watch.added_by) - .execute(db_pool) - .await - .is_ok() - { +pub async fn add_watch_omega(db_pool: &SqlitePool, movie: ImportMovieOmega) { + let watch: Watch = movie.into(); + if add_new_watch_impl(db_pool, &watch, None).await.is_ok() { println!("{}", watch.id); } else { eprintln!("failed to add \"{}\"", watch.title); @@ -75,7 +87,6 @@ async fn check_omega_exists(db_pool: &SqlitePool) -> bool { .fetch_one(db_pool) .await .unwrap_or(0); - dbg!(count); count > 0 } From e93e43786ff5cf64cff0221cb24b71e5273f7704 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Tue, 4 Jul 2023 11:30:42 -0700 Subject: [PATCH 3/6] inserts 10k rows in 6 seconds --- Cargo.lock | 13 +++++++++++++ Cargo.toml | 2 ++ src/bin/import_omega.rs | 38 +++++++++++++++++++++++++++++++++----- src/db.rs | 12 +++++++++--- src/import_utils.rs | 18 +++++++++++++++++- 5 files changed, 74 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75687f7..e3123f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2080,6 +2080,17 @@ dependencies = [ "syn 2.0.18", ] +[[package]] +name = "tokio-retry" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f" +dependencies = [ + "pin-project", + "rand", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.23.4" @@ -2560,6 +2571,8 @@ dependencies = [ "sqlx", "thiserror", "tokio", + "tokio-retry", + "tokio-stream", "tower", "tower-http 0.4.1", "tracing", diff --git a/Cargo.toml b/Cargo.toml index c2518bc..cf4421d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,8 @@ optional_optional_user = {path = "optional_optional_user"} chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } clap = { version = "4.3.10", features = ["derive", "env", "unicode", "suggestions", "usage"] } futures-util = { version = "0.3.28", features = ["tokio-io"] } +tokio-retry = "0.3.0" +tokio-stream = "0.1.14" [dev-dependencies] axum-test = "9.0.0" diff --git a/src/bin/import_omega.rs b/src/bin/import_omega.rs index 8f92447..d744d66 100644 --- a/src/bin/import_omega.rs +++ b/src/bin/import_omega.rs @@ -1,11 +1,15 @@ -use std::{ffi::OsString, time::Duration}; +use std::{ffi::OsString, pin::Pin, time::Duration}; use clap::Parser; -use futures_util::stream::TryStreamExt; +use futures_util::Stream; +//use futures_util::stream::TryStreamExt; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; +use tokio::task::JoinSet; +use tokio_retry::Retry; +use tokio_stream::StreamExt; use witch_watch::{ get_db_pool, - import_utils::{add_watch_omega, ensure_omega}, + import_utils::{add_watch_omega, ensure_omega, ImportMovieOmega}, }; const MOVIE_QUERY: &str = "select * from movies order by random() limit 10000"; @@ -30,11 +34,35 @@ async fn main() { let ww_db = get_db_pool().await; - let mut movies = sqlx::query_as(MOVIE_QUERY).fetch(&movie_db); + let mut movies: Pin> + Send>> = + sqlx::query_as(MOVIE_QUERY).fetch(&movie_db); ensure_omega(&ww_db).await; + let mut set = JoinSet::new(); + + let retry_strategy = tokio_retry::strategy::ExponentialBackoff::from_millis(100) + .map(tokio_retry::strategy::jitter) + .take(4); + while let Ok(Some(movie)) = movies.try_next().await { - add_watch_omega(&ww_db, movie).await; + let db = ww_db.clone(); + let title = movie.title.as_str(); + let year = movie.year.clone().unwrap(); + let len = movie.length.clone().unwrap(); + let retry_strategy = retry_strategy.clone(); + + let key = format!("{title}{year}{len}"); + set.spawn(async move { + ( + key, + Retry::spawn(retry_strategy, || async { + add_watch_omega(&db, &movie).await + }) + .await, + ) + }); } + // stragglers + while (set.join_next().await).is_some() {} } diff --git a/src/db.rs b/src/db.rs index 70540b6..9e80e72 100644 --- a/src/db.rs +++ b/src/db.rs @@ -8,8 +8,8 @@ use axum_login::{ use session_store::SqliteSessionStore; use sqlx::{ migrate::Migrator, - sqlite::{SqliteConnectOptions, SqlitePoolOptions}, - SqlitePool, + sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}, + Executor, SqlitePool, }; use crate::{db_id::DbId, User}; @@ -45,7 +45,9 @@ pub async fn get_db_pool() -> SqlitePool { let conn_opts = SqliteConnectOptions::new() .foreign_keys(true) - .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental) + //.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental) + .journal_mode(SqliteJournalMode::Memory) + .synchronous(sqlx::sqlite::SqliteSynchronous::Off) .filename(&db_filename) .busy_timeout(Duration::from_secs(TIMEOUT)) .create_if_missing(true); @@ -59,6 +61,10 @@ pub async fn get_db_pool() -> SqlitePool { .await .expect("can't connect to database"); + let mut conn = pool.acquire().await.unwrap(); + conn.execute("PRAGMA cache_size = 1000000;").await.unwrap(); + conn.execute("PRAGMA temp_store = MEMORY;").await.unwrap(); + // let the filesystem settle before trying anything // possibly not effective? tokio::time::sleep(Duration::from_millis(500)).await; diff --git a/src/import_utils.rs b/src/import_utils.rs index 3bd958f..df2b99f 100644 --- a/src/import_utils.rs +++ b/src/import_utils.rs @@ -29,15 +29,31 @@ impl From for Watch { } } +impl From<&ImportMovieOmega> for Watch { + fn from(value: &ImportMovieOmega) -> Self { + Watch { + title: value.title.to_string(), + release_date: year_to_epoch(value.year.as_deref()), + length: value.length.as_ref().and_then(|v| v.parse::().ok()), + id: DbId::new(), + kind: ShowKind::Movie, + metadata_url: None, + added_by: OMEGA_ID.into(), + } + } +} + //-************************************************************************ // utility functions for building CLI tools, currently just for benchmarking //-************************************************************************ -pub async fn add_watch_omega(db_pool: &SqlitePool, movie: ImportMovieOmega) { +pub async fn add_watch_omega(db_pool: &SqlitePool, movie: &ImportMovieOmega) -> Result<(), ()> { let watch: Watch = movie.into(); if add_new_watch_impl(db_pool, &watch, None).await.is_ok() { println!("{}", watch.id); + Ok(()) } else { eprintln!("failed to add \"{}\"", watch.title); + Err(()) } } From f58106ee368260a36d1fa63f257ef57daf11bc9a Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Tue, 4 Jul 2023 12:22:17 -0700 Subject: [PATCH 4/6] checkpoint; logins broken somehow, maybe just from remote host --- src/db.rs | 8 ++++---- src/db_id.rs | 16 ++++++++++++++++ src/login.rs | 9 ++++++--- src/main.rs | 2 +- 4 files changed, 27 insertions(+), 8 deletions(-) diff --git a/src/db.rs b/src/db.rs index 9e80e72..fd844d8 100644 --- a/src/db.rs +++ b/src/db.rs @@ -46,7 +46,7 @@ pub async fn get_db_pool() -> SqlitePool { let conn_opts = SqliteConnectOptions::new() .foreign_keys(true) //.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental) - .journal_mode(SqliteJournalMode::Memory) + .journal_mode(SqliteJournalMode::Wal) .synchronous(sqlx::sqlite::SqliteSynchronous::Off) .filename(&db_filename) .busy_timeout(Duration::from_secs(TIMEOUT)) @@ -61,9 +61,9 @@ pub async fn get_db_pool() -> SqlitePool { .await .expect("can't connect to database"); - let mut conn = pool.acquire().await.unwrap(); - conn.execute("PRAGMA cache_size = 1000000;").await.unwrap(); - conn.execute("PRAGMA temp_store = MEMORY;").await.unwrap(); + // let mut conn = pool.acquire().await.unwrap(); + // conn.execute("PRAGMA cache_size = 1000000;").await.unwrap(); + // conn.execute("PRAGMA temp_store = MEMORY;").await.unwrap(); // let the filesystem settle before trying anything // possibly not effective? diff --git a/src/db_id.rs b/src/db_id.rs index 0a18f0c..24ba42d 100644 --- a/src/db_id.rs +++ b/src/db_id.rs @@ -130,6 +130,22 @@ impl<'de> Visitor<'de> for DbIdVisitor { } } + fn visit_string(self, v: String) -> Result + where + E: serde::de::Error, + { + DbId::from_string(&v) + .map_err(|_| serde::de::Error::invalid_value(serde::de::Unexpected::Str(&v), &self)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + DbId::from_string(v) + .map_err(|_| serde::de::Error::invalid_value(serde::de::Unexpected::Str(v), &self)) + } + fn visit_byte_buf(self, v: Vec) -> Result where E: serde::de::Error, diff --git a/src/login.rs b/src/login.rs index 2a4453d..3c9ef8d 100644 --- a/src/login.rs +++ b/src/login.rs @@ -70,9 +70,12 @@ pub async fn post_login( let pw = &login.password; let pw = pw.trim(); - let user = User::try_get(username, &pool) - .await - .map_err(|_| LoginErrorKind::Unknown)?; + let user = User::try_get(username, &pool).await.map_err(|e| { + tracing::debug!("{e}"); + LoginErrorKind::Unknown + })?; + + dbg!(&user); let verifier = Argon2::default(); let hash = PasswordHash::new(&user.pwhash).map_err(|_| LoginErrorKind::Internal)?; diff --git a/src/main.rs b/src/main.rs index 9ee5d62..d9c1062 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,7 +25,7 @@ async fn main() { let app = witch_watch::app(pool, &secret).await; - let addr: SocketAddr = ([127, 0, 0, 1], 3000).into(); + let addr: SocketAddr = ([0, 0, 0, 0], 3000).into(); tracing::debug!("binding to {addr:?}"); axum::Server::bind(&addr) From 24926fed6a9dec7a65932d65beb69b4c8c3bb088 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Tue, 4 Jul 2023 12:56:22 -0700 Subject: [PATCH 5/6] added results for ulids added by the omega user --- results.txt | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 results.txt diff --git a/results.txt b/results.txt new file mode 100644 index 0000000..5e063f8 --- /dev/null +++ b/results.txt @@ -0,0 +1,5 @@ +-rw-r--r-- 1 ardent ardent 1.6M Jul 4 12:27 .witch-watch.db +-rw-r--r-- 1 ardent ardent 161K Jul 4 12:29 .witch-watch.db-wal +-rw-r--r-- 1 ardent ardent 32K Jul 4 12:29 .witch-watch.db-shm + +3.5 seconds wall to add 10k movies, added by the omega user. From 5e4f5c07d8fad52ee1514527f9516d18a13387a9 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Tue, 4 Jul 2023 16:24:28 -0700 Subject: [PATCH 6/6] Update db config to production-friendly. Four seconds to insert 10k records, with a prod-safe config. Stick with this design. --- Cargo.lock | 64 +++++++++-------------------------------- Cargo.toml | 1 - results.txt | 2 +- src/bin/import_omega.rs | 4 +-- src/db.rs | 10 ++----- src/import_utils.rs | 2 +- 6 files changed, 19 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3123f6..8fae242 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -241,7 +241,7 @@ dependencies = [ "axum-core", "axum-macros", "bitflags 1.3.2", - "bytes 1.4.0", + "bytes", "futures-util", "headers", "http", @@ -272,7 +272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.4.0", + "bytes", "futures-util", "http", "http-body", @@ -290,7 +290,7 @@ checksum = "febf23ab04509bd7672e6abe76bd8277af31b679e89fa5ffc6087dc289a448a3" dependencies = [ "axum", "axum-core", - "bytes 1.4.0", + "bytes", "cookie", "futures-util", "http", @@ -317,7 +317,7 @@ dependencies = [ "base64 0.13.1", "dyn-clone", "eyre", - "futures 0.3.28", + "futures", "ring", "secrecy", "serde", @@ -350,7 +350,7 @@ dependencies = [ "async-session", "axum", "axum-extra", - "futures 0.3.28", + "futures", "http-body", "tokio", "tower", @@ -476,16 +476,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.4.0" @@ -775,12 +765,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "futures" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" - [[package]] name = "futures" version = "0.3.28" @@ -879,7 +863,6 @@ dependencies = [ "pin-project-lite", "pin-utils", "slab", - "tokio-io", ] [[package]] @@ -936,7 +919,7 @@ checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ "base64 0.13.1", "bitflags 1.3.2", - "bytes 1.4.0", + "bytes", "headers-core", "http", "httpdate", @@ -1008,7 +991,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.4.0", + "bytes", "fnv", "itoa", ] @@ -1019,7 +1002,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.4.0", + "bytes", "http", "pin-project-lite", ] @@ -1057,7 +1040,7 @@ version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -1132,15 +1115,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "is-terminal" version = "0.4.8" @@ -1848,7 +1822,7 @@ dependencies = [ "atoi", "bitflags 1.3.2", "byteorder", - "bytes 1.4.0", + "bytes", "chrono", "crc", "crossbeam-queue", @@ -2045,7 +2019,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", - "bytes 1.4.0", + "bytes", "libc", "mio", "num_cpus", @@ -2058,17 +2032,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "tokio-io" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", -] - [[package]] name = "tokio-macros" version = "2.1.0" @@ -2137,7 +2100,7 @@ checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ "base64 0.13.1", "bitflags 1.3.2", - "bytes 1.4.0", + "bytes", "futures-core", "futures-util", "http", @@ -2155,7 +2118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" dependencies = [ "bitflags 2.3.3", - "bytes 1.4.0", + "bytes", "futures-core", "futures-util", "http", @@ -2561,7 +2524,6 @@ dependencies = [ "axum-test", "chrono", "clap", - "futures-util", "justerror", "optional_optional_user", "password-hash", diff --git a/Cargo.toml b/Cargo.toml index cf4421d..28da39b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,6 @@ ulid = { version = "1", features = ["rand"] } optional_optional_user = {path = "optional_optional_user"} chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } clap = { version = "4.3.10", features = ["derive", "env", "unicode", "suggestions", "usage"] } -futures-util = { version = "0.3.28", features = ["tokio-io"] } tokio-retry = "0.3.0" tokio-stream = "0.1.14" diff --git a/results.txt b/results.txt index 5e063f8..d3171b2 100644 --- a/results.txt +++ b/results.txt @@ -2,4 +2,4 @@ -rw-r--r-- 1 ardent ardent 161K Jul 4 12:29 .witch-watch.db-wal -rw-r--r-- 1 ardent ardent 32K Jul 4 12:29 .witch-watch.db-shm -3.5 seconds wall to add 10k movies, added by the omega user. +4 seconds wall to add 10k movies, added by the omega user. diff --git a/src/bin/import_omega.rs b/src/bin/import_omega.rs index d744d66..893ca7b 100644 --- a/src/bin/import_omega.rs +++ b/src/bin/import_omega.rs @@ -1,12 +1,10 @@ use std::{ffi::OsString, pin::Pin, time::Duration}; use clap::Parser; -use futures_util::Stream; -//use futures_util::stream::TryStreamExt; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use tokio::task::JoinSet; use tokio_retry::Retry; -use tokio_stream::StreamExt; +use tokio_stream::{Stream, StreamExt}; use witch_watch::{ get_db_pool, import_utils::{add_watch_omega, ensure_omega, ImportMovieOmega}, diff --git a/src/db.rs b/src/db.rs index fd844d8..926f3c4 100644 --- a/src/db.rs +++ b/src/db.rs @@ -9,7 +9,7 @@ use session_store::SqliteSessionStore; use sqlx::{ migrate::Migrator, sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}, - Executor, SqlitePool, + SqlitePool, }; use crate::{db_id::DbId, User}; @@ -45,9 +45,9 @@ pub async fn get_db_pool() -> SqlitePool { let conn_opts = SqliteConnectOptions::new() .foreign_keys(true) - //.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental) + .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental) .journal_mode(SqliteJournalMode::Wal) - .synchronous(sqlx::sqlite::SqliteSynchronous::Off) + .synchronous(sqlx::sqlite::SqliteSynchronous::Normal) .filename(&db_filename) .busy_timeout(Duration::from_secs(TIMEOUT)) .create_if_missing(true); @@ -61,10 +61,6 @@ pub async fn get_db_pool() -> SqlitePool { .await .expect("can't connect to database"); - // let mut conn = pool.acquire().await.unwrap(); - // conn.execute("PRAGMA cache_size = 1000000;").await.unwrap(); - // conn.execute("PRAGMA temp_store = MEMORY;").await.unwrap(); - // let the filesystem settle before trying anything // possibly not effective? tokio::time::sleep(Duration::from_millis(500)).await; diff --git a/src/import_utils.rs b/src/import_utils.rs index df2b99f..d10c018 100644 --- a/src/import_utils.rs +++ b/src/import_utils.rs @@ -8,7 +8,7 @@ const USER_EXISTS_QUERY: &str = "select count(*) from witches where id = $1"; const OMEGA_ID: u128 = u128::MAX; -#[derive(Debug, sqlx::FromRow)] +#[derive(Debug, sqlx::FromRow, Clone)] pub struct ImportMovieOmega { pub title: String, pub year: Option,