The DB options are prod-worthy, and speed up inserts by nearly 20x.

This commit is contained in:
Joe Ardent 2023-07-04 21:27:50 -07:00
commit 039fe2e5ec
13 changed files with 439 additions and 73 deletions

195
Cargo.lock generated
View File

@ -45,6 +45,55 @@ dependencies = [
"libc",
]
[[package]]
name = "anstream"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"is-terminal",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
[[package]]
name = "anstyle-parse"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
dependencies = [
"windows-sys",
]
[[package]]
name = "anstyle-wincon"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
dependencies = [
"anstyle",
"windows-sys",
]
[[package]]
name = "anyhow"
version = "1.0.71"
@ -369,9 +418,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.3.2"
version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
[[package]]
name = "blake2"
@ -464,6 +513,55 @@ dependencies = [
"winapi",
]
[[package]]
name = "clap"
version = "4.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a"
dependencies = [
"clap_builder",
"clap_derive",
"once_cell",
]
[[package]]
name = "clap_builder"
version = "4.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim",
"unicase",
"unicode-width",
]
[[package]]
name = "clap_derive"
version = "4.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.18",
]
[[package]]
name = "clap_lex"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
[[package]]
name = "colorchoice"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "constant_time_eq"
version = "0.1.5"
@ -603,6 +701,27 @@ version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "errno"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
dependencies = [
"errno-dragonfly",
"libc",
"windows-sys",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "event-listener"
version = "2.5.3"
@ -835,6 +954,12 @@ dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
[[package]]
name = "hex"
version = "0.4.3"
@ -990,6 +1115,17 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "is-terminal"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb"
dependencies = [
"hermit-abi 0.3.1",
"rustix",
"windows-sys",
]
[[package]]
name = "itertools"
version = "0.10.5"
@ -1054,6 +1190,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
[[package]]
name = "lock_api"
version = "0.4.10"
@ -1159,7 +1301,7 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
dependencies = [
"hermit-abi",
"hermit-abi 0.2.6",
"libc",
]
@ -1424,6 +1566,19 @@ dependencies = [
"winapi",
]
[[package]]
name = "rustix"
version = "0.38.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4"
dependencies = [
"bitflags 2.3.3",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "rustls"
version = "0.20.8"
@ -1745,6 +1900,12 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "subtle"
version = "2.4.1"
@ -1882,6 +2043,17 @@ dependencies = [
"syn 2.0.18",
]
[[package]]
name = "tokio-retry"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f57eb36ecbe0fc510036adff84824dd3c24bb781e21bfa67b69d556aa85214f"
dependencies = [
"pin-project",
"rand",
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.23.4"
@ -1945,7 +2117,7 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c"
dependencies = [
"bitflags 2.3.2",
"bitflags 2.3.3",
"bytes",
"futures-core",
"futures-util",
@ -2090,6 +2262,12 @@ version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
[[package]]
name = "unicode-width"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
[[package]]
name = "unicode_categories"
version = "0.1.1"
@ -2113,6 +2291,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "utf8parse"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "valuable"
version = "0.1.0"
@ -2339,6 +2523,7 @@ dependencies = [
"axum-macros",
"axum-test",
"chrono",
"clap",
"justerror",
"optional_optional_user",
"password-hash",
@ -2348,6 +2533,8 @@ dependencies = [
"sqlx",
"thiserror",
"tokio",
"tokio-retry",
"tokio-stream",
"tower",
"tower-http 0.4.1",
"tracing",

View File

@ -2,6 +2,7 @@
name = "witch_watch"
version = "0.0.1"
edition = "2021"
default-run = "witch_watch"
[dependencies]
axum = { version = "0.6", features = ["macros", "headers"] }
@ -28,6 +29,9 @@ ulid = { version = "1", features = ["rand"] }
# proc macros:
optional_optional_user = {path = "optional_optional_user"}
chrono = { version = "0.4", default-features = false, features = ["std", "clock"] }
clap = { version = "4.3.10", features = ["derive", "env", "unicode", "suggestions", "usage"] }
tokio-retry = "0.3.0"
tokio-stream = "0.1.14"
[dev-dependencies]
axum-test = "9.0.0"

5
results.txt Normal file
View File

@ -0,0 +1,5 @@
-rw-r--r-- 1 ardent ardent 1.6M Jul 4 12:27 .witch-watch.db
-rw-r--r-- 1 ardent ardent 161K Jul 4 12:29 .witch-watch.db-wal
-rw-r--r-- 1 ardent ardent 32K Jul 4 12:29 .witch-watch.db-shm
4 seconds wall to add 10k movies, added by the omega user.

66
src/bin/import_omega.rs Normal file
View File

@ -0,0 +1,66 @@
use std::{ffi::OsString, pin::Pin, time::Duration};
use clap::Parser;
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use tokio::task::JoinSet;
use tokio_retry::Retry;
use tokio_stream::{Stream, StreamExt};
use witch_watch::{
get_db_pool,
import_utils::{add_watch_omega, ensure_omega, ImportMovieOmega},
};
const MOVIE_QUERY: &str = "select * from movies order by random() limit 10000";
#[derive(Debug, Parser)]
struct Cli {
#[clap(long, short)]
pub db_path: OsString,
}
#[tokio::main]
async fn main() {
let cli = Cli::parse();
let path = cli.db_path;
let opts = SqliteConnectOptions::new().filename(&path).read_only(true);
let movie_db = SqlitePoolOptions::new()
.idle_timeout(Duration::from_secs(90))
.connect_with(opts)
.await
.expect("could not open movies db");
let ww_db = get_db_pool().await;
let mut movies: Pin<Box<dyn Stream<Item = Result<ImportMovieOmega, _>> + Send>> =
sqlx::query_as(MOVIE_QUERY).fetch(&movie_db);
ensure_omega(&ww_db).await;
let mut set = JoinSet::new();
let retry_strategy = tokio_retry::strategy::ExponentialBackoff::from_millis(100)
.map(tokio_retry::strategy::jitter)
.take(4);
while let Ok(Some(movie)) = movies.try_next().await {
let db = ww_db.clone();
let title = movie.title.as_str();
let year = movie.year.clone().unwrap();
let len = movie.length.clone().unwrap();
let retry_strategy = retry_strategy.clone();
let key = format!("{title}{year}{len}");
set.spawn(async move {
(
key,
Retry::spawn(retry_strategy, || async {
add_watch_omega(&db, &movie).await
})
.await,
)
});
}
// stragglers
while (set.join_next().await).is_some() {}
}

View File

@ -8,7 +8,7 @@ use axum_login::{
use session_store::SqliteSessionStore;
use sqlx::{
migrate::Migrator,
sqlite::{SqliteConnectOptions, SqlitePoolOptions},
sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions},
SqlitePool,
};
@ -46,6 +46,8 @@ pub async fn get_db_pool() -> SqlitePool {
let conn_opts = SqliteConnectOptions::new()
.foreign_keys(true)
.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental)
.journal_mode(SqliteJournalMode::Wal)
.synchronous(sqlx::sqlite::SqliteSynchronous::Normal)
.filename(&db_filename)
.busy_timeout(Duration::from_secs(TIMEOUT))
.create_if_missing(true);

View File

@ -130,6 +130,22 @@ impl<'de> Visitor<'de> for DbIdVisitor {
}
}
fn visit_string<E>(self, v: String) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
DbId::from_string(&v)
.map_err(|_| serde::de::Error::invalid_value(serde::de::Unexpected::Str(&v), &self))
}
fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
where
E: serde::de::Error,
{
DbId::from_string(v)
.map_err(|_| serde::de::Error::invalid_value(serde::de::Unexpected::Str(v), &self))
}
fn visit_byte_buf<E>(self, v: Vec<u8>) -> Result<Self::Value, E>
where
E: serde::de::Error,

View File

@ -1,31 +1,59 @@
use sqlx::{query, query_scalar, SqlitePool};
use crate::{db_id::DbId, Watch};
use crate::{
db_id::DbId, util::year_to_epoch, watches::handlers::add_new_watch_impl, ShowKind, Watch,
};
const USER_EXISTS_QUERY: &str = "select count(*) from witches where id = $1";
const ADD_WATCH_QUERY: &str = "insert into watches (id, title, kind, metadata_url, length, release_date, added_by) values ($1, $2, $3, $4, $5, $6, $7)";
const OMEGA_ID: u128 = u128::MAX;
#[derive(Debug, sqlx::FromRow, Clone)]
pub struct ImportMovieOmega {
pub title: String,
pub year: Option<String>,
pub length: Option<String>,
}
impl From<ImportMovieOmega> for Watch {
fn from(value: ImportMovieOmega) -> Self {
Watch {
title: value.title,
release_date: year_to_epoch(value.year.as_deref()),
length: value.length.and_then(|v| v.parse::<i64>().ok()),
id: DbId::new(),
kind: ShowKind::Movie,
metadata_url: None,
added_by: OMEGA_ID.into(),
}
}
}
impl From<&ImportMovieOmega> for Watch {
fn from(value: &ImportMovieOmega) -> Self {
Watch {
title: value.title.to_string(),
release_date: year_to_epoch(value.year.as_deref()),
length: value.length.as_ref().and_then(|v| v.parse::<i64>().ok()),
id: DbId::new(),
kind: ShowKind::Movie,
metadata_url: None,
added_by: OMEGA_ID.into(),
}
}
}
//-************************************************************************
// utility functions for building CLI tools, currently just for benchmarking
//-************************************************************************
pub async fn add_watch(db_pool: &SqlitePool, watch: &Watch) {
if query(ADD_WATCH_QUERY)
.bind(watch.id)
.bind(&watch.title)
.bind(watch.kind)
.bind(&watch.metadata_url)
.bind(watch.length)
.bind(watch.release_date)
.bind(watch.added_by)
.execute(db_pool)
.await
.is_ok()
{
pub async fn add_watch_omega(db_pool: &SqlitePool, movie: &ImportMovieOmega) -> Result<(), ()> {
let watch: Watch = movie.into();
if add_new_watch_impl(db_pool, &watch, None).await.is_ok() {
println!("{}", watch.id);
Ok(())
} else {
eprintln!("failed to add \"{}\"", watch.title);
Err(())
}
}
@ -75,7 +103,6 @@ async fn check_omega_exists(db_pool: &SqlitePool) -> bool {
.fetch_one(db_pool)
.await
.unwrap_or(0);
dbg!(count);
count > 0
}

View File

@ -10,6 +10,9 @@ pub use db::get_db_pool;
pub use db_id::DbId;
pub mod import_utils;
pub use users::User;
pub use watches::{ShowKind, Watch, WatchQuest};
// everything else is private to the crate
mod db;
mod db_id;
@ -24,10 +27,7 @@ mod watches;
// things we want in the crate namespace
use optional_optional_user::OptionalOptionalUser;
use templates::*;
use users::User;
use watches::{templates::*, ShowKind, Watch};
use crate::watches::handlers::get_watch;
use watches::templates::*;
type AuthContext = axum_login::extractors::AuthContext<DbId, User, axum_login::SqliteStore<User>>;
@ -42,7 +42,7 @@ pub async fn app(db_pool: sqlx::SqlitePool, session_secret: &[u8]) -> axum::Rout
use login::{get_login, get_logout, post_login, post_logout};
use signup::{get_create_user, get_signup_success, post_create_user};
use watches::handlers::{
get_add_new_watch, get_search_watch, get_watches, post_add_existing_watch,
get_add_new_watch, get_search_watch, get_watch, get_watches, post_add_existing_watch,
post_add_new_watch,
};

View File

@ -70,9 +70,12 @@ pub async fn post_login(
let pw = &login.password;
let pw = pw.trim();
let user = User::try_get(username, &pool)
.await
.map_err(|_| LoginErrorKind::Unknown)?;
let user = User::try_get(username, &pool).await.map_err(|e| {
tracing::debug!("{e}");
LoginErrorKind::Unknown
})?;
dbg!(&user);
let verifier = Argon2::default();
let hash = PasswordHash::new(&user.pwhash).map_err(|_| LoginErrorKind::Internal)?;

View File

@ -25,7 +25,7 @@ async fn main() {
let app = witch_watch::app(pool, &secret).await;
let addr: SocketAddr = ([127, 0, 0, 1], 3000).into();
let addr: SocketAddr = ([0, 0, 0, 0], 3000).into();
tracing::debug!("binding to {addr:?}");
axum::Server::bind(&addr)

View File

@ -20,7 +20,7 @@ pub fn validate_optional_length<E: Error>(
}
}
/// Serde deserialization decorator to map empty Strings to None,
/// Serde deserialization decorator to map empty Strings to None
pub fn empty_string_as_none<'de, D, T>(de: D) -> Result<Option<T>, D::Error>
where
D: serde::Deserializer<'de>,
@ -35,3 +35,17 @@ where
.map(Some),
}
}
/// Convert a stringy number like "1999" to a 64-bit signed unix epoch-based
/// timestamp
pub fn year_to_epoch(year: Option<&str>) -> Option<i64> {
year?
.trim()
.parse::<i32>()
.map(|year| {
let years = (year - 1970) as f32;
let days = (years * 365.2425) as i64;
days * 24 * 60 * 60
})
.ok()
}

View File

@ -7,7 +7,11 @@ use serde::Deserialize;
use sqlx::{query, query_as, SqlitePool};
use super::templates::{AddNewWatchPage, GetWatchPage, SearchWatchesPage};
use crate::{db_id::DbId, util::empty_string_as_none, AuthContext, MyWatchesPage, ShowKind, Watch};
use crate::{
db_id::DbId,
util::{empty_string_as_none, year_to_epoch},
AuthContext, MyWatchesPage, ShowKind, Watch, WatchQuest,
};
//-************************************************************************
// Constants
@ -116,48 +120,26 @@ pub async fn post_add_new_watch(
{
let watch_id = DbId::new();
let witch_watch_id = DbId::new();
let release_date = form.year.map(|year| match year.trim().parse::<i32>() {
Ok(year) => {
let years = (year - 1970) as i64;
let days = (years as f32 * 365.2425) as i64;
Some(days * 24 * 60 * 60)
}
Err(_) => None,
});
let mut tx = pool
.begin()
.await
.map_err(|_| WatchAddErrorKind::UnknownDBError)?;
query(ADD_WATCH_QUERY)
.bind(watch_id)
.bind(&form.title)
.bind(form.kind)
.bind(release_date)
.bind(form.metadata_url)
.bind(user.id)
.execute(&mut tx)
.await
.map_err(|err| {
tracing::error!("Got error: {err}");
WatchAddErrorKind::UnknownDBError
})?;
let release_date = year_to_epoch(form.year.as_deref());
let watch = Watch {
id: watch_id,
title: form.title,
kind: form.kind,
metadata_url: form.metadata_url,
length: None,
release_date,
added_by: user.id,
};
let quest = WatchQuest {
id: witch_watch_id,
user: user.id,
watch: watch_id,
is_public: !form.private,
already_watched: form.watched_already,
};
add_new_watch_impl(&pool, &watch, Some(quest)).await?;
query(ADD_WITCH_WATCH_QUERY)
.bind(witch_watch_id)
.bind(user.id)
.bind(watch_id)
.bind(!form.private)
.bind(form.watched_already)
.execute(&mut tx)
.await
.map_err(|err| {
tracing::error!("Got error: {err}");
WatchAddErrorKind::UnknownDBError
})?;
tx.commit().await.map_err(|err| {
tracing::error!("Got error: {err}");
WatchAddErrorKind::UnknownDBError
})?;
let location = format!("/watch/{watch_id}");
Ok(Redirect::to(&location))
}
@ -166,6 +148,51 @@ pub async fn post_add_new_watch(
}
}
pub(crate) async fn add_new_watch_impl(
db_pool: &SqlitePool,
watch: &Watch,
quest: Option<WatchQuest>,
) -> Result<(), WatchAddError> {
let mut tx = db_pool
.begin()
.await
.map_err(|_| WatchAddErrorKind::UnknownDBError)?;
query(ADD_WATCH_QUERY)
.bind(watch.id)
.bind(&watch.title)
.bind(watch.kind)
.bind(watch.release_date)
.bind(&watch.metadata_url)
.bind(watch.added_by)
.execute(&mut tx)
.await
.map_err(|err| {
tracing::error!("Got error: {err}");
WatchAddErrorKind::UnknownDBError
})?;
if let Some(quest) = quest {
query(ADD_WITCH_WATCH_QUERY)
.bind(quest.id)
.bind(quest.user)
.bind(quest.watch)
.bind(quest.is_public)
.bind(quest.already_watched)
.execute(&mut tx)
.await
.map_err(|err| {
tracing::error!("Got error: {err}");
WatchAddErrorKind::UnknownDBError
})?;
}
tx.commit().await.map_err(|err| {
tracing::error!("Got error: {err}");
WatchAddErrorKind::UnknownDBError
})?;
Ok(())
}
/// Add a Watch to your watchlist by selecting it with a checkbox
pub async fn post_add_existing_watch(
_auth: AuthContext,

View File

@ -52,6 +52,9 @@ impl From<i64> for ShowKind {
}
}
//-************************************************************************
/// Something able to be watched.
//-************************************************************************
#[derive(
Debug,
Default,
@ -86,3 +89,15 @@ impl Watch {
}
}
}
//-************************************************************************
/// Something a user wants to watch
//-************************************************************************
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct WatchQuest {
pub id: DbId,
pub user: DbId,
pub watch: DbId,
pub is_public: bool,
pub already_watched: bool,
}