From 5e4f5c07d8fad52ee1514527f9516d18a13387a9 Mon Sep 17 00:00:00 2001 From: Joe Ardent Date: Tue, 4 Jul 2023 16:24:28 -0700 Subject: [PATCH] Update db config to production-friendly. Four seconds to insert 10k records, with a prod-safe config. Stick with this design. --- Cargo.lock | 64 +++++++++-------------------------------- Cargo.toml | 1 - results.txt | 2 +- src/bin/import_omega.rs | 4 +-- src/db.rs | 10 ++----- src/import_utils.rs | 2 +- 6 files changed, 19 insertions(+), 64 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e3123f6..8fae242 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -241,7 +241,7 @@ dependencies = [ "axum-core", "axum-macros", "bitflags 1.3.2", - "bytes 1.4.0", + "bytes", "futures-util", "headers", "http", @@ -272,7 +272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c" dependencies = [ "async-trait", - "bytes 1.4.0", + "bytes", "futures-util", "http", "http-body", @@ -290,7 +290,7 @@ checksum = "febf23ab04509bd7672e6abe76bd8277af31b679e89fa5ffc6087dc289a448a3" dependencies = [ "axum", "axum-core", - "bytes 1.4.0", + "bytes", "cookie", "futures-util", "http", @@ -317,7 +317,7 @@ dependencies = [ "base64 0.13.1", "dyn-clone", "eyre", - "futures 0.3.28", + "futures", "ring", "secrecy", "serde", @@ -350,7 +350,7 @@ dependencies = [ "async-session", "axum", "axum-extra", - "futures 0.3.28", + "futures", "http-body", "tokio", "tower", @@ -476,16 +476,6 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" -[[package]] -name = "bytes" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" -dependencies = [ - "byteorder", - "iovec", -] - [[package]] name = "bytes" version = "1.4.0" @@ -775,12 +765,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "futures" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678" - [[package]] name = "futures" version = "0.3.28" @@ -879,7 +863,6 @@ dependencies = [ "pin-project-lite", "pin-utils", "slab", - "tokio-io", ] [[package]] @@ -936,7 +919,7 @@ checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584" dependencies = [ "base64 0.13.1", "bitflags 1.3.2", - "bytes 1.4.0", + "bytes", "headers-core", "http", "httpdate", @@ -1008,7 +991,7 @@ version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ - "bytes 1.4.0", + "bytes", "fnv", "itoa", ] @@ -1019,7 +1002,7 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" dependencies = [ - "bytes 1.4.0", + "bytes", "http", "pin-project-lite", ] @@ -1057,7 +1040,7 @@ version = "0.14.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" dependencies = [ - "bytes 1.4.0", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -1132,15 +1115,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] - [[package]] name = "is-terminal" version = "0.4.8" @@ -1848,7 +1822,7 @@ dependencies = [ "atoi", "bitflags 1.3.2", "byteorder", - "bytes 1.4.0", + "bytes", "chrono", "crc", "crossbeam-queue", @@ -2045,7 +2019,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", - "bytes 1.4.0", + "bytes", "libc", "mio", "num_cpus", @@ -2058,17 +2032,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "tokio-io" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" -dependencies = [ - "bytes 0.4.12", - "futures 0.1.31", - "log", -] - [[package]] name = "tokio-macros" version = "2.1.0" @@ -2137,7 +2100,7 @@ checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858" dependencies = [ "base64 0.13.1", "bitflags 1.3.2", - "bytes 1.4.0", + "bytes", "futures-core", "futures-util", "http", @@ -2155,7 +2118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c" dependencies = [ "bitflags 2.3.3", - "bytes 1.4.0", + "bytes", "futures-core", "futures-util", "http", @@ -2561,7 +2524,6 @@ dependencies = [ "axum-test", "chrono", "clap", - "futures-util", "justerror", "optional_optional_user", "password-hash", diff --git a/Cargo.toml b/Cargo.toml index cf4421d..28da39b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,6 @@ ulid = { version = "1", features = ["rand"] } optional_optional_user = {path = "optional_optional_user"} chrono = { version = "0.4", default-features = false, features = ["std", "clock"] } clap = { version = "4.3.10", features = ["derive", "env", "unicode", "suggestions", "usage"] } -futures-util = { version = "0.3.28", features = ["tokio-io"] } tokio-retry = "0.3.0" tokio-stream = "0.1.14" diff --git a/results.txt b/results.txt index 5e063f8..d3171b2 100644 --- a/results.txt +++ b/results.txt @@ -2,4 +2,4 @@ -rw-r--r-- 1 ardent ardent 161K Jul 4 12:29 .witch-watch.db-wal -rw-r--r-- 1 ardent ardent 32K Jul 4 12:29 .witch-watch.db-shm -3.5 seconds wall to add 10k movies, added by the omega user. +4 seconds wall to add 10k movies, added by the omega user. diff --git a/src/bin/import_omega.rs b/src/bin/import_omega.rs index d744d66..893ca7b 100644 --- a/src/bin/import_omega.rs +++ b/src/bin/import_omega.rs @@ -1,12 +1,10 @@ use std::{ffi::OsString, pin::Pin, time::Duration}; use clap::Parser; -use futures_util::Stream; -//use futures_util::stream::TryStreamExt; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions}; use tokio::task::JoinSet; use tokio_retry::Retry; -use tokio_stream::StreamExt; +use tokio_stream::{Stream, StreamExt}; use witch_watch::{ get_db_pool, import_utils::{add_watch_omega, ensure_omega, ImportMovieOmega}, diff --git a/src/db.rs b/src/db.rs index fd844d8..926f3c4 100644 --- a/src/db.rs +++ b/src/db.rs @@ -9,7 +9,7 @@ use session_store::SqliteSessionStore; use sqlx::{ migrate::Migrator, sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}, - Executor, SqlitePool, + SqlitePool, }; use crate::{db_id::DbId, User}; @@ -45,9 +45,9 @@ pub async fn get_db_pool() -> SqlitePool { let conn_opts = SqliteConnectOptions::new() .foreign_keys(true) - //.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental) + .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental) .journal_mode(SqliteJournalMode::Wal) - .synchronous(sqlx::sqlite::SqliteSynchronous::Off) + .synchronous(sqlx::sqlite::SqliteSynchronous::Normal) .filename(&db_filename) .busy_timeout(Duration::from_secs(TIMEOUT)) .create_if_missing(true); @@ -61,10 +61,6 @@ pub async fn get_db_pool() -> SqlitePool { .await .expect("can't connect to database"); - // let mut conn = pool.acquire().await.unwrap(); - // conn.execute("PRAGMA cache_size = 1000000;").await.unwrap(); - // conn.execute("PRAGMA temp_store = MEMORY;").await.unwrap(); - // let the filesystem settle before trying anything // possibly not effective? tokio::time::sleep(Duration::from_millis(500)).await; diff --git a/src/import_utils.rs b/src/import_utils.rs index df2b99f..d10c018 100644 --- a/src/import_utils.rs +++ b/src/import_utils.rs @@ -8,7 +8,7 @@ const USER_EXISTS_QUERY: &str = "select count(*) from witches where id = $1"; const OMEGA_ID: u128 = u128::MAX; -#[derive(Debug, sqlx::FromRow)] +#[derive(Debug, sqlx::FromRow, Clone)] pub struct ImportMovieOmega { pub title: String, pub year: Option,