Update db config to production-friendly.

Four seconds to insert 10k records, with a prod-safe config. Stick with this design.

Cargo.lock

@@ -241,7 +241,7 @@ dependencies = [
  "axum-core",
  "axum-macros",
  "bitflags 1.3.2",
- "bytes 1.4.0",
+ "bytes",
  "futures-util",
  "headers",
  "http",
@@ -272,7 +272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
 dependencies = [
  "async-trait",
- "bytes 1.4.0",
+ "bytes",
  "futures-util",
  "http",
  "http-body",
@@ -290,7 +290,7 @@ checksum = "febf23ab04509bd7672e6abe76bd8277af31b679e89fa5ffc6087dc289a448a3"
 dependencies = [
  "axum",
  "axum-core",
- "bytes 1.4.0",
+ "bytes",
  "cookie",
  "futures-util",
  "http",
@@ -317,7 +317,7 @@ dependencies = [
  "base64 0.13.1",
  "dyn-clone",
  "eyre",
- "futures 0.3.28",
+ "futures",
  "ring",
  "secrecy",
  "serde",
@@ -350,7 +350,7 @@ dependencies = [
  "async-session",
  "axum",
  "axum-extra",
- "futures 0.3.28",
+ "futures",
  "http-body",
  "tokio",
  "tower",
@@ -476,16 +476,6 @@ version = "1.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
 
-[[package]]
-name = "bytes"
-version = "0.4.12"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
-dependencies = [
- "byteorder",
- "iovec",
-]
-
 [[package]]
 name = "bytes"
 version = "1.4.0"
@@ -775,12 +765,6 @@ dependencies = [
  "percent-encoding",
 ]
 
-[[package]]
-name = "futures"
-version = "0.1.31"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
-
 [[package]]
 name = "futures"
 version = "0.3.28"
@@ -879,7 +863,6 @@ dependencies = [
  "pin-project-lite",
  "pin-utils",
  "slab",
- "tokio-io",
 ]
 
 [[package]]
@@ -936,7 +919,7 @@ checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584"
 dependencies = [
  "base64 0.13.1",
  "bitflags 1.3.2",
- "bytes 1.4.0",
+ "bytes",
  "headers-core",
  "http",
  "httpdate",
@@ -1008,7 +991,7 @@ version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
 dependencies = [
- "bytes 1.4.0",
+ "bytes",
  "fnv",
  "itoa",
 ]
@@ -1019,7 +1002,7 @@ version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
 dependencies = [
- "bytes 1.4.0",
+ "bytes",
  "http",
  "pin-project-lite",
 ]
@@ -1057,7 +1040,7 @@ version = "0.14.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4"
 dependencies = [
- "bytes 1.4.0",
+ "bytes",
  "futures-channel",
  "futures-core",
  "futures-util",
@@ -1132,15 +1115,6 @@ dependencies = [
  "cfg-if 1.0.0",
 ]
 
-[[package]]
-name = "iovec"
-version = "0.1.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
-dependencies = [
- "libc",
-]
-
 [[package]]
 name = "is-terminal"
 version = "0.4.8"
@@ -1848,7 +1822,7 @@ dependencies = [
  "atoi",
  "bitflags 1.3.2",
  "byteorder",
- "bytes 1.4.0",
+ "bytes",
  "chrono",
  "crc",
  "crossbeam-queue",
@@ -2045,7 +2019,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2"
 dependencies = [
  "autocfg",
- "bytes 1.4.0",
+ "bytes",
  "libc",
  "mio",
  "num_cpus",
@@ -2058,17 +2032,6 @@ dependencies = [
  "windows-sys",
 ]
 
-[[package]]
-name = "tokio-io"
-version = "0.1.13"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
-dependencies = [
- "bytes 0.4.12",
- "futures 0.1.31",
- "log",
-]
-
 [[package]]
 name = "tokio-macros"
 version = "2.1.0"
@@ -2137,7 +2100,7 @@ checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"
 dependencies = [
  "base64 0.13.1",
  "bitflags 1.3.2",
- "bytes 1.4.0",
+ "bytes",
  "futures-core",
  "futures-util",
  "http",
@@ -2155,7 +2118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c"
 dependencies = [
  "bitflags 2.3.3",
- "bytes 1.4.0",
+ "bytes",
  "futures-core",
  "futures-util",
  "http",
@@ -2561,7 +2524,6 @@ dependencies = [
  "axum-test",
  "chrono",
  "clap",
- "futures-util",
  "justerror",
  "optional_optional_user",
  "password-hash",

Cargo.toml

@@ -30,7 +30,6 @@ ulid = { version = "1", features = ["rand"] }
 optional_optional_user = {path = "optional_optional_user"}
 chrono = { version = "0.4", default-features = false, features = ["std", "clock"] }
 clap = { version = "4.3.10", features = ["derive", "env", "unicode", "suggestions", "usage"] }
-futures-util = { version = "0.3.28", features = ["tokio-io"] }
 tokio-retry = "0.3.0"
 tokio-stream = "0.1.14"
 

results.txt

@@ -2,4 +2,4 @@
 -rw-r--r--   1 ardent ardent 161K Jul  4 12:29 .witch-watch.db-wal
 -rw-r--r--   1 ardent ardent  32K Jul  4 12:29 .witch-watch.db-shm
 
-3.5 seconds wall to add 10k movies, added by the omega user.
+4 seconds wall to add 10k movies, added by the omega user.

src/bin/import_omega.rs

@@ -1,12 +1,10 @@
 use std::{ffi::OsString, pin::Pin, time::Duration};
 
 use clap::Parser;
-use futures_util::Stream;
-//use futures_util::stream::TryStreamExt;
 use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
 use tokio::task::JoinSet;
 use tokio_retry::Retry;
-use tokio_stream::StreamExt;
+use tokio_stream::{Stream, StreamExt};
 use witch_watch::{
     get_db_pool,
     import_utils::{add_watch_omega, ensure_omega, ImportMovieOmega},

src/db.rs

@@ -9,7 +9,7 @@ use session_store::SqliteSessionStore;
 use sqlx::{
     migrate::Migrator,
     sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions},
-    Executor, SqlitePool,
+    SqlitePool,
 };
 
 use crate::{db_id::DbId, User};
@@ -45,9 +45,9 @@ pub async fn get_db_pool() -> SqlitePool {
 
     let conn_opts = SqliteConnectOptions::new()
         .foreign_keys(true)
-        //.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental)
+        .auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental)
         .journal_mode(SqliteJournalMode::Wal)
-        .synchronous(sqlx::sqlite::SqliteSynchronous::Off)
+        .synchronous(sqlx::sqlite::SqliteSynchronous::Normal)
         .filename(&db_filename)
         .busy_timeout(Duration::from_secs(TIMEOUT))
         .create_if_missing(true);
@@ -61,10 +61,6 @@ pub async fn get_db_pool() -> SqlitePool {
         .await
         .expect("can't connect to database");
 
-    // let mut conn = pool.acquire().await.unwrap();
-    // conn.execute("PRAGMA cache_size = 1000000;").await.unwrap();
-    // conn.execute("PRAGMA temp_store = MEMORY;").await.unwrap();
-
     // let the filesystem settle before trying anything
     // possibly not effective?
     tokio::time::sleep(Duration::from_millis(500)).await;

src/import_utils.rs

@@ -8,7 +8,7 @@ const USER_EXISTS_QUERY: &str = "select count(*) from witches where id = $1";
 
 const OMEGA_ID: u128 = u128::MAX;
 
-#[derive(Debug, sqlx::FromRow)]
+#[derive(Debug, sqlx::FromRow, Clone)]
 pub struct ImportMovieOmega {
     pub title: String,
     pub year: Option<String>,