Update db config to production-friendly.
Four seconds to insert 10k records, with a prod-safe config. Stick with this design.
This commit is contained in:
Joe Ardent
parent
24926fed6a
commit
5e4f5c07d8
6 changed files with 19 additions and 64 deletions
64
Cargo.lock
generated
64
Cargo.lock
generated
|
|
@ -241,7 +241,7 @@ dependencies = [
|
|||
"axum-core",
|
||||
"axum-macros",
|
||||
"bitflags 1.3.2",
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"headers",
|
||||
"http",
|
||||
|
|
@ -272,7 +272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http",
|
||||
"http-body",
|
||||
|
|
@ -290,7 +290,7 @@ checksum = "febf23ab04509bd7672e6abe76bd8277af31b679e89fa5ffc6087dc289a448a3"
|
|||
dependencies = [
|
||||
"axum",
|
||||
"axum-core",
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"cookie",
|
||||
"futures-util",
|
||||
"http",
|
||||
|
|
@ -317,7 +317,7 @@ dependencies = [
|
|||
"base64 0.13.1",
|
||||
"dyn-clone",
|
||||
"eyre",
|
||||
"futures 0.3.28",
|
||||
"futures",
|
||||
"ring",
|
||||
"secrecy",
|
||||
"serde",
|
||||
|
|
@ -350,7 +350,7 @@ dependencies = [
|
|||
"async-session",
|
||||
"axum",
|
||||
"axum-extra",
|
||||
"futures 0.3.28",
|
||||
"futures",
|
||||
"http-body",
|
||||
"tokio",
|
||||
"tower",
|
||||
|
|
@ -476,16 +476,6 @@ version = "1.4.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "0.4.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"iovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.4.0"
|
||||
|
|
@ -775,12 +765,6 @@ dependencies = [
|
|||
"percent-encoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.1.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.28"
|
||||
|
|
@ -879,7 +863,6 @@ dependencies = [
|
|||
"pin-project-lite",
|
||||
"pin-utils",
|
||||
"slab",
|
||||
"tokio-io",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -936,7 +919,7 @@ checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584"
|
|||
dependencies = [
|
||||
"base64 0.13.1",
|
||||
"bitflags 1.3.2",
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"headers-core",
|
||||
"http",
|
||||
"httpdate",
|
||||
|
|
@ -1008,7 +991,7 @@ version = "0.2.9"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
|
||||
dependencies = [
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"fnv",
|
||||
"itoa",
|
||||
]
|
||||
|
|
@ -1019,7 +1002,7 @@ version = "0.4.5"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
|
||||
dependencies = [
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"http",
|
||||
"pin-project-lite",
|
||||
]
|
||||
|
|
@ -1057,7 +1040,7 @@ version = "0.14.26"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4"
|
||||
dependencies = [
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
|
|
@ -1132,15 +1115,6 @@ dependencies = [
|
|||
"cfg-if 1.0.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iovec"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "is-terminal"
|
||||
version = "0.4.8"
|
||||
|
|
@ -1848,7 +1822,7 @@ dependencies = [
|
|||
"atoi",
|
||||
"bitflags 1.3.2",
|
||||
"byteorder",
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"crc",
|
||||
"crossbeam-queue",
|
||||
|
|
@ -2045,7 +2019,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"libc",
|
||||
"mio",
|
||||
"num_cpus",
|
||||
|
|
@ -2058,17 +2032,6 @@ dependencies = [
|
|||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-io"
|
||||
version = "0.1.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
|
||||
dependencies = [
|
||||
"bytes 0.4.12",
|
||||
"futures 0.1.31",
|
||||
"log",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio-macros"
|
||||
version = "2.1.0"
|
||||
|
|
@ -2137,7 +2100,7 @@ checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"
|
|||
dependencies = [
|
||||
"base64 0.13.1",
|
||||
"bitflags 1.3.2",
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
|
|
@ -2155,7 +2118,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c"
|
||||
dependencies = [
|
||||
"bitflags 2.3.3",
|
||||
"bytes 1.4.0",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"futures-util",
|
||||
"http",
|
||||
|
|
@ -2561,7 +2524,6 @@ dependencies = [
|
|||
"axum-test",
|
||||
"chrono",
|
||||
"clap",
|
||||
"futures-util",
|
||||
"justerror",
|
||||
"optional_optional_user",
|
||||
"password-hash",
|
||||
|
|
|
|||
|
|
@ -30,7 +30,6 @@ ulid = { version = "1", features = ["rand"] }
|
|||
optional_optional_user = {path = "optional_optional_user"}
|
||||
chrono = { version = "0.4", default-features = false, features = ["std", "clock"] }
|
||||
clap = { version = "4.3.10", features = ["derive", "env", "unicode", "suggestions", "usage"] }
|
||||
futures-util = { version = "0.3.28", features = ["tokio-io"] }
|
||||
tokio-retry = "0.3.0"
|
||||
tokio-stream = "0.1.14"
|
||||
|
||||
|
|
|
|||
|
|
@ -2,4 +2,4 @@
|
|||
-rw-r--r-- 1 ardent ardent 161K Jul 4 12:29 .witch-watch.db-wal
|
||||
-rw-r--r-- 1 ardent ardent 32K Jul 4 12:29 .witch-watch.db-shm
|
||||
|
||||
3.5 seconds wall to add 10k movies, added by the omega user.
|
||||
4 seconds wall to add 10k movies, added by the omega user.
|
||||
|
|
|
|||
|
|
@ -1,12 +1,10 @@
|
|||
use std::{ffi::OsString, pin::Pin, time::Duration};
|
||||
|
||||
use clap::Parser;
|
||||
use futures_util::Stream;
|
||||
//use futures_util::stream::TryStreamExt;
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
|
||||
use tokio::task::JoinSet;
|
||||
use tokio_retry::Retry;
|
||||
use tokio_stream::StreamExt;
|
||||
use tokio_stream::{Stream, StreamExt};
|
||||
use witch_watch::{
|
||||
get_db_pool,
|
||||
import_utils::{add_watch_omega, ensure_omega, ImportMovieOmega},
|
||||
|
|
|
|||
10
src/db.rs
10
src/db.rs
|
|
@ -9,7 +9,7 @@ use session_store::SqliteSessionStore;
|
|||
use sqlx::{
|
||||
migrate::Migrator,
|
||||
sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions},
|
||||
Executor, SqlitePool,
|
||||
SqlitePool,
|
||||
};
|
||||
|
||||
use crate::{db_id::DbId, User};
|
||||
|
|
@ -45,9 +45,9 @@ pub async fn get_db_pool() -> SqlitePool {
|
|||
|
||||
let conn_opts = SqliteConnectOptions::new()
|
||||
.foreign_keys(true)
|
||||
//.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental)
|
||||
.auto_vacuum(sqlx::sqlite::SqliteAutoVacuum::Incremental)
|
||||
.journal_mode(SqliteJournalMode::Wal)
|
||||
.synchronous(sqlx::sqlite::SqliteSynchronous::Off)
|
||||
.synchronous(sqlx::sqlite::SqliteSynchronous::Normal)
|
||||
.filename(&db_filename)
|
||||
.busy_timeout(Duration::from_secs(TIMEOUT))
|
||||
.create_if_missing(true);
|
||||
|
|
@ -61,10 +61,6 @@ pub async fn get_db_pool() -> SqlitePool {
|
|||
.await
|
||||
.expect("can't connect to database");
|
||||
|
||||
// let mut conn = pool.acquire().await.unwrap();
|
||||
// conn.execute("PRAGMA cache_size = 1000000;").await.unwrap();
|
||||
// conn.execute("PRAGMA temp_store = MEMORY;").await.unwrap();
|
||||
|
||||
// let the filesystem settle before trying anything
|
||||
// possibly not effective?
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ const USER_EXISTS_QUERY: &str = "select count(*) from witches where id = $1";
|
|||
|
||||
const OMEGA_ID: u128 = u128::MAX;
|
||||
|
||||
#[derive(Debug, sqlx::FromRow)]
|
||||
#[derive(Debug, sqlx::FromRow, Clone)]
|
||||
pub struct ImportMovieOmega {
|
||||
pub title: String,
|
||||
pub year: Option<String>,
|
||||
|
|
|
|||
Loading…
Reference in a new issue