imports 10k random movies

This commit is contained in:
Joe Ardent 2023-07-03 15:20:19 -07:00
parent 83da336a3f
commit 825db88039
4 changed files with 299 additions and 33 deletions

246
Cargo.lock generated
View file

@ -45,6 +45,55 @@ dependencies = [
"libc",
]
[[package]]
name = "anstream"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"is-terminal",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd"
[[package]]
name = "anstyle-parse"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b"
dependencies = [
"windows-sys",
]
[[package]]
name = "anstyle-wincon"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188"
dependencies = [
"anstyle",
"windows-sys",
]
[[package]]
name = "anyhow"
version = "1.0.71"
@ -192,7 +241,7 @@ dependencies = [
"axum-core",
"axum-macros",
"bitflags 1.3.2",
"bytes",
"bytes 1.4.0",
"futures-util",
"headers",
"http",
@ -223,7 +272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "759fa577a247914fd3f7f76d62972792636412fbfd634cd452f6a385a74d2d2c"
dependencies = [
"async-trait",
"bytes",
"bytes 1.4.0",
"futures-util",
"http",
"http-body",
@ -241,7 +290,7 @@ checksum = "febf23ab04509bd7672e6abe76bd8277af31b679e89fa5ffc6087dc289a448a3"
dependencies = [
"axum",
"axum-core",
"bytes",
"bytes 1.4.0",
"cookie",
"futures-util",
"http",
@ -268,7 +317,7 @@ dependencies = [
"base64 0.13.1",
"dyn-clone",
"eyre",
"futures",
"futures 0.3.28",
"ring",
"secrecy",
"serde",
@ -301,7 +350,7 @@ dependencies = [
"async-session",
"axum",
"axum-extra",
"futures",
"futures 0.3.28",
"http-body",
"tokio",
"tower",
@ -369,9 +418,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.3.2"
version = "2.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6dbe3c979c178231552ecba20214a8272df4e09f232a87aef4320cf06539aded"
checksum = "630be753d4e58660abd17930c71b647fe46c27ea6b63cc59e1e3851406972e42"
[[package]]
name = "blake2"
@ -427,6 +476,16 @@ version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c"
dependencies = [
"byteorder",
"iovec",
]
[[package]]
name = "bytes"
version = "1.4.0"
@ -464,6 +523,55 @@ dependencies = [
"winapi",
]
[[package]]
name = "clap"
version = "4.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384e169cc618c613d5e3ca6404dda77a8685a63e08660dcc64abaf7da7cb0c7a"
dependencies = [
"clap_builder",
"clap_derive",
"once_cell",
]
[[package]]
name = "clap_builder"
version = "4.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef137bbe35aab78bdb468ccfba75a5f4d8321ae011d34063770780545176af2d"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim",
"unicase",
"unicode-width",
]
[[package]]
name = "clap_derive"
version = "4.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f"
dependencies = [
"heck",
"proc-macro2",
"quote",
"syn 2.0.18",
]
[[package]]
name = "clap_lex"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b"
[[package]]
name = "colorchoice"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "constant_time_eq"
version = "0.1.5"
@ -603,6 +711,27 @@ version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "errno"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a"
dependencies = [
"errno-dragonfly",
"libc",
"windows-sys",
]
[[package]]
name = "errno-dragonfly"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
dependencies = [
"cc",
"libc",
]
[[package]]
name = "event-listener"
version = "2.5.3"
@ -646,6 +775,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "futures"
version = "0.1.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a471a38ef8ed83cd6e40aa59c1ffe17db6855c18e3604d9c4ed8c08ebc28678"
[[package]]
name = "futures"
version = "0.3.28"
@ -744,6 +879,7 @@ dependencies = [
"pin-project-lite",
"pin-utils",
"slab",
"tokio-io",
]
[[package]]
@ -800,7 +936,7 @@ checksum = "f3e372db8e5c0d213e0cd0b9be18be2aca3d44cf2fe30a9d46a65581cd454584"
dependencies = [
"base64 0.13.1",
"bitflags 1.3.2",
"bytes",
"bytes 1.4.0",
"headers-core",
"http",
"httpdate",
@ -835,6 +971,12 @@ dependencies = [
"libc",
]
[[package]]
name = "hermit-abi"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286"
[[package]]
name = "hex"
version = "0.4.3"
@ -866,7 +1008,7 @@ version = "0.2.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482"
dependencies = [
"bytes",
"bytes 1.4.0",
"fnv",
"itoa",
]
@ -877,7 +1019,7 @@ version = "0.4.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1"
dependencies = [
"bytes",
"bytes 1.4.0",
"http",
"pin-project-lite",
]
@ -915,7 +1057,7 @@ version = "0.14.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4"
dependencies = [
"bytes",
"bytes 1.4.0",
"futures-channel",
"futures-core",
"futures-util",
@ -990,6 +1132,26 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "iovec"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e"
dependencies = [
"libc",
]
[[package]]
name = "is-terminal"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24fddda5af7e54bf7da53067d6e802dbcc381d0a8eef629df528e3ebf68755cb"
dependencies = [
"hermit-abi 0.3.1",
"rustix",
"windows-sys",
]
[[package]]
name = "itertools"
version = "0.10.5"
@ -1054,6 +1216,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "linux-raw-sys"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09fc20d2ca12cb9f044c93e3bd6d32d523e6e2ec3db4f7b2939cd99026ecd3f0"
[[package]]
name = "lock_api"
version = "0.4.10"
@ -1159,7 +1327,7 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b"
dependencies = [
"hermit-abi",
"hermit-abi 0.2.6",
"libc",
]
@ -1424,6 +1592,19 @@ dependencies = [
"winapi",
]
[[package]]
name = "rustix"
version = "0.38.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aabcb0461ebd01d6b79945797c27f8529082226cb630a9865a71870ff63532a4"
dependencies = [
"bitflags 2.3.3",
"errno",
"libc",
"linux-raw-sys",
"windows-sys",
]
[[package]]
name = "rustls"
version = "0.20.8"
@ -1667,7 +1848,7 @@ dependencies = [
"atoi",
"bitflags 1.3.2",
"byteorder",
"bytes",
"bytes 1.4.0",
"chrono",
"crc",
"crossbeam-queue",
@ -1745,6 +1926,12 @@ dependencies = [
"unicode-normalization",
]
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "subtle"
version = "2.4.1"
@ -1858,7 +2045,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2"
dependencies = [
"autocfg",
"bytes",
"bytes 1.4.0",
"libc",
"mio",
"num_cpus",
@ -1871,6 +2058,17 @@ dependencies = [
"windows-sys",
]
[[package]]
name = "tokio-io"
version = "0.1.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674"
dependencies = [
"bytes 0.4.12",
"futures 0.1.31",
"log",
]
[[package]]
name = "tokio-macros"
version = "2.1.0"
@ -1928,7 +2126,7 @@ checksum = "f873044bf02dd1e8239e9c1293ea39dad76dc594ec16185d0a1bf31d8dc8d858"
dependencies = [
"base64 0.13.1",
"bitflags 1.3.2",
"bytes",
"bytes 1.4.0",
"futures-core",
"futures-util",
"http",
@ -1945,8 +2143,8 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8bd22a874a2d0b70452d5597b12c537331d49060824a95f49f108994f94aa4c"
dependencies = [
"bitflags 2.3.2",
"bytes",
"bitflags 2.3.3",
"bytes 1.4.0",
"futures-core",
"futures-util",
"http",
@ -2090,6 +2288,12 @@ version = "1.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
[[package]]
name = "unicode-width"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b"
[[package]]
name = "unicode_categories"
version = "0.1.1"
@ -2113,6 +2317,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "utf8parse"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a"
[[package]]
name = "valuable"
version = "0.1.0"
@ -2339,6 +2549,8 @@ dependencies = [
"axum-macros",
"axum-test",
"chrono",
"clap",
"futures-util",
"justerror",
"optional_optional_user",
"password-hash",

View file

@ -2,6 +2,7 @@
name = "witch_watch"
version = "0.0.1"
edition = "2021"
default-run = "witch_watch"
[dependencies]
axum = { version = "0.6", features = ["macros", "headers"] }
@ -28,6 +29,8 @@ ulid = { version = "1", features = ["rand"] }
# proc macros:
optional_optional_user = {path = "optional_optional_user"}
chrono = { version = "0.4", default-features = false, features = ["std", "clock"] }
clap = { version = "4.3.10", features = ["derive", "env", "unicode", "suggestions", "usage"] }
futures-util = { version = "0.3.28", features = ["tokio-io"] }
[dev-dependencies]
axum-test = "9.0.0"

40
src/bin/import_omega.rs Normal file
View file

@ -0,0 +1,40 @@
use std::{ffi::OsString, time::Duration};
use clap::Parser;
use futures_util::stream::TryStreamExt;
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use witch_watch::{
get_db_pool,
import_utils::{add_watch_omega, ensure_omega},
};
const MOVIE_QUERY: &str = "select * from movies order by random() limit 10000";
#[derive(Debug, Parser)]
struct Cli {
#[clap(long, short)]
pub db_path: OsString,
}
#[tokio::main]
async fn main() {
let cli = Cli::parse();
let path = cli.db_path;
let opts = SqliteConnectOptions::new().filename(&path).read_only(true);
let movie_db = SqlitePoolOptions::new()
.idle_timeout(Duration::from_secs(90))
.connect_with(opts)
.await
.expect("could not open movies db");
let ww_db = get_db_pool().await;
let mut movies = sqlx::query_as(MOVIE_QUERY).fetch(&movie_db);
ensure_omega(&ww_db).await;
while let Ok(Some(movie)) = movies.try_next().await {
add_watch_omega(&ww_db, movie).await;
}
}

View file

@ -1,28 +1,40 @@
use sqlx::{query, query_scalar, SqlitePool};
use crate::{db_id::DbId, Watch};
use crate::{
db_id::DbId, util::year_to_epoch, watches::handlers::add_new_watch_impl, ShowKind, Watch,
};
const USER_EXISTS_QUERY: &str = "select count(*) from witches where id = $1";
const ADD_WATCH_QUERY: &str = "insert into watches (id, title, kind, metadata_url, length, release_date, added_by) values ($1, $2, $3, $4, $5, $6, $7)";
const OMEGA_ID: u128 = u128::MAX;
#[derive(Debug, sqlx::FromRow)]
pub struct ImportMovieOmega {
pub title: String,
pub year: Option<String>,
pub length: Option<String>,
}
impl From<ImportMovieOmega> for Watch {
fn from(value: ImportMovieOmega) -> Self {
Watch {
title: value.title,
release_date: year_to_epoch(value.year.as_deref()),
length: value.length.and_then(|v| v.parse::<i64>().ok()),
id: DbId::new(),
kind: ShowKind::Movie,
metadata_url: None,
added_by: OMEGA_ID.into(),
}
}
}
//-************************************************************************
// utility functions for building CLI tools, currently just for benchmarking
//-************************************************************************
pub async fn add_watch(db_pool: &SqlitePool, watch: &Watch) {
if query(ADD_WATCH_QUERY)
.bind(watch.id)
.bind(&watch.title)
.bind(watch.kind)
.bind(&watch.metadata_url)
.bind(watch.length)
.bind(watch.release_date)
.bind(watch.added_by)
.execute(db_pool)
.await
.is_ok()
{
pub async fn add_watch_omega(db_pool: &SqlitePool, movie: ImportMovieOmega) {
let watch: Watch = movie.into();
if add_new_watch_impl(db_pool, &watch, None).await.is_ok() {
println!("{}", watch.id);
} else {
eprintln!("failed to add \"{}\"", watch.title);
@ -75,7 +87,6 @@ async fn check_omega_exists(db_pool: &SqlitePool) -> bool {
.fetch_one(db_pool)
.await
.unwrap_or(0);
dbg!(count);
count > 0
}