redo migrations

This commit is contained in:
Joe Ardent 2024-01-16 20:21:38 -08:00
parent 8ca6751a87
commit fab52203e5
19 changed files with 367 additions and 252 deletions

View file

@ -1,16 +0,0 @@
-- indices
drop index if exists user_username_dex;
drop index if exists user_email_dex;
drop index if exists watch_title_dex;
drop index if exists witch_added_by_dex;
drop index if exists quests_user_dex;
drop index if exists quests_watch_dex;
drop index if exists note_user_dex;
drop index if exists note_watch_dex;
-- tables
drop table if exists watch_quests;
drop table if exists watch_notes;
drop table if exists follows;
drop table if exists users;
drop table if exists watches;

View file

@ -1,5 +0,0 @@
drop trigger if exists update_last_updated_users;
drop trigger if exists update_last_updated_watches;
drop trigger if exists update_last_updated_watch_quests;
drop trigger if exists update_last_updated_follows;
drop trigger if exists update_last_updated_watch_notes;

View file

@ -1,34 +0,0 @@
create trigger if not exists update_last_updated_users
after update on users
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update users set last_updated = (select unixepoch()) where id=NEW.id;
END;
create trigger if not exists update_last_updated_invites
after update on invites
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update invites set last_updated = (select unixepoch()) where id=NEW.id;
END;
create trigger if not exists update_last_updated_watches
after update on watches
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update watches set last_updated = (select unixepoch()) where id=NEW.id;
END;
create trigger if not exists update_last_updated_watch_quests
after update on watch_quests
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update watch_quests set last_updated = (select unixepoch()) where watch=NEW.watch and user=NEW.user;
END;
create trigger if not exists update_last_updated_watch_notes
after update on watch_notes
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update watch_notes set last_updated = (select unixepoch()) where id=NEW.id;
END;

View file

@ -0,0 +1,2 @@
drop table if exists users;
drop table if exists invites;

View file

@ -0,0 +1,41 @@
create table if not exists users (
id blob not null primary key default (julid_new()),
username text not null unique,
displayname text,
email text,
last_seen int,
pwhash blob not null,
invited_by blob not null,
is_active boolean not null default true,
last_updated int not null default (unixepoch()),
foreign key (invited_by) references users (id)
);
create index if not exists users_username_dex on users (lower(username));
create index if not exists users_email_dex on users (lower(email));
create index if not exists users_invited_by_dex on users (invited_by);
create trigger if not exists update_last_updated_users
after update on users
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update users set last_updated = (select unixepoch()) where id=NEW.id;
END;
-- invitations
create table if not exists invites (
id blob not null primary key default (julid_new()),
owner blob not null,
expires_at int,
remaining int not null default 1,
last_updated int not null default (unixepoch()),
foreign key (owner) references users (id) on delete cascade on update no action
);
create index if not exists invites_owner_dex on invites (owner);
create trigger if not exists update_last_updated_invites
after update on invites
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update invites set last_updated = (select unixepoch()) where id=NEW.id;
END;

View file

@ -0,0 +1,3 @@
drop table if exists watches;
drop table if exists watch_quests;
drop table if exists watch_notes;

View file

@ -1,37 +1,3 @@
-- note: sqlite-specific migration due to the types of the columns
-- When used for an ID, a blob is a UUID in byte form, or a vector of those like for friends list.
-- Otherwise, for content, a blob is just binary data, possibly representing UTF-8 text.
-- Dates are ints, unix epoch style
-- users
create table if not exists users (
id blob not null primary key default (julid_new()),
username text not null unique,
displayname text,
email text,
last_seen int,
pwhash blob not null,
invited_by blob not null,
is_active int not null default 1,
last_updated int not null default (unixepoch()),
foreign key (invited_by) references users (id)
);
create index if not exists users_username_dex on users (lower(username));
create index if not exists users_email_dex on users (lower(email));
create index if not exists users_invited_by_dex on users (invited_by);
-- invitations
create table if not exists invites (
id blob not null primary key default (julid_new()),
owner blob not null,
expires_at int,
remaining int not null default 1,
last_updated int not null default (unixepoch()),
foreign key (owner) references users (id) on delete cascade on update no action
);
create index if not exists invites_owner_dex on invites (owner);
-- table of things to watch
create table if not exists watches (
id blob not null primary key default (julid_new()),
@ -46,6 +12,13 @@ create table if not exists watches (
);
create index if not exists watches_title_dex on watches (lower(title));
create trigger if not exists update_last_updated_watches
after update on watches
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update watches set last_updated = (select unixepoch()) where id=NEW.id;
END;
-- table of what people want to watch
create table if not exists watch_quests (
user blob not null,
@ -63,16 +36,14 @@ create table if not exists watch_quests (
create index if not exists quests_user_dex on watch_quests (user);
create index if not exists quests_watch_dex on watch_quests (watch);
create table if not exists follows (
follower blob not null,
followee blob not null,
created_at int not null default (unixepoch()),
foreign key (follower) references users (id) on delete cascade on update no action
foreign key (followee) references users (id) on delete cascade on update no action
);
create index if not exists follows_follower_dex on follows (follower);
create index if not exists follows_followee_dex on follows (followee);
create trigger if not exists update_last_updated_watch_quests
after update on watch_quests
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update watch_quests set last_updated = (select unixepoch()) where watch=NEW.watch and user=NEW.user;
END;
-- notes on stuff to watch
create table if not exists watch_notes (
id blob not null primary key default (julid_new()), -- a user can have multiple notes about the same thing
user blob not null,
@ -86,4 +57,9 @@ create table if not exists watch_notes (
create index if not exists notes_user_dex on watch_notes (user);
create index if not exists notes_watch_dex on watch_notes (watch);
-- indices
create trigger if not exists update_last_updated_watch_notes
after update on watch_notes
when OLD.last_updated = NEW.last_updated or OLD.last_updated is null
BEGIN
update watch_notes set last_updated = (select unixepoch()) where id=NEW.id;
END;

View file

@ -0,0 +1,2 @@
drop table if exists credits; -- must be first
drop table if exists stars;

View file

@ -0,0 +1,20 @@
create table if not exists stars (
id blob not null primary key default (julid_new()),
name text not null,
metadata_url text,
born int,
died int
);
create index if not exists stars_name_dex on stars (lower(name));
-- as in screen credits for a movie
create table if not exists credits (
star blob not null,
watch blob not null,
credit text, -- "actor", "director", whatevs
unique (star, watch, credit),
foreign key (star) references stars (id),
foreign key (watch) references watches (id)
);
create index if not exists credits_star_dex on credits (star);
create index if not exists credits_watch_dex on credits (watch);

View file

@ -0,0 +1 @@
drop table if exists follows;

View file

@ -0,0 +1,11 @@
create table if not exists follows (
follower blob not null,
followee blob not null,
created_at int not null default (unixepoch()),
foreign key (follower) references users (id) on delete cascade on update no action,
foreign key (followee) references users (id) on delete cascade on update no action,
unique (follower, followee)
);
create index if not exists follows_follower_dex on follows (follower);
create index if not exists follows_followee_dex on follows (followee);

View file

@ -1,8 +1,15 @@
use std::{ffi::OsString, time::Duration};
use std::{
ffi::{OsStr, OsString},
path::Path,
time::Duration,
};
use clap::Parser;
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use what2watch::{get_db_pool, import_utils::add_imdb_movies};
use sqlx::{
sqlite::{SqliteConnectOptions, SqlitePoolOptions},
Connection, SqliteConnection, SqlitePool,
};
use what2watch::{get_db_pool, imdb_utils::*};
#[derive(Debug, Parser)]
struct Cli {
@ -18,49 +25,62 @@ struct Cli {
}
fn main() {
let w2w_db = get_db_pool();
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
let cli = Cli::parse();
let path = cli.db_path;
let ids = rt.block_on(import_watches(&w2w_db, &cli));
rt.block_on(save_ids(&cli.db_path, &ids));
}
async fn import_watches(w2w_db: &SqlitePool, cli: &Cli) -> IdMap {
let path = &cli.db_path;
let num = cli.number;
let opts = SqliteConnectOptions::new().filename(path).read_only(true);
let movie_db = {
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
rt.block_on(
SqlitePoolOptions::new()
.idle_timeout(Duration::from_secs(90))
.connect_with(opts),
)
.expect("could not open movies db")
};
let movie_db = SqlitePoolOptions::new()
.idle_timeout(Duration::from_secs(90))
.connect_with(opts)
.await
.unwrap();
let w2w_db = get_db_pool();
let _ = what2watch::import_utils::ensure_omega(w2w_db).await;
let (dur, rows) = {
let rt = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.unwrap();
let mut map = IdMap::new();
rt.block_on(async {
let dur = add_imdb_movies(&w2w_db, &movie_db, num).await.unwrap();
let rows: i32 = sqlx::query_scalar("select count(*) from watches")
.fetch_one(&w2w_db)
.await
.unwrap();
import_imdb_data(w2w_db, &movie_db, &mut map, num).await;
w2w_db.close().await;
w2w_db.close().await;
(dur, rows)
})
};
println!(
"Added {rows} movies in {} seconds ({}ms, {}us)",
dur.as_secs_f64(),
dur.as_millis(),
dur.as_micros()
);
map
}
async fn save_ids(path: &OsStr, ids: &IdMap) {
let path = Path::new(path);
let file = path.file_name().unwrap();
let file = file.to_str().unwrap();
let path = format!("{}/w2w-{file}", path.parent().unwrap().to_str().unwrap());
let conn_opts = SqliteConnectOptions::new()
.filename(path)
.create_if_missing(true);
let mut conn = SqliteConnection::connect_with(&conn_opts).await.unwrap();
let create =
"create table if not exists id_map (imdb text not null primary key, w2w blob not null)";
let _ = sqlx::query(create).execute(&mut conn).await.unwrap();
for (imdb, w2w) in ids.iter() {
sqlx::query("insert into id_map (imdb, w2w) values (?, ?)")
.bind(imdb)
.bind(w2w)
.execute(&mut conn)
.await
.unwrap();
}
conn.close().await.unwrap();
}

149
src/imdb_utils.rs Normal file
View file

@ -0,0 +1,149 @@
use julid::Julid;
use sqlx::{Sqlite, SqlitePool};
use crate::{
import_utils::{insert_credit, insert_star, insert_watch},
misc_util::year_to_epoch,
Credit, ShowKind, Star, Watch,
};
pub type IdMap = std::collections::BTreeMap<String, Julid>;
const OMEGA_ID: Julid = Julid::omega();
#[derive(Debug, sqlx::FromRow, Clone)]
pub struct ImportImdbMovie {
pub title: String,
pub year: Option<String>,
pub length: Option<String>,
pub id: String,
pub kind: Option<String>,
}
impl From<ImportImdbMovie> for Watch {
fn from(value: ImportImdbMovie) -> Self {
Watch {
id: OMEGA_ID, // this is ignored by the inserter
title: value.title,
release_date: year_to_epoch(value.year.as_deref()),
length: value.length.and_then(|v| v.parse::<i64>().ok()),
kind: ShowKind::Movie,
metadata_url: Some(format!("https://imdb.com/title/{}/", &value.id)),
added_by: OMEGA_ID,
}
}
}
impl From<&ImportImdbMovie> for Watch {
fn from(value: &ImportImdbMovie) -> Self {
Watch {
id: OMEGA_ID,
title: value.title.to_string(),
release_date: year_to_epoch(value.year.as_deref()),
length: value.length.as_ref().and_then(|v| v.parse::<i64>().ok()),
kind: ShowKind::Movie,
metadata_url: Some(format!("https://imdb.com/title/{}/", value.id)),
added_by: OMEGA_ID,
}
}
}
#[derive(Debug, sqlx::FromRow, Clone)]
pub struct ImdbStar {
pub nconst: String,
#[sqlx(rename = "primaryName")]
pub name: String,
#[sqlx(rename = "birthYear")]
pub born: Option<String>,
#[sqlx(rename = "deathYear")]
pub died: Option<String>,
}
impl From<&ImdbStar> for Star {
fn from(value: &ImdbStar) -> Self {
let id = &value.nconst;
let metadata_url = Some(format!("https://imdb.com/name/{id}/"));
Self {
name: value.name.clone(),
metadata_url,
born: year_to_epoch(value.born.as_deref()),
died: year_to_epoch(value.died.as_deref()),
..Default::default()
}
}
}
pub async fn import_imdb_data(w2w_db: &SqlitePool, imdb: &SqlitePool, ids: &mut IdMap, num: u32) {
const IMDB_QUERY: &str = "select * from watches order by year, title asc limit ?";
let iwatches: Vec<ImportImdbMovie> = sqlx::query_as(IMDB_QUERY)
.bind(num)
.fetch_all(imdb)
.await
.unwrap();
for iwatch in iwatches {
let aid = iwatch.id.clone();
let kind = show_kind(iwatch.kind.as_ref().unwrap());
let mut watch: Watch = iwatch.into();
watch.kind = kind;
let watch_id: Julid = insert_watch(watch, w2w_db).await;
add_imdb_stars(w2w_db, imdb, &aid, watch_id, ids).await;
ids.insert(aid, watch_id);
}
}
async fn add_imdb_stars(
w2w_db: &SqlitePool,
imdb: &SqlitePool,
iwatch: &str,
watch: Julid,
ids: &mut IdMap,
) {
let principals_query = "select nconst, category from principals where tconst = ?";
let principals = sqlx::query_as::<Sqlite, (String, String)>(principals_query)
.bind(iwatch)
.fetch_all(imdb)
.await
.unwrap();
for row in principals {
let (name_id, cat) = row;
let name_query =
"select nconst, primaryName, birthYear, deathYear from names where nconst = ?";
let istar: Option<ImdbStar> = sqlx::query_as(name_query)
.bind(&name_id)
.fetch_optional(imdb)
.await
.unwrap();
if let Some(star) = istar {
let star = (&star).into();
let star_id = insert_star(&star, w2w_db).await;
ids.insert(name_id, star_id);
let credit = Credit {
star: star_id,
watch,
credit: Some(cat.to_string()),
};
insert_credit(&credit, w2w_db).await;
}
}
}
fn show_kind(kind: &str) -> ShowKind {
/*
tvSeries
tvMiniSeries
tvMovie
tvShort
tvSpecial
*/
match &kind[0..4] {
"tvSe" => ShowKind::Series,
"tvSh" => ShowKind::Short,
"tvMi" => ShowKind::LimitedSeries,
"tvSp" => ShowKind::Other,
"tvMo" | "movi" => ShowKind::Movie,
_ => ShowKind::Unknown,
}
}

View file

@ -1,131 +1,53 @@
use std::time::{Duration, Instant};
use julid::Julid;
use sqlx::{query_scalar, SqlitePool};
use crate::{util::year_to_epoch, ShowKind, User, Watch, WatchQuest};
const USER_EXISTS_QUERY: &str = "select count(*) from users where id = $1";
use crate::{Credit, Star, User, Watch};
//-************************************************************************
// the omega user is the system ID, but has no actual power in the app
//-************************************************************************
const OMEGA_ID: Julid = Julid::omega();
const BULK_INSERT: usize = 2_000;
#[derive(Debug, sqlx::FromRow, Clone)]
pub struct ImportImdbMovie {
pub title: String,
pub year: Option<String>,
pub runtime: Option<String>,
pub aid: String,
}
impl From<ImportImdbMovie> for Watch {
fn from(value: ImportImdbMovie) -> Self {
Watch {
id: OMEGA_ID, // this is ignored by the inserter
title: value.title,
release_date: year_to_epoch(value.year.as_deref()),
length: value.runtime.and_then(|v| v.parse::<i64>().ok()),
kind: ShowKind::Movie,
metadata_url: Some(format!("https://imdb.com/title/{}/", &value.aid)),
added_by: OMEGA_ID,
}
}
}
impl From<&ImportImdbMovie> for Watch {
fn from(value: &ImportImdbMovie) -> Self {
Watch {
id: OMEGA_ID,
title: value.title.to_string(),
release_date: year_to_epoch(value.year.as_deref()),
length: value.runtime.as_ref().and_then(|v| v.parse::<i64>().ok()),
kind: ShowKind::Movie,
metadata_url: Some(format!("https://imdb.com/title/{}/", value.aid)),
added_by: OMEGA_ID,
}
}
}
//-************************************************************************
// utility functions for building CLI tools, currently just for benchmarking
//-************************************************************************
pub async fn add_watch_quests(pool: &SqlitePool, quests: &[WatchQuest]) -> Result<(), ()> {
let mut builder = sqlx::QueryBuilder::new("insert into watch_quests (user, watch) ");
builder.push_values(quests, |mut b, quest| {
let user = quest.user;
let watch = quest.watch;
//eprintln!("{user}, {watch}");
b.push_bind(user).push_bind(watch);
});
let q = builder.build();
q.execute(pool).await.map_err(|e| {
dbg!(e);
})?;
Ok(())
pub async fn insert_watch(watch: Watch, db: &SqlitePool) -> Julid {
let q = "insert into watches (kind, title, length, release_date, added_by, metadata_url) values (?, ?, ?, ?, ?, ?) returning id";
sqlx::query_scalar(q)
.bind(watch.kind)
.bind(watch.title)
.bind(watch.length)
.bind(watch.release_date)
.bind(watch.added_by)
.bind(watch.metadata_url)
.fetch_one(db)
.await
.unwrap()
}
pub async fn add_users(db_pool: &SqlitePool, users: &[User]) -> Result<Duration, ()> {
let mut builder =
sqlx::QueryBuilder::new("insert into users (username, displayname, email, pwhash)");
let start = Instant::now();
builder.push_values(users.iter(), |mut b, user| {
b.push_bind(&user.username)
.push_bind(&user.displayname)
.push_bind(&user.email)
.push_bind(&user.pwhash);
});
let q = builder.build();
q.execute(db_pool).await.map_err(|_| ())?;
let end = Instant::now();
let dur = end - start;
Ok(dur)
pub async fn insert_star(star: &Star, db: &SqlitePool) -> Julid {
let q = "insert into stars (name, metadata_url, born, died) values (?, ?, ?, ?) returning id";
sqlx::query_scalar(q)
.bind(&star.name)
.bind(&star.metadata_url)
.bind(star.born)
.bind(star.died)
.fetch_one(db)
.await
.unwrap()
}
pub async fn add_imdb_movies(
w2w_db: &SqlitePool,
movie_db: &SqlitePool,
num: u32,
) -> Result<Duration, ()> {
const IMDB_MOVIE_QUERY: &str =
"select * from movie_titles where porn = 0 order by year, title asc limit ?";
pub async fn insert_credit(credit: &Credit, db: &SqlitePool) {
let q = "insert into credits (star, watch, credit) values (?, ?, ?)";
let omega = ensure_omega(w2w_db).await;
let movies: Vec<ImportImdbMovie> = sqlx::query_as(IMDB_MOVIE_QUERY)
.bind(num)
.fetch_all(movie_db)
sqlx::query(q)
.bind(credit.star)
.bind(credit.watch)
.bind(credit.credit.as_deref())
.execute(db)
.await
.unwrap();
let start = Instant::now();
for movies in movies.as_slice().chunks(BULK_INSERT) {
let mut builder = sqlx::QueryBuilder::new(
"insert into watches (kind, title, length, release_date, added_by, metadata_url) ",
);
builder.push_values(movies, |mut b, movie| {
let movie: Watch = movie.into();
b.push_bind(ShowKind::Movie)
.push_bind(movie.title)
.push_bind(movie.length)
.push_bind(movie.release_date)
.push_bind(omega)
.push_bind(movie.metadata_url);
});
let q = builder.build();
q.execute(w2w_db).await.map_err(|_| ())?;
}
let end = Instant::now();
let dur = end - start;
Ok(dur)
}
pub async fn ensure_omega(db_pool: &SqlitePool) -> Julid {
@ -136,6 +58,7 @@ pub async fn ensure_omega(db_pool: &SqlitePool) -> Julid {
}
async fn check_omega_exists(db_pool: &SqlitePool) -> bool {
const USER_EXISTS_QUERY: &str = "select count(*) from users where id = $1";
let count = query_scalar(USER_EXISTS_QUERY)
.bind(OMEGA_ID)
.fetch_one(db_pool)

View file

@ -11,9 +11,12 @@ extern crate justerror;
/// Some public interfaces for interacting with the database outside of the web
/// app
pub use db::get_db_pool;
pub mod imdb_utils;
pub mod import_utils;
pub mod misc_util;
pub use signup::Invitation;
pub use stars::*;
pub use users::User;
pub use watches::{ShowKind, Watch, WatchQuest};
@ -25,9 +28,9 @@ mod db;
mod generic_handlers;
mod login;
mod signup;
mod stars;
mod templates;
mod users;
mod util;
mod watches;
// things we want in the crate namespace

View file

@ -2,7 +2,7 @@ use std::{error::Error, ops::Range};
use unicode_segmentation::UnicodeSegmentation;
pub fn validate_optional_length<E: Error>(
pub(crate) fn validate_optional_length<E: Error>(
opt: &Option<String>,
len_range: Range<usize>,
err: E,
@ -21,7 +21,7 @@ pub fn validate_optional_length<E: Error>(
}
/// Serde deserialization decorator to map empty Strings to None
pub fn empty_string_as_none<'de, D, T>(de: D) -> Result<Option<T>, D::Error>
pub(crate) fn empty_string_as_none<'de, D, T>(de: D) -> Result<Option<T>, D::Error>
where
D: serde::Deserializer<'de>,
T: std::str::FromStr,

View file

@ -13,7 +13,7 @@ use sqlx::{query_as, Sqlite, SqlitePool};
use unicode_segmentation::UnicodeSegmentation;
use super::{templates::*, Invitation};
use crate::{util::empty_string_as_none, User};
use crate::{misc_util::empty_string_as_none, User};
//-************************************************************************
// Error types for user creation
@ -94,7 +94,7 @@ pub async fn post_create_user(
State(pool): State<SqlitePool>,
Form(signup): Form<SignupForm>,
) -> Result<impl IntoResponse, CreateUserError> {
use crate::util::validate_optional_length;
use crate::misc_util::validate_optional_length;
let username = signup.username.trim();
let password = signup.password.trim();
let verify = signup.pw_verify.trim();

19
src/stars.rs Normal file
View file

@ -0,0 +1,19 @@
use julid::Julid;
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, FromRow)]
pub struct Star {
pub id: Julid,
pub name: String,
pub metadata_url: Option<String>,
pub born: Option<i64>,
pub died: Option<i64>,
}
#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, FromRow)]
pub struct Credit {
pub star: Julid,
pub watch: Julid,
pub credit: Option<String>,
}

View file

@ -10,7 +10,7 @@ use sqlx::{query, query_as, query_scalar, SqlitePool};
use super::templates::{AddNewWatchPage, AddWatchButton, GetWatchPage, SearchWatchesPage};
use crate::{
util::{empty_string_as_none, year_to_epoch},
misc_util::{empty_string_as_none, year_to_epoch},
AuthSession, MyWatchesPage, ShowKind, Watch, WatchQuest,
};