try to make imdb import more robust, add convenience views

julid

@@ -1 +1 @@
-Subproject commit 705afc19e953133aadf811a0a51597e169f7aa62
+Subproject commit 1e93d0b1e4bc76ff19e1ce8e638c60204f458604

migrations/20240409233522_views.down.sql

@@ -0,0 +1,2 @@
+drop view if exists s;
+drop view if exists w;

migrations/20240409233522_views.up.sql

@@ -0,0 +1,3 @@
+create view if not exists w as select julid_string(id) id, kind, title, metadata_url, length, release_date, last_updated from watches;
+
+create view if not exists s as select julid_string(id) id, name, born, died;

src/bin/import_imdb.rs

@@ -33,8 +33,18 @@ fn main() {
         .unwrap();
 
     let cli = Cli::parse();
+    let now = std::time::Instant::now();
     let ids = rt.block_on(import_watches(&w2w_db, &cli));
+    let dur = std::time::Instant::now() - now;
+    println!(
+        "Imported {} watches in {} seconds",
+        ids.len(),
+        dur.as_secs()
+    );
+    let now = std::time::Instant::now();
     rt.block_on(save_ids(&cli.db_path, ids));
+    let dur = std::time::Instant::now() - now;
+    println!("Saved the IDs in {} seconds", dur.as_secs());
 }
 
 async fn import_watches(w2w_db: &SqlitePool, cli: &Cli) -> IdMap {
@@ -64,6 +74,7 @@ async fn save_ids(path: &OsStr, ids: IdMap) {
     let file = path.file_name().unwrap();
     let file = file.to_str().unwrap();
     let path = format!("{}/w2w-{file}", path.parent().unwrap().to_str().unwrap());
+    println!("Writing IDs to {path}");
 
     let conn_opts = SqliteConnectOptions::new()
         .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)

src/db.rs

@@ -8,7 +8,7 @@ use sqlx::{
 
 const MAX_CONNS: u32 = 200;
 const MIN_CONNS: u32 = 5;
-const TIMEOUT: u64 = 3;
+const TIMEOUT: u64 = 20;
 
 pub fn get_db_pool() -> SqlitePool {
     let db_filename = {

src/imdb_utils.rs

@@ -1,5 +1,5 @@
 use julid::Julid;
-use sqlx::{Connection, Sqlite, SqlitePool};
+use sqlx::{Sqlite, SqlitePool};
 
 use crate::{
     import_utils::{insert_credit, insert_star, insert_watch},
@@ -82,8 +82,7 @@ pub async fn import_imdb_data(w2w_db: &SqlitePool, imdb: &SqlitePool, ids: &mut
         .unwrap();
 
     for batch in iwatches.chunks(5_000) {
-        let mut tx = w2w_db.acquire().await.unwrap();
+        let mut tx = w2w_db.begin().await.unwrap();
-        let mut tx = tx.begin().await.unwrap();
         for iwatch in batch {
             let aid = iwatch.id.clone();
             let kind = show_kind(iwatch.kind.as_ref().unwrap());