From def08b69b86f9c3f030374b9547d61972884e22e Mon Sep 17 00:00:00 2001
From: Joe Ardent <code@ardent.nebcorp.com>
Date: Tue, 9 Apr 2024 17:14:32 -0700
Subject: [PATCH] try to make imdb import more robust, add convenience views

---
 julid                                    |  2 +-
 migrations/20240409233522_views.down.sql |  2 ++
 migrations/20240409233522_views.up.sql   |  3 +++
 src/bin/import_imdb.rs                   | 11 +++++++++++
 src/db.rs                                |  2 +-
 src/imdb_utils.rs                        |  5 ++---
 6 files changed, 20 insertions(+), 5 deletions(-)
 create mode 100644 migrations/20240409233522_views.down.sql
 create mode 100644 migrations/20240409233522_views.up.sql

diff --git a/julid b/julid
index 705afc1..1e93d0b 160000
--- a/julid
+++ b/julid
@@ -1 +1 @@
-Subproject commit 705afc19e953133aadf811a0a51597e169f7aa62
+Subproject commit 1e93d0b1e4bc76ff19e1ce8e638c60204f458604
diff --git a/migrations/20240409233522_views.down.sql b/migrations/20240409233522_views.down.sql
new file mode 100644
index 0000000..e4302ca
--- /dev/null
+++ b/migrations/20240409233522_views.down.sql
@@ -0,0 +1,2 @@
+drop view if exists s;
+drop view if exists w;
diff --git a/migrations/20240409233522_views.up.sql b/migrations/20240409233522_views.up.sql
new file mode 100644
index 0000000..7700e81
--- /dev/null
+++ b/migrations/20240409233522_views.up.sql
@@ -0,0 +1,3 @@
+create view if not exists w as select julid_string(id) id, kind, title, metadata_url, length, release_date, last_updated from watches;
+
+create view if not exists s as select julid_string(id) id, name, born, died;
diff --git a/src/bin/import_imdb.rs b/src/bin/import_imdb.rs
index f37cedf..db78979 100644
--- a/src/bin/import_imdb.rs
+++ b/src/bin/import_imdb.rs
@@ -33,8 +33,18 @@ fn main() {
         .unwrap();
 
     let cli = Cli::parse();
+    let now = std::time::Instant::now();
     let ids = rt.block_on(import_watches(&w2w_db, &cli));
+    let dur = std::time::Instant::now() - now;
+    println!(
+        "Imported {} watches in {} seconds",
+        ids.len(),
+        dur.as_secs()
+    );
+    let now = std::time::Instant::now();
     rt.block_on(save_ids(&cli.db_path, ids));
+    let dur = std::time::Instant::now() - now;
+    println!("Saved the IDs in {} seconds", dur.as_secs());
 }
 
 async fn import_watches(w2w_db: &SqlitePool, cli: &Cli) -> IdMap {
@@ -64,6 +74,7 @@ async fn save_ids(path: &OsStr, ids: IdMap) {
     let file = path.file_name().unwrap();
     let file = file.to_str().unwrap();
     let path = format!("{}/w2w-{file}", path.parent().unwrap().to_str().unwrap());
+    println!("Writing IDs to {path}");
 
     let conn_opts = SqliteConnectOptions::new()
         .journal_mode(sqlx::sqlite::SqliteJournalMode::Wal)
diff --git a/src/db.rs b/src/db.rs
index 62afddd..198bdd3 100644
--- a/src/db.rs
+++ b/src/db.rs
@@ -8,7 +8,7 @@ use sqlx::{
 
 const MAX_CONNS: u32 = 200;
 const MIN_CONNS: u32 = 5;
-const TIMEOUT: u64 = 3;
+const TIMEOUT: u64 = 20;
 
 pub fn get_db_pool() -> SqlitePool {
     let db_filename = {
diff --git a/src/imdb_utils.rs b/src/imdb_utils.rs
index 99f4d87..9900053 100644
--- a/src/imdb_utils.rs
+++ b/src/imdb_utils.rs
@@ -1,5 +1,5 @@
 use julid::Julid;
-use sqlx::{Connection, Sqlite, SqlitePool};
+use sqlx::{Sqlite, SqlitePool};
 
 use crate::{
     import_utils::{insert_credit, insert_star, insert_watch},
@@ -82,8 +82,7 @@ pub async fn import_imdb_data(w2w_db: &SqlitePool, imdb: &SqlitePool, ids: &mut
         .unwrap();
 
     for batch in iwatches.chunks(5_000) {
-        let mut tx = w2w_db.acquire().await.unwrap();
-        let mut tx = tx.begin().await.unwrap();
+        let mut tx = w2w_db.begin().await.unwrap();
         for iwatch in batch {
             let aid = iwatch.id.clone();
             let kind = show_kind(iwatch.kind.as_ref().unwrap());