From 4343abfb7b1e424b20db6cbafa06b8ebfb0afb08 Mon Sep 17 00:00:00 2001
From: Joe Ardent <code@ardent.nebcorp.com>
Date: Sat, 27 Jan 2024 14:38:50 -0800
Subject: [PATCH] batch import inserts in a transaction.

---
 src/imdb_utils.rs   | 25 +++++++++++++++----------
 src/import_utils.rs |  8 ++++----
 2 files changed, 19 insertions(+), 14 deletions(-)

diff --git a/src/imdb_utils.rs b/src/imdb_utils.rs
index b124e87..2f135b7 100644
--- a/src/imdb_utils.rs
+++ b/src/imdb_utils.rs
@@ -1,5 +1,5 @@
 use julid::Julid;
-use sqlx::{Sqlite, SqlitePool};
+use sqlx::{Connection, Sqlite, SqlitePool};
 
 use crate::{
     import_utils::{insert_credit, insert_star, insert_watch},
@@ -82,19 +82,24 @@ pub async fn import_imdb_data(w2w_db: &SqlitePool, imdb: &SqlitePool, ids: &mut
         .await
         .unwrap();
 
-    for iwatch in iwatches {
-        let aid = iwatch.id.clone();
-        let kind = show_kind(iwatch.kind.as_ref().unwrap());
-        let mut watch: Watch = iwatch.into();
-        watch.kind = kind;
-        let watch_id: Julid = insert_watch(watch, w2w_db).await;
-        add_imdb_stars(w2w_db, imdb, &aid, watch_id, ids).await;
-        ids.insert(aid, watch_id);
+    for batch in iwatches.chunks(1_000) {
+        let mut tx = w2w_db.acquire().await.unwrap();
+        let mut tx = tx.begin().await.unwrap();
+        for iwatch in batch {
+            let aid = iwatch.id.clone();
+            let kind = show_kind(iwatch.kind.as_ref().unwrap());
+            let mut watch: Watch = iwatch.into();
+            watch.kind = kind;
+            let watch_id: Julid = insert_watch(watch, &mut tx).await;
+            add_imdb_stars(&mut tx, imdb, &aid, watch_id, ids).await;
+            ids.insert(aid, watch_id);
+        }
+        tx.commit().await.unwrap();
     }
 }
 
 async fn add_imdb_stars(
-    w2w_db: &SqlitePool,
+    w2w_db: &mut sqlx::Transaction<'_, Sqlite>,
     imdb: &SqlitePool,
     iwatch: &str,
     watch: Julid,
diff --git a/src/import_utils.rs b/src/import_utils.rs
index 603fa5f..c214631 100644
--- a/src/import_utils.rs
+++ b/src/import_utils.rs
@@ -1,5 +1,5 @@
 use julid::Julid;
-use sqlx::{query_scalar, SqlitePool};
+use sqlx::{query_scalar, SqliteConnection, SqlitePool};
 
 use crate::{Credit, Star, User, Watch};
 
@@ -12,7 +12,7 @@ const OMEGA_ID: Julid = Julid::omega();
 // utility functions for building CLI tools, currently just for benchmarking
 //-************************************************************************
 
-pub async fn insert_watch(watch: Watch, db: &SqlitePool) -> Julid {
+pub async fn insert_watch(watch: Watch, db: &mut SqliteConnection) -> Julid {
     let q = "insert into watches (kind, title, length, release_date, added_by, metadata_url) values (?, ?, ?, ?, ?, ?) returning id";
     sqlx::query_scalar(q)
         .bind(watch.kind)
@@ -26,7 +26,7 @@ pub async fn insert_watch(watch: Watch, db: &SqlitePool) -> Julid {
         .unwrap()
 }
 
-pub async fn insert_star(star: &Star, db: &SqlitePool) -> Julid {
+pub async fn insert_star(star: &Star, db: &mut SqliteConnection) -> Julid {
     let q = "insert into stars (name, metadata_url, born, died) values (?, ?, ?, ?) returning id";
     sqlx::query_scalar(q)
         .bind(&star.name)
@@ -38,7 +38,7 @@ pub async fn insert_star(star: &Star, db: &SqlitePool) -> Julid {
         .unwrap()
 }
 
-pub async fn insert_credit(credit: &Credit, db: &SqlitePool) {
+pub async fn insert_credit(credit: &Credit, db: &mut SqliteConnection) {
     let q = "insert into credits (star, watch, credit) values (?, ?, ?)";
 
     sqlx::query(q)