nebkor/what2watch
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

batch import inserts in a transaction.

Browse source
This commit is contained in:
Joe Ardent 2024-01-27 14:38:50 -08:00
parent d1e2e95248
commit 4343abfb7b
2 changed files with 19 additions and 14 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
src/imdb_utils.rs
View file

@ -1,5 +1,5 @@
use julid::Julid; use julid::Julid;
use sqlx::{Sqlite, SqlitePool}; use sqlx::{Connection, Sqlite, SqlitePool};
use crate::{ use crate::{
import_utils::{insert_credit, insert_star, insert_watch}, import_utils::{insert_credit, insert_star, insert_watch},
@ -82,19 +82,24 @@ pub async fn import_imdb_data(w2w_db: &SqlitePool, imdb: &SqlitePool, ids: &mut
.await .await
.unwrap(); .unwrap();
for iwatch in iwatches { for batch in iwatches.chunks(1_000) {
let mut tx = w2w_db.acquire().await.unwrap();
let mut tx = tx.begin().await.unwrap();
for iwatch in batch {
let aid = iwatch.id.clone(); let aid = iwatch.id.clone();
let kind = show_kind(iwatch.kind.as_ref().unwrap()); let kind = show_kind(iwatch.kind.as_ref().unwrap());
let mut watch: Watch = iwatch.into(); let mut watch: Watch = iwatch.into();
watch.kind = kind; watch.kind = kind;
let watch_id: Julid = insert_watch(watch, w2w_db).await; let watch_id: Julid = insert_watch(watch, &mut tx).await;
add_imdb_stars(w2w_db, imdb, &aid, watch_id, ids).await; add_imdb_stars(&mut tx, imdb, &aid, watch_id, ids).await;
ids.insert(aid, watch_id); ids.insert(aid, watch_id);
} }
tx.commit().await.unwrap();
}
} }
async fn add_imdb_stars( async fn add_imdb_stars(
w2w_db: &SqlitePool, w2w_db: &mut sqlx::Transaction<'_, Sqlite>,
imdb: &SqlitePool, imdb: &SqlitePool,
iwatch: &str, iwatch: &str,
watch: Julid, watch: Julid,

8
src/import_utils.rs
View file

@ -1,5 +1,5 @@
use julid::Julid; use julid::Julid;
use sqlx::{query_scalar, SqlitePool}; use sqlx::{query_scalar, SqliteConnection, SqlitePool};
use crate::{Credit, Star, User, Watch}; use crate::{Credit, Star, User, Watch};
@ -12,7 +12,7 @@ const OMEGA_ID: Julid = Julid::omega();
// utility functions for building CLI tools, currently just for benchmarking // utility functions for building CLI tools, currently just for benchmarking
//-************************************************************************ //-************************************************************************
pub async fn insert_watch(watch: Watch, db: &SqlitePool) -> Julid { pub async fn insert_watch(watch: Watch, db: &mut SqliteConnection) -> Julid {
let q = "insert into watches (kind, title, length, release_date, added_by, metadata_url) values (?, ?, ?, ?, ?, ?) returning id"; let q = "insert into watches (kind, title, length, release_date, added_by, metadata_url) values (?, ?, ?, ?, ?, ?) returning id";
sqlx::query_scalar(q) sqlx::query_scalar(q)
.bind(watch.kind) .bind(watch.kind)
@ -26,7 +26,7 @@ pub async fn insert_watch(watch: Watch, db: &SqlitePool) -> Julid {
.unwrap() .unwrap()
} }
pub async fn insert_star(star: &Star, db: &SqlitePool) -> Julid { pub async fn insert_star(star: &Star, db: &mut SqliteConnection) -> Julid {
let q = "insert into stars (name, metadata_url, born, died) values (?, ?, ?, ?) returning id"; let q = "insert into stars (name, metadata_url, born, died) values (?, ?, ?, ?) returning id";
sqlx::query_scalar(q) sqlx::query_scalar(q)
.bind(&star.name) .bind(&star.name)
@ -38,7 +38,7 @@ pub async fn insert_star(star: &Star, db: &SqlitePool) -> Julid {
.unwrap() .unwrap()
} }
pub async fn insert_credit(credit: &Credit, db: &SqlitePool) { pub async fn insert_credit(credit: &Credit, db: &mut SqliteConnection) {
let q = "insert into credits (star, watch, credit) values (?, ?, ?)"; let q = "insert into credits (star, watch, credit) values (?, ?, ?)";
sqlx::query(q) sqlx::query(q)