make import more robust and with fewer duplicates for stars.

src/imdb_utils.rs

@@ -82,7 +82,7 @@ pub async fn import_imdb_data(w2w_db: &SqlitePool, imdb: &SqlitePool, ids: &mut
         .await
         .unwrap();
 
-    for batch in iwatches.chunks(2_000) {
+    for batch in iwatches.chunks(5_000) {
         let mut tx = w2w_db.acquire().await.unwrap();
         let mut tx = tx.begin().await.unwrap();
         for iwatch in batch {
@@ -114,24 +114,32 @@ async fn add_imdb_stars(
 
     for row in principals {
         let (name_id, cat) = row;
-        let name_query =
+        let star = if let Some(id) = ids.get(&name_id) {
-            "select nconst, primaryName, birthYear, deathYear from names where nconst = ?";
+            *id
-        let istar: Option<ImdbStar> = sqlx::query_as(name_query)
+        } else {
-            .bind(&name_id)
+            let name_query =
-            .fetch_optional(imdb)
+                "select nconst, primaryName, birthYear, deathYear from names where nconst = ?";
-            .await
+            let istar: Option<ImdbStar> = sqlx::query_as(name_query)
-            .unwrap();
+                .bind(&name_id)
-        if let Some(star) = istar {
+                .fetch_optional(imdb)
-            let star = (&star).into();
+                .await
-            let star_id = insert_star(&star, w2w_db).await;
+                .unwrap();
-            ids.insert(name_id, star_id);
+            if let Some(star) = istar {
-            let credit = Credit {
+                let star = (&star).into();
-                star: star_id,
+                let star_id = insert_star(&star, w2w_db).await;
-                watch,
+                ids.insert(name_id, star_id);
-                credit: Some(cat.to_string()),
+                star_id
-            };
+            } else {
-            insert_credit(&credit, w2w_db).await;
+                continue;
-        }
+            }
+        };
+
+        let credit = Credit {
+            star,
+            watch,
+            credit: Some(cat.to_string()),
+        };
+        insert_credit(&credit, w2w_db).await;
     }
 }
 

src/import_utils.rs

@@ -47,6 +47,19 @@ pub async fn insert_credit(credit: &Credit, db: &mut SqliteConnection) {
         .bind(credit.credit.as_deref())
         .execute(db)
         .await
+        .map(|_| ())
+        .or_else(|e| match e {
+            sqlx::Error::Database(ref db) => {
+                let exit = db.code().unwrap_or_default().parse().unwrap_or(0u32);
+                // https://www.sqlite.org/rescode.html codes for unique constraint violations:
+                if exit == 2067 || exit == 1555 {
+                    Ok(())
+                } else {
+                    Err(e)
+                }
+            }
+            _ => Err(e),
+        })
         .unwrap();
 }