From 193c75a2bb7be7a69e6abb29f3ccb26207d79e7c Mon Sep 17 00:00:00 2001
From: Namekuji <nmkj@wahh.foo>
Date: Thu, 1 Jun 2023 12:56:23 -0400
Subject: [PATCH] skip empty array

---
 .../crates/migration/src/vec_to_json.rs       | 335 +++++++++++++++---
 1 file changed, 278 insertions(+), 57 deletions(-)

diff --git a/packages/backend/native-utils/crates/migration/src/vec_to_json.rs b/packages/backend/native-utils/crates/migration/src/vec_to_json.rs
index eccd801149..c762dcf1cc 100644
--- a/packages/backend/native-utils/crates/migration/src/vec_to_json.rs
+++ b/packages/backend/native-utils/crates/migration/src/vec_to_json.rs
@@ -1,11 +1,14 @@
 #![allow(dead_code)]
 
-use sea_orm_migration::{prelude::*, sea_orm::{DbConn, DbBackend, Statement, TryGetable, Database}};
-use serde_json::json;
-use std::time::Duration;
-use indicatif::{ProgressBar, ProgressStyle, MultiProgress};
+use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
 use model::entity::newtype::{I32Vec, StringVec};
+use sea_orm_migration::{
+    prelude::*,
+    sea_orm::{Database, DbBackend, DbConn, Statement, TryGetable},
+};
+use serde_json::json;
 use std::env;
+use std::time::Duration;
 
 pub async fn convert() {
     let uri = env::var("DATABASE_URL").expect("Environment variable 'DATABASE_URL' not set");
@@ -14,41 +17,251 @@ pub async fn convert() {
     let mp = MultiProgress::new();
 
     let handlers = vec![
-        tokio::spawn(to_json::<AccessToken, Vec<String>, StringVec>(db.clone(), mp.clone(), AccessToken::Table, AccessToken::Id, AccessToken::Permission)),
-        tokio::spawn(to_json::<Antenna, Vec<String>, StringVec>(db.clone(), mp.clone(), Antenna::Table, Antenna::Id, Antenna::Users)),
-        tokio::spawn(to_json::<App, Vec<String>, StringVec>(db.clone(), mp.clone(), App::Table, App::Id, App::Permission)),
-        tokio::spawn(to_json::<Emoji, Vec<String>, StringVec>(db.clone(), mp.clone(), Emoji::Table, Emoji::Id, Emoji::Aliases)),
-        tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(db.clone(), mp.clone(), GalleryPost::Table, GalleryPost::Id, GalleryPost::FileIds)),
-        tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(db.clone(), mp.clone(), GalleryPost::Table, GalleryPost::Id, GalleryPost::Tags)),
-        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedUserIds)),
-        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedLocalUserIds)),
-        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedRemoteUserIds)),
-        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedUserIds)),
-        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedLocalUserIds)),
-        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedRemoteUserIds)),
-        tokio::spawn(to_json::<MessagingMessage, Vec<String>, StringVec>(db.clone(), mp.clone(), MessagingMessage::Table, MessagingMessage::Id, MessagingMessage::Reads)),
-        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::Langs)),
-        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::BlockedHosts)),
-        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::HiddenTags)),
-        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::PinnedUsers)),
-        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::PinnedPages)),
-        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::RecommendedInstances)),
-        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::SilencedHosts)),
-        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::FileIds)),
-        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::AttachedFileTypes)),
-        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::VisibleUserIds)),
-        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Mentions)),
-        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Emojis)),
-        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Tags)),
-        tokio::spawn(to_json::<NoteEdit, Vec<String>, StringVec>(db.clone(), mp.clone(), NoteEdit::Table, NoteEdit::Id, NoteEdit::FileIds)),
-        tokio::spawn(to_json::<Page, Vec<String>, StringVec>(db.clone(), mp.clone(), Page::Table, Page::Id, Page::VisibleUserIds)),
-        tokio::spawn(to_json::<RegistryItem, Vec<String>, StringVec>(db.clone(), mp.clone(), RegistryItem::Table, RegistryItem::Id, RegistryItem::Scope)),
-        tokio::spawn(to_json::<User, Vec<String>, StringVec>(db.clone(), mp.clone(), User::Table, User::Id, User::Tags)),
-        tokio::spawn(to_json::<User, Vec<String>, StringVec>(db.clone(), mp.clone(), User::Table, User::Id, User::Emojis)),
-        tokio::spawn(to_json::<Webhook, Vec<String>, StringVec>(db.clone(), mp.clone(), Webhook::Table, Webhook::Id, Webhook::On)),
-        tokio::spawn(to_json::<Poll, Vec<String>, StringVec>(db.clone(), mp.clone(), Poll::Table, Poll::NoteId, Poll::Choices)),
-        tokio::spawn(to_json::<Poll, Vec<i32>, I32Vec>(db.clone(), mp.clone(), Poll::Table, Poll::NoteId, Poll::Votes)),
-        tokio::spawn(to_json::<UserProfile, Vec<String>, StringVec>(db.clone(), mp.clone(), UserProfile::Table, UserProfile::UserId, UserProfile::MutingNotificationTypes)),
+        tokio::spawn(to_json::<AccessToken, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            AccessToken::Table,
+            AccessToken::Id,
+            AccessToken::Permission,
+        )),
+        tokio::spawn(to_json::<Antenna, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Antenna::Table,
+            Antenna::Id,
+            Antenna::Users,
+        )),
+        tokio::spawn(to_json::<App, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            App::Table,
+            App::Id,
+            App::Permission,
+        )),
+        tokio::spawn(to_json::<Emoji, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Emoji::Table,
+            Emoji::Id,
+            Emoji::Aliases,
+        )),
+        tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            GalleryPost::Table,
+            GalleryPost::Id,
+            GalleryPost::FileIds,
+        )),
+        tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            GalleryPost::Table,
+            GalleryPost::Id,
+            GalleryPost::Tags,
+        )),
+        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::MentionedUserIds,
+        )),
+        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::MentionedLocalUserIds,
+        )),
+        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::MentionedRemoteUserIds,
+        )),
+        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::AttachedUserIds,
+        )),
+        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::AttachedLocalUserIds,
+        )),
+        tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Hashtag::Table,
+            Hashtag::Id,
+            Hashtag::AttachedRemoteUserIds,
+        )),
+        tokio::spawn(to_json::<MessagingMessage, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            MessagingMessage::Table,
+            MessagingMessage::Id,
+            MessagingMessage::Reads,
+        )),
+        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Meta::Table,
+            Meta::Id,
+            Meta::Langs,
+        )),
+        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Meta::Table,
+            Meta::Id,
+            Meta::BlockedHosts,
+        )),
+        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Meta::Table,
+            Meta::Id,
+            Meta::HiddenTags,
+        )),
+        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Meta::Table,
+            Meta::Id,
+            Meta::PinnedUsers,
+        )),
+        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Meta::Table,
+            Meta::Id,
+            Meta::PinnedPages,
+        )),
+        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Meta::Table,
+            Meta::Id,
+            Meta::RecommendedInstances,
+        )),
+        tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Meta::Table,
+            Meta::Id,
+            Meta::SilencedHosts,
+        )),
+        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Note::Table,
+            Note::Id,
+            Note::FileIds,
+        )),
+        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Note::Table,
+            Note::Id,
+            Note::AttachedFileTypes,
+        )),
+        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Note::Table,
+            Note::Id,
+            Note::VisibleUserIds,
+        )),
+        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Note::Table,
+            Note::Id,
+            Note::Mentions,
+        )),
+        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Note::Table,
+            Note::Id,
+            Note::Emojis,
+        )),
+        tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Note::Table,
+            Note::Id,
+            Note::Tags,
+        )),
+        tokio::spawn(to_json::<NoteEdit, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            NoteEdit::Table,
+            NoteEdit::Id,
+            NoteEdit::FileIds,
+        )),
+        tokio::spawn(to_json::<Page, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Page::Table,
+            Page::Id,
+            Page::VisibleUserIds,
+        )),
+        tokio::spawn(to_json::<RegistryItem, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            RegistryItem::Table,
+            RegistryItem::Id,
+            RegistryItem::Scope,
+        )),
+        tokio::spawn(to_json::<User, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            User::Table,
+            User::Id,
+            User::Tags,
+        )),
+        tokio::spawn(to_json::<User, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            User::Table,
+            User::Id,
+            User::Emojis,
+        )),
+        tokio::spawn(to_json::<Webhook, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Webhook::Table,
+            Webhook::Id,
+            Webhook::On,
+        )),
+        tokio::spawn(to_json::<Poll, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            Poll::Table,
+            Poll::NoteId,
+            Poll::Choices,
+        )),
+        tokio::spawn(to_json::<Poll, Vec<i32>, I32Vec>(
+            db.clone(),
+            mp.clone(),
+            Poll::Table,
+            Poll::NoteId,
+            Poll::Votes,
+        )),
+        tokio::spawn(to_json::<UserProfile, Vec<String>, StringVec>(
+            db.clone(),
+            mp.clone(),
+            UserProfile::Table,
+            UserProfile::UserId,
+            UserProfile::MutingNotificationTypes,
+        )),
     ];
 
     futures::future::join_all(handlers).await;
@@ -62,10 +275,7 @@ fn select_query<T: Iden + 'static>(table: T, id: T, col: T) -> String {
         .to_string(PostgresQueryBuilder)
 }
 
-async fn get_vec<T: TryGetable>(
-    db: &DbConn,
-    query: String,
-) -> Result<Vec<(String, T)>, DbErr> {
+async fn get_vec<T: TryGetable>(db: &DbConn, query: String) -> Result<Vec<(String, T)>, DbErr> {
     let res: Vec<(String, T)> = db
         .query_all(Statement::from_string(DbBackend::Postgres, query))
         .await?
@@ -81,15 +291,17 @@ async fn convert_col<T: Iden + Clone + 'static>(
     col: T,
 ) -> Result<(), DbErr> {
     let stmt = Table::alter()
-                .table(table)
-                .drop_column(col.to_owned())
-                .add_column(
-                    ColumnDef::new(col.to_owned())
-                        .json_binary()
-                        .not_null()
-                        .default(json!([])),
-                ).to_string(PostgresQueryBuilder);
-    db.query_one(Statement::from_string(DbBackend::Postgres, stmt)).await?;
+        .table(table)
+        .drop_column(col.to_owned())
+        .add_column(
+            ColumnDef::new(col.to_owned())
+                .json_binary()
+                .not_null()
+                .default(json!([])),
+        )
+        .to_string(PostgresQueryBuilder);
+    db.query_one(Statement::from_string(DbBackend::Postgres, stmt))
+        .await?;
     Ok(())
 }
 
@@ -102,26 +314,35 @@ async fn to_json<T, U, V>(
 ) -> Result<(), DbErr>
 where
     T: Iden + Clone + 'static,
-    U: TryGetable + Clone,
+    U: TryGetable + IntoIterator + Clone,
     V: From<U> + Into<SimpleExpr>,
 {
     let query = select_query(table.clone(), id.clone(), col.clone());
     let loading = ProgressBar::new_spinner()
         .with_style(ProgressStyle::with_template("{prefix} {msg} {spinner}").unwrap())
         .with_prefix("[-]")
-        .with_message(format!("Loading data from {}.{}", table.to_string(), col.to_string()));
-    loading.enable_steady_tick(Duration::from_millis(100));
+        .with_message(format!(
+            "Loading data from {}.{}",
+            table.to_string(),
+            col.to_string()
+        ));
     let loading = mp.add(loading);
+    loading.enable_steady_tick(Duration::from_millis(100));
     let res = get_vec::<U>(&db, query).await?;
     let models: Vec<(String, V)> = res
         .iter()
+        .filter(|(_, r)| r.clone().into_iter().count() > 0)
         .map(|(id, r)| (id.clone(), <V>::from(r.clone())))
         .collect();
     loading.finish_and_clear();
     convert_col(&db, table.clone(), col.clone()).await?;
 
     let progress = ProgressBar::new(models.len() as u64)
-        .with_style(ProgressStyle::with_template("{prefix} {msg} {wide_bar} {pos}/{len}").unwrap().progress_chars("##-"))
+        .with_style(
+            ProgressStyle::with_template("{prefix} {msg} {wide_bar} {pos}/{len}")
+                .unwrap()
+                .progress_chars("##-"),
+        )
         .with_prefix("[*]")
         .with_message(format!("Copying {}.{}", table.to_string(), col.to_string()));
     let progress = mp.add(progress);
@@ -136,7 +357,7 @@ where
         db.query_one(Statement::from_string(DbBackend::Postgres, q))
             .await?;
     }
-    progress.finish_with_message(format!("Complete {}.{}", table.to_string(), col.to_string()));
+    progress.finish_with_message(format!("Done {}.{}", table.to_string(), col.to_string()));
 
     Ok(())
 }