skip empty array

This commit is contained in:
Namekuji 2023-06-01 12:56:23 -04:00
parent d9d2673939
commit 193c75a2bb
No known key found for this signature in database
GPG Key ID: B541BD6E646CABC7
1 changed files with 278 additions and 57 deletions

View File

@ -1,11 +1,14 @@
#![allow(dead_code)] #![allow(dead_code)]
use sea_orm_migration::{prelude::*, sea_orm::{DbConn, DbBackend, Statement, TryGetable, Database}}; use indicatif::{MultiProgress, ProgressBar, ProgressStyle};
use serde_json::json;
use std::time::Duration;
use indicatif::{ProgressBar, ProgressStyle, MultiProgress};
use model::entity::newtype::{I32Vec, StringVec}; use model::entity::newtype::{I32Vec, StringVec};
use sea_orm_migration::{
prelude::*,
sea_orm::{Database, DbBackend, DbConn, Statement, TryGetable},
};
use serde_json::json;
use std::env; use std::env;
use std::time::Duration;
pub async fn convert() { pub async fn convert() {
let uri = env::var("DATABASE_URL").expect("Environment variable 'DATABASE_URL' not set"); let uri = env::var("DATABASE_URL").expect("Environment variable 'DATABASE_URL' not set");
@ -14,41 +17,251 @@ pub async fn convert() {
let mp = MultiProgress::new(); let mp = MultiProgress::new();
let handlers = vec![ let handlers = vec![
tokio::spawn(to_json::<AccessToken, Vec<String>, StringVec>(db.clone(), mp.clone(), AccessToken::Table, AccessToken::Id, AccessToken::Permission)), tokio::spawn(to_json::<AccessToken, Vec<String>, StringVec>(
tokio::spawn(to_json::<Antenna, Vec<String>, StringVec>(db.clone(), mp.clone(), Antenna::Table, Antenna::Id, Antenna::Users)), db.clone(),
tokio::spawn(to_json::<App, Vec<String>, StringVec>(db.clone(), mp.clone(), App::Table, App::Id, App::Permission)), mp.clone(),
tokio::spawn(to_json::<Emoji, Vec<String>, StringVec>(db.clone(), mp.clone(), Emoji::Table, Emoji::Id, Emoji::Aliases)), AccessToken::Table,
tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(db.clone(), mp.clone(), GalleryPost::Table, GalleryPost::Id, GalleryPost::FileIds)), AccessToken::Id,
tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(db.clone(), mp.clone(), GalleryPost::Table, GalleryPost::Id, GalleryPost::Tags)), AccessToken::Permission,
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedUserIds)), )),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedLocalUserIds)), tokio::spawn(to_json::<Antenna, Vec<String>, StringVec>(
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::MentionedRemoteUserIds)), db.clone(),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedUserIds)), mp.clone(),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedLocalUserIds)), Antenna::Table,
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(db.clone(), mp.clone(), Hashtag::Table, Hashtag::Id, Hashtag::AttachedRemoteUserIds)), Antenna::Id,
tokio::spawn(to_json::<MessagingMessage, Vec<String>, StringVec>(db.clone(), mp.clone(), MessagingMessage::Table, MessagingMessage::Id, MessagingMessage::Reads)), Antenna::Users,
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::Langs)), )),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::BlockedHosts)), tokio::spawn(to_json::<App, Vec<String>, StringVec>(
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::HiddenTags)), db.clone(),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::PinnedUsers)), mp.clone(),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::PinnedPages)), App::Table,
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::RecommendedInstances)), App::Id,
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(db.clone(), mp.clone(), Meta::Table, Meta::Id, Meta::SilencedHosts)), App::Permission,
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::FileIds)), )),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::AttachedFileTypes)), tokio::spawn(to_json::<Emoji, Vec<String>, StringVec>(
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::VisibleUserIds)), db.clone(),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Mentions)), mp.clone(),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Emojis)), Emoji::Table,
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(db.clone(), mp.clone(), Note::Table, Note::Id, Note::Tags)), Emoji::Id,
tokio::spawn(to_json::<NoteEdit, Vec<String>, StringVec>(db.clone(), mp.clone(), NoteEdit::Table, NoteEdit::Id, NoteEdit::FileIds)), Emoji::Aliases,
tokio::spawn(to_json::<Page, Vec<String>, StringVec>(db.clone(), mp.clone(), Page::Table, Page::Id, Page::VisibleUserIds)), )),
tokio::spawn(to_json::<RegistryItem, Vec<String>, StringVec>(db.clone(), mp.clone(), RegistryItem::Table, RegistryItem::Id, RegistryItem::Scope)), tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(
tokio::spawn(to_json::<User, Vec<String>, StringVec>(db.clone(), mp.clone(), User::Table, User::Id, User::Tags)), db.clone(),
tokio::spawn(to_json::<User, Vec<String>, StringVec>(db.clone(), mp.clone(), User::Table, User::Id, User::Emojis)), mp.clone(),
tokio::spawn(to_json::<Webhook, Vec<String>, StringVec>(db.clone(), mp.clone(), Webhook::Table, Webhook::Id, Webhook::On)), GalleryPost::Table,
tokio::spawn(to_json::<Poll, Vec<String>, StringVec>(db.clone(), mp.clone(), Poll::Table, Poll::NoteId, Poll::Choices)), GalleryPost::Id,
tokio::spawn(to_json::<Poll, Vec<i32>, I32Vec>(db.clone(), mp.clone(), Poll::Table, Poll::NoteId, Poll::Votes)), GalleryPost::FileIds,
tokio::spawn(to_json::<UserProfile, Vec<String>, StringVec>(db.clone(), mp.clone(), UserProfile::Table, UserProfile::UserId, UserProfile::MutingNotificationTypes)), )),
tokio::spawn(to_json::<GalleryPost, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
GalleryPost::Table,
GalleryPost::Id,
GalleryPost::Tags,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedLocalUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedRemoteUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedLocalUserIds,
)),
tokio::spawn(to_json::<Hashtag, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedRemoteUserIds,
)),
tokio::spawn(to_json::<MessagingMessage, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
MessagingMessage::Table,
MessagingMessage::Id,
MessagingMessage::Reads,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::Langs,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::BlockedHosts,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::HiddenTags,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::PinnedUsers,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::PinnedPages,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::RecommendedInstances,
)),
tokio::spawn(to_json::<Meta, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Meta::Table,
Meta::Id,
Meta::SilencedHosts,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::FileIds,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::AttachedFileTypes,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::VisibleUserIds,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::Mentions,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::Emojis,
)),
tokio::spawn(to_json::<Note, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Note::Table,
Note::Id,
Note::Tags,
)),
tokio::spawn(to_json::<NoteEdit, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
NoteEdit::Table,
NoteEdit::Id,
NoteEdit::FileIds,
)),
tokio::spawn(to_json::<Page, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Page::Table,
Page::Id,
Page::VisibleUserIds,
)),
tokio::spawn(to_json::<RegistryItem, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
RegistryItem::Table,
RegistryItem::Id,
RegistryItem::Scope,
)),
tokio::spawn(to_json::<User, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
User::Table,
User::Id,
User::Tags,
)),
tokio::spawn(to_json::<User, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
User::Table,
User::Id,
User::Emojis,
)),
tokio::spawn(to_json::<Webhook, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Webhook::Table,
Webhook::Id,
Webhook::On,
)),
tokio::spawn(to_json::<Poll, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
Poll::Table,
Poll::NoteId,
Poll::Choices,
)),
tokio::spawn(to_json::<Poll, Vec<i32>, I32Vec>(
db.clone(),
mp.clone(),
Poll::Table,
Poll::NoteId,
Poll::Votes,
)),
tokio::spawn(to_json::<UserProfile, Vec<String>, StringVec>(
db.clone(),
mp.clone(),
UserProfile::Table,
UserProfile::UserId,
UserProfile::MutingNotificationTypes,
)),
]; ];
futures::future::join_all(handlers).await; futures::future::join_all(handlers).await;
@ -62,10 +275,7 @@ fn select_query<T: Iden + 'static>(table: T, id: T, col: T) -> String {
.to_string(PostgresQueryBuilder) .to_string(PostgresQueryBuilder)
} }
async fn get_vec<T: TryGetable>( async fn get_vec<T: TryGetable>(db: &DbConn, query: String) -> Result<Vec<(String, T)>, DbErr> {
db: &DbConn,
query: String,
) -> Result<Vec<(String, T)>, DbErr> {
let res: Vec<(String, T)> = db let res: Vec<(String, T)> = db
.query_all(Statement::from_string(DbBackend::Postgres, query)) .query_all(Statement::from_string(DbBackend::Postgres, query))
.await? .await?
@ -88,8 +298,10 @@ async fn convert_col<T: Iden + Clone + 'static>(
.json_binary() .json_binary()
.not_null() .not_null()
.default(json!([])), .default(json!([])),
).to_string(PostgresQueryBuilder); )
db.query_one(Statement::from_string(DbBackend::Postgres, stmt)).await?; .to_string(PostgresQueryBuilder);
db.query_one(Statement::from_string(DbBackend::Postgres, stmt))
.await?;
Ok(()) Ok(())
} }
@ -102,26 +314,35 @@ async fn to_json<T, U, V>(
) -> Result<(), DbErr> ) -> Result<(), DbErr>
where where
T: Iden + Clone + 'static, T: Iden + Clone + 'static,
U: TryGetable + Clone, U: TryGetable + IntoIterator + Clone,
V: From<U> + Into<SimpleExpr>, V: From<U> + Into<SimpleExpr>,
{ {
let query = select_query(table.clone(), id.clone(), col.clone()); let query = select_query(table.clone(), id.clone(), col.clone());
let loading = ProgressBar::new_spinner() let loading = ProgressBar::new_spinner()
.with_style(ProgressStyle::with_template("{prefix} {msg} {spinner}").unwrap()) .with_style(ProgressStyle::with_template("{prefix} {msg} {spinner}").unwrap())
.with_prefix("[-]") .with_prefix("[-]")
.with_message(format!("Loading data from {}.{}", table.to_string(), col.to_string())); .with_message(format!(
loading.enable_steady_tick(Duration::from_millis(100)); "Loading data from {}.{}",
table.to_string(),
col.to_string()
));
let loading = mp.add(loading); let loading = mp.add(loading);
loading.enable_steady_tick(Duration::from_millis(100));
let res = get_vec::<U>(&db, query).await?; let res = get_vec::<U>(&db, query).await?;
let models: Vec<(String, V)> = res let models: Vec<(String, V)> = res
.iter() .iter()
.filter(|(_, r)| r.clone().into_iter().count() > 0)
.map(|(id, r)| (id.clone(), <V>::from(r.clone()))) .map(|(id, r)| (id.clone(), <V>::from(r.clone())))
.collect(); .collect();
loading.finish_and_clear(); loading.finish_and_clear();
convert_col(&db, table.clone(), col.clone()).await?; convert_col(&db, table.clone(), col.clone()).await?;
let progress = ProgressBar::new(models.len() as u64) let progress = ProgressBar::new(models.len() as u64)
.with_style(ProgressStyle::with_template("{prefix} {msg} {wide_bar} {pos}/{len}").unwrap().progress_chars("##-")) .with_style(
ProgressStyle::with_template("{prefix} {msg} {wide_bar} {pos}/{len}")
.unwrap()
.progress_chars("##-"),
)
.with_prefix("[*]") .with_prefix("[*]")
.with_message(format!("Copying {}.{}", table.to_string(), col.to_string())); .with_message(format!("Copying {}.{}", table.to_string(), col.to_string()));
let progress = mp.add(progress); let progress = mp.add(progress);
@ -136,7 +357,7 @@ where
db.query_one(Statement::from_string(DbBackend::Postgres, q)) db.query_one(Statement::from_string(DbBackend::Postgres, q))
.await?; .await?;
} }
progress.finish_with_message(format!("Complete {}.{}", table.to_string(), col.to_string())); progress.finish_with_message(format!("Done {}.{}", table.to_string(), col.to_string()));
Ok(()) Ok(())
} }