add migration to convert array to jsonb

This commit is contained in:
Namekuji 2023-06-01 04:01:25 -04:00
parent 745384ff58
commit 98f2397fbb
No known key found for this signature in database
GPG Key ID: B541BD6E646CABC7
7 changed files with 365 additions and 153 deletions

View File

@ -1,7 +1,11 @@
use model::entity::{antenna, newtype::StringVec}; use model::entity::{
access_token, antenna, app, emoji, gallery_post, hashtag, messaging_message, meta,
newtype::{I32Vec, StringVec},
note, note_edit, poll, registry_item, user, user_profile, webhook,
};
use sea_orm_migration::{ use sea_orm_migration::{
prelude::*, prelude::*,
sea_orm::{DbBackend, EntityTrait, Statement}, sea_orm::{DbBackend, EntityTrait, Statement, TryGetable},
}; };
use serde_json::json; use serde_json::json;
@ -11,50 +15,220 @@ pub struct Migration;
#[async_trait::async_trait] #[async_trait::async_trait]
impl MigrationTrait for Migration { impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.drop_table(Table::drop().table(Alias::new("reversi_game")).to_owned())
.await?;
manager
.drop_table(
Table::drop()
.table(Alias::new("reversi_matching"))
.to_owned(),
)
.await?;
if manager.get_database_backend() == DbBackend::Sqlite { if manager.get_database_backend() == DbBackend::Sqlite {
return Ok(()); return Ok(());
} }
let db = manager.get_connection(); let db = manager.get_connection();
let query = Query::select()
.column(Antenna::Id)
.column(Antenna::Users)
.from(Antenna::Table)
.to_string(PostgresQueryBuilder);
let res: Vec<(String, Vec<String>)> = db
.query_all(Statement::from_string(DbBackend::Postgres, query))
.await?
.iter()
.filter_map(|r| r.try_get_many_by_index().ok())
.collect();
manager macro_rules! copy_data {
.alter_table( ($data:ident, $ent:ident, $col:tt) => {
Table::alter() let models: Vec<$ent::ActiveModel> = $data
.table(Antenna::Table) .iter()
.drop_column(Antenna::Users) .map(|(id, r)| $ent::ActiveModel {
.add_column( id: sea_orm::Set(id.to_owned()),
ColumnDef::new(Antenna::Users) $col: sea_orm::Set(StringVec::from(r.to_owned())),
.json_binary() ..Default::default()
.not_null() })
.default(json!([])), .collect();
) for model in models {
.to_owned(), $ent::Entity::update(model).exec(db).await?;
) }
.await?; };
}
let models: Vec<antenna::ActiveModel> = res macro_rules! convert_to_stringvec_json {
($table:expr, $id:expr, $col:expr, $ent:ident, $col_name:tt) => {
let query = select_query($table, $id, $col);
let res = get_vec::<Vec<String>>(db, query).await?;
convert_col(manager, $table, $col).await?;
copy_data!(res, $ent, $col_name);
};
}
convert_to_stringvec_json!(
AccessToken::Table,
AccessToken::Id,
AccessToken::Permission,
access_token,
permission
);
convert_to_stringvec_json!(Antenna::Table, Antenna::Id, Antenna::Users, antenna, users);
convert_to_stringvec_json!(App::Table, App::Id, App::Permission, app, permission);
convert_to_stringvec_json!(Emoji::Table, Emoji::Id, Emoji::Aliases, emoji, aliases);
convert_to_stringvec_json!(
GalleryPost::Table,
GalleryPost::Id,
GalleryPost::FileIds,
gallery_post,
file_ids
);
convert_to_stringvec_json!(
GalleryPost::Table,
GalleryPost::Id,
GalleryPost::Tags,
gallery_post,
tags
);
convert_to_stringvec_json!(
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedUserIds,
hashtag,
mentioned_user_ids
);
convert_to_stringvec_json!(
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedLocalUserIds,
hashtag,
mentioned_local_user_ids
);
convert_to_stringvec_json!(
Hashtag::Table,
Hashtag::Id,
Hashtag::MentionedRemoteUserIds,
hashtag,
mentioned_remote_user_ids
);
convert_to_stringvec_json!(
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedUserIds,
hashtag,
attached_user_ids
);
convert_to_stringvec_json!(
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedLocalUserIds,
hashtag,
attached_local_user_ids
);
convert_to_stringvec_json!(
Hashtag::Table,
Hashtag::Id,
Hashtag::AttachedRemoteUserIds,
hashtag,
attached_remote_user_ids
);
convert_to_stringvec_json!(
MessagingMessage::Table,
MessagingMessage::Id,
MessagingMessage::Reads,
messaging_message,
reads
);
convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::Langs, meta, langs);
convert_to_stringvec_json!(
Meta::Table,
Meta::Id,
Meta::BlockedHosts,
meta,
blocked_hosts
);
convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::HiddenTags, meta, hidden_tags);
convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::PinnedUsers, meta, pinned_users);
convert_to_stringvec_json!(Meta::Table, Meta::Id, Meta::PinnedPages, meta, pinned_pages);
convert_to_stringvec_json!(
Meta::Table,
Meta::Id,
Meta::RecommendedInstances,
meta,
recommended_instances
);
convert_to_stringvec_json!(
Meta::Table,
Meta::Id,
Meta::SilencedHosts,
meta,
silenced_hosts
);
convert_to_stringvec_json!(Note::Table, Note::Id, Note::FileIds, note, file_ids);
convert_to_stringvec_json!(
Note::Table,
Note::Id,
Note::AttachedFileTypes,
note,
attached_file_types
);
convert_to_stringvec_json!(
Note::Table,
Note::Id,
Note::VisibleUserIds,
note,
visible_user_ids
);
convert_to_stringvec_json!(Note::Table, Note::Id, Note::Mentions, note, mentions);
convert_to_stringvec_json!(Note::Table, Note::Id, Note::Emojis, note, emojis);
convert_to_stringvec_json!(Note::Table, Note::Id, Note::Tags, note, tags);
convert_to_stringvec_json!(
NoteEdit::Table,
NoteEdit::Id,
NoteEdit::FileIds,
note_edit,
file_ids
);
// Convert poll here because its primary key is not id, but note_id.
let query = select_query(Poll::Table, Poll::NoteId, Poll::Choices);
let res = get_vec::<Vec<String>>(db, query).await?;
convert_col(manager, Poll::Table, Poll::Choices).await?;
let poll_models: Vec<poll::ActiveModel> = res
.iter() .iter()
.map(|(id, users)| antenna::ActiveModel { .map(|(id, r)| poll::ActiveModel {
id: sea_orm::Set(id.to_owned()), note_id: sea_orm::Set(id.to_owned()),
users: sea_orm::Set(StringVec::from(users.to_owned())), choices: sea_orm::Set(StringVec::from(r.to_owned())),
..Default::default() ..Default::default()
}) })
.collect(); .collect();
for model in poll_models {
for model in models { poll::Entity::update(model).exec(db).await?;
antenna::Entity::update(model).exec(db).await?;
} }
let query = select_query(Poll::Table, Poll::NoteId, Poll::Votes);
let res = get_vec::<Vec<i32>>(db, query).await?;
convert_col(manager, Poll::Table, Poll::Votes).await?;
let poll_models: Vec<poll::ActiveModel> = res
.iter()
.map(|(id, r)| poll::ActiveModel {
note_id: sea_orm::Set(id.to_owned()),
votes: sea_orm::Set(I32Vec::from(r.to_owned())),
..Default::default()
})
.collect();
for model in poll_models {
poll::Entity::update(model).exec(db).await?;
}
convert_to_stringvec_json!(
RegistryItem::Table,
RegistryItem::Id,
RegistryItem::Scope,
registry_item,
scope
);
convert_to_stringvec_json!(User::Table, User::Id, User::Tags, user, tags);
convert_to_stringvec_json!(User::Table, User::Id, User::Emojis, user, emojis);
convert_to_stringvec_json!(
UserProfile::Table,
UserProfile::Id,
UserProfile::MutingNotificationTypes,
user_profile,
muting_notification_types
);
convert_to_stringvec_json!(Webhook::Table, Webhook::Id, Webhook::On, webhook, on);
Ok(()) Ok(())
} }
@ -66,9 +240,160 @@ impl MigrationTrait for Migration {
} }
/// Learn more at https://docs.rs/sea-query#iden /// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)] #[derive(Iden, Clone)]
enum Antenna { enum Antenna {
Table, Table,
Id, Id,
Users, Users,
} }
#[derive(Iden, Clone)]
enum AccessToken {
Table,
Id,
Permission,
}
#[derive(Iden, Clone)]
enum App {
Table,
Id,
Permission,
}
#[derive(Iden, Clone)]
enum Emoji {
Table,
Id,
Aliases,
}
#[derive(Iden, Clone)]
enum GalleryPost {
Table,
Id,
FileIds,
Tags,
}
#[derive(Iden, Clone)]
enum Hashtag {
Table,
Id,
MentionedUserIds,
MentionedLocalUserIds,
MentionedRemoteUserIds,
AttachedUserIds,
AttachedLocalUserIds,
AttachedRemoteUserIds,
}
#[derive(Iden, Clone)]
enum MessagingMessage {
Table,
Id,
Reads,
}
#[derive(Iden, Clone)]
enum Meta {
Table,
Id,
Langs,
HiddenTags,
BlockedHosts,
PinnedUsers,
PinnedPages,
RecommendedInstances,
SilencedHosts,
}
#[derive(Iden, Clone)]
enum Note {
Table,
Id,
FileIds,
AttachedFileTypes,
VisibleUserIds,
Mentions,
Emojis,
Tags,
}
#[derive(Iden, Clone)]
enum NoteEdit {
Table,
Id,
FileIds,
}
#[derive(Iden, Clone)]
enum Page {
Table,
Id,
VisibleUserIds,
}
#[derive(Iden, Clone)]
enum Poll {
Table,
NoteId,
Choices,
Votes, // I32Vec
}
#[derive(Iden, Clone)]
enum RegistryItem {
Table,
Id,
Scope,
}
#[derive(Iden, Clone)]
enum User {
Table,
Id,
Tags,
Emojis,
}
#[derive(Iden, Clone)]
enum UserProfile {
Table,
Id,
MutingNotificationTypes,
}
#[derive(Iden, Clone)]
enum Webhook {
Table,
Id,
On,
}
fn select_query<T: Iden + 'static>(table: T, id: T, col: T) -> String {
Query::select()
.column(id)
.column(col)
.from(table)
.to_string(PostgresQueryBuilder)
}
async fn get_vec<'a, T: TryGetable>(
db: &SchemaManagerConnection<'a>,
query: String,
) -> Result<Vec<(String, T)>, DbErr> {
let res: Vec<(String, T)> = db
.query_all(Statement::from_string(DbBackend::Postgres, query))
.await?
.iter()
.filter_map(|r| r.try_get_many_by_index().ok())
.collect();
Ok(res)
}
async fn convert_col<'a, T: Iden + Clone + 'static>(
manager: &SchemaManager<'a>,
table: T,
col: T,
) -> Result<(), DbErr> {
manager
.alter_table(
Table::alter()
.table(table)
.drop_column(col.to_owned())
.add_column(
ColumnDef::new(col.to_owned())
.json_binary()
.not_null()
.default(json!([])),
)
.to_owned(),
)
.await
}

View File

@ -5,8 +5,8 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features] [features]
default = ["legacy"] default = []
legacy = ["sea-orm/postgres-array"] legacy = []
[dependencies] [dependencies]
async-trait = "0.1.68" async-trait = "0.1.68"
@ -18,7 +18,7 @@ jsonschema = "0.17.0"
once_cell = "1.17.1" once_cell = "1.17.1"
parse-display = "0.8.0" parse-display = "0.8.0"
schemars = { version = "0.8.12", features = ["chrono"] } schemars = { version = "0.8.12", features = ["chrono"] }
sea-orm = { version = "0.11.3", features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls"] } sea-orm = { version = "0.11.3", features = ["sqlx-postgres", "postgres-array", "sqlx-sqlite", "runtime-tokio-rustls"] }
serde = { version = "1.0.163", features = ["derive"] } serde = { version = "1.0.163", features = ["derive"] }
serde_json = "1.0.96" serde_json = "1.0.96"
thiserror = "1.0.40" thiserror = "1.0.40"

View File

@ -53,8 +53,6 @@ pub mod registration_ticket;
pub mod registry_item; pub mod registry_item;
pub mod relay; pub mod relay;
pub mod renote_muting; pub mod renote_muting;
pub mod reversi_game;
pub mod reversi_matching;
pub mod sea_orm_active_enums; pub mod sea_orm_active_enums;
pub mod signin; pub mod signin;
pub mod sw_subscription; pub mod sw_subscription;

View File

@ -50,8 +50,6 @@ pub use super::registration_ticket::Entity as RegistrationTicket;
pub use super::registry_item::Entity as RegistryItem; pub use super::registry_item::Entity as RegistryItem;
pub use super::relay::Entity as Relay; pub use super::relay::Entity as Relay;
pub use super::renote_muting::Entity as RenoteMuting; pub use super::renote_muting::Entity as RenoteMuting;
pub use super::reversi_game::Entity as ReversiGame;
pub use super::reversi_matching::Entity as ReversiMatching;
pub use super::signin::Entity as Signin; pub use super::signin::Entity as Signin;
pub use super::sw_subscription::Entity as SwSubscription; pub use super::sw_subscription::Entity as SwSubscription;
pub use super::used_username::Entity as UsedUsername; pub use super::used_username::Entity as UsedUsername;

View File

@ -1,69 +0,0 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
use sea_orm::entity::prelude::*;
use super::newtype::StringVec;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "reversi_game")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: String,
#[sea_orm(column_name = "createdAt")]
pub created_at: DateTimeWithTimeZone,
#[sea_orm(column_name = "startedAt")]
pub started_at: Option<DateTimeWithTimeZone>,
#[sea_orm(column_name = "user1Id")]
pub user1_id: String,
#[sea_orm(column_name = "user2Id")]
pub user2_id: String,
#[sea_orm(column_name = "user1Accepted")]
pub user1_accepted: bool,
#[sea_orm(column_name = "user2Accepted")]
pub user2_accepted: bool,
pub black: Option<i32>,
#[sea_orm(column_name = "isStarted")]
pub is_started: bool,
#[sea_orm(column_name = "isEnded")]
pub is_ended: bool,
#[sea_orm(column_name = "winnerId")]
pub winner_id: Option<String>,
pub surrendered: Option<String>,
#[sea_orm(column_type = "JsonBinary")]
pub logs: Json,
pub map: StringVec,
pub bw: String,
#[sea_orm(column_name = "isLlotheo")]
pub is_llotheo: bool,
#[sea_orm(column_name = "canPutEverywhere")]
pub can_put_everywhere: bool,
#[sea_orm(column_name = "loopedBoard")]
pub looped_board: bool,
#[sea_orm(column_type = "JsonBinary", nullable)]
pub form1: Option<Json>,
#[sea_orm(column_type = "JsonBinary", nullable)]
pub form2: Option<Json>,
pub crc32: Option<String>,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(
belongs_to = "super::user::Entity",
from = "Column::User2Id",
to = "super::user::Column::Id",
on_update = "NoAction",
on_delete = "Cascade"
)]
User2,
#[sea_orm(
belongs_to = "super::user::Entity",
from = "Column::User1Id",
to = "super::user::Column::Id",
on_update = "NoAction",
on_delete = "Cascade"
)]
User1,
}
impl ActiveModelBehavior for ActiveModel {}

View File

@ -1,38 +0,0 @@
//! `SeaORM` Entity. Generated by sea-orm-codegen 0.11.3
use sea_orm::entity::prelude::*;
#[derive(Clone, Debug, PartialEq, DeriveEntityModel, Eq)]
#[sea_orm(table_name = "reversi_matching")]
pub struct Model {
#[sea_orm(primary_key, auto_increment = false)]
pub id: String,
#[sea_orm(column_name = "createdAt")]
pub created_at: DateTimeWithTimeZone,
#[sea_orm(column_name = "parentId")]
pub parent_id: String,
#[sea_orm(column_name = "childId")]
pub child_id: String,
}
#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)]
pub enum Relation {
#[sea_orm(
belongs_to = "super::user::Entity",
from = "Column::ParentId",
to = "super::user::Column::Id",
on_update = "NoAction",
on_delete = "Cascade"
)]
User2,
#[sea_orm(
belongs_to = "super::user::Entity",
from = "Column::ChildId",
to = "super::user::Column::Id",
on_update = "NoAction",
on_delete = "Cascade"
)]
User1,
}
impl ActiveModelBehavior for ActiveModel {}

View File

@ -31,7 +31,7 @@ async fn setup_schema(db: DbConn) {
let stmt = schema.create_table_from_entity(antenna::Entity); let stmt = schema.create_table_from_entity(antenna::Entity);
db.execute(db.get_database_backend().build(&stmt)) db.execute(db.get_database_backend().build(&stmt))
.await .await
.expect("Unable to initialize in-memoty sqlite"); .expect("Unable to setup schemas for in-memoty sqlite");
} }
/// Delete all entries in the database. /// Delete all entries in the database.
@ -101,9 +101,7 @@ async fn setup_model(db: &DbConn) {
mod int_test { mod int_test {
use sea_orm::Database; use sea_orm::Database;
use crate::setup_schema; use super::{cleanup, prepare, setup_schema};
use super::{cleanup, prepare};
#[tokio::test] #[tokio::test]
async fn can_prepare_and_cleanup() { async fn can_prepare_and_cleanup() {