diff --git a/packages/backend/native-utils/crates/database/src/lib.rs b/packages/backend/native-utils/crates/database/src/lib.rs index 6907a12816..0012d2292f 100644 --- a/packages/backend/native-utils/crates/database/src/lib.rs +++ b/packages/backend/native-utils/crates/database/src/lib.rs @@ -1,10 +1,10 @@ pub mod error; -use sea_orm::{Database, DatabaseConnection}; +use sea_orm::{Database, DbConn}; use crate::error::Error; -static DB_CONN: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); +static DB_CONN: once_cell::sync::OnceCell = once_cell::sync::OnceCell::new(); pub async fn init_database(connection_uri: impl Into) -> Result<(), Error> { let conn = Database::connect(connection_uri.into()).await?; @@ -12,7 +12,7 @@ pub async fn init_database(connection_uri: impl Into) -> Result<(), Erro Ok(()) } -pub fn get_database() -> Result<&'static DatabaseConnection, Error> { +pub fn get_database() -> Result<&'static DbConn, Error> { DB_CONN.get().ok_or(Error::Uninitialized) } diff --git a/packages/backend/native-utils/crates/migration/Cargo.toml b/packages/backend/native-utils/crates/migration/Cargo.toml new file mode 100644 index 0000000000..46108c68c5 --- /dev/null +++ b/packages/backend/native-utils/crates/migration/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "migration" +version = "0.1.0" +edition = "2021" +publish = false + +[lib] +name = "migration" +path = "src/lib.rs" + +[dependencies] +async-std = { version = "1", features = ["attributes", "tokio1"] } +serde_json = "1.0.96" +model = { path = "../model" } + +[dependencies.sea-orm-migration] +version = "0.11.0" +features = [ + # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI. + # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime. + # e.g. + "runtime-tokio-rustls", # `ASYNC_RUNTIME` feature + "sqlx-postgres", # `DATABASE_DRIVER` feature + "sqlx-sqlite", +] diff --git a/packages/backend/native-utils/crates/migration/README.md b/packages/backend/native-utils/crates/migration/README.md new file mode 100644 index 0000000000..b3ea53eb44 --- /dev/null +++ b/packages/backend/native-utils/crates/migration/README.md @@ -0,0 +1,41 @@ +# Running Migrator CLI + +- Generate a new migration file + ```sh + cargo run -- migrate generate MIGRATION_NAME + ``` +- Apply all pending migrations + ```sh + cargo run + ``` + ```sh + cargo run -- up + ``` +- Apply first 10 pending migrations + ```sh + cargo run -- up -n 10 + ``` +- Rollback last applied migrations + ```sh + cargo run -- down + ``` +- Rollback last 10 applied migrations + ```sh + cargo run -- down -n 10 + ``` +- Drop all tables from the database, then reapply all migrations + ```sh + cargo run -- fresh + ``` +- Rollback all applied migrations, then reapply all migrations + ```sh + cargo run -- refresh + ``` +- Rollback all applied migrations + ```sh + cargo run -- reset + ``` +- Check the status of all migrations + ```sh + cargo run -- status + ``` diff --git a/packages/backend/native-utils/crates/migration/src/lib.rs b/packages/backend/native-utils/crates/migration/src/lib.rs new file mode 100644 index 0000000000..a2a1b932f2 --- /dev/null +++ b/packages/backend/native-utils/crates/migration/src/lib.rs @@ -0,0 +1,12 @@ +pub use sea_orm_migration::prelude::*; + +mod m20230531_180824_stringvec; + +pub struct Migrator; + +#[async_trait::async_trait] +impl MigratorTrait for Migrator { + fn migrations() -> Vec> { + vec![Box::new(m20230531_180824_stringvec::Migration)] + } +} diff --git a/packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs b/packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs new file mode 100644 index 0000000000..ba1fc257d0 --- /dev/null +++ b/packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs @@ -0,0 +1,74 @@ +use model::entity::{antenna, newtype::StringVec}; +use sea_orm_migration::{ + prelude::*, + sea_orm::{DbBackend, EntityTrait, Statement}, +}; +use serde_json::json; + +#[derive(DeriveMigrationName)] +pub struct Migration; + +#[async_trait::async_trait] +impl MigrationTrait for Migration { + async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> { + if manager.get_database_backend() == DbBackend::Sqlite { + return Ok(()); + } + + let db = manager.get_connection(); + let query = Query::select() + .column(Antenna::Id) + .column(Antenna::Users) + .from(Antenna::Table) + .to_string(PostgresQueryBuilder); + let res: Vec<(String, Vec)> = db + .query_all(Statement::from_string(DbBackend::Postgres, query)) + .await? + .iter() + .filter_map(|r| r.try_get_many_by_index().ok()) + .collect(); + + manager + .alter_table( + Table::alter() + .table(Antenna::Table) + .drop_column(Antenna::Users) + .add_column( + ColumnDef::new(Antenna::Users) + .json_binary() + .not_null() + .default(json!([])), + ) + .to_owned(), + ) + .await?; + + let models: Vec = res + .iter() + .map(|(id, users)| antenna::ActiveModel { + id: sea_orm::Set(id.to_owned()), + users: sea_orm::Set(StringVec::from(users.to_owned())), + ..Default::default() + }) + .collect(); + + for model in models { + antenna::Entity::update(model).exec(db).await?; + } + + Ok(()) + } + + async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> { + // Replace the sample below with your own migration scripts + Ok(()) + } +} + +/// Learn more at https://docs.rs/sea-query#iden +#[derive(Iden)] +enum Antenna { + Table, + Id, + Users, +} diff --git a/packages/backend/native-utils/crates/migration/src/main.rs b/packages/backend/native-utils/crates/migration/src/main.rs new file mode 100644 index 0000000000..c6b6e48dbc --- /dev/null +++ b/packages/backend/native-utils/crates/migration/src/main.rs @@ -0,0 +1,6 @@ +use sea_orm_migration::prelude::*; + +#[async_std::main] +async fn main() { + cli::run_cli(migration::Migrator).await; +} diff --git a/packages/backend/native-utils/crates/model/Cargo.toml b/packages/backend/native-utils/crates/model/Cargo.toml index 47d2e15537..4f121a1e11 100644 --- a/packages/backend/native-utils/crates/model/Cargo.toml +++ b/packages/backend/native-utils/crates/model/Cargo.toml @@ -4,9 +4,13 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[features] +default = ["compat"] +compat = ["sea-orm/postgres-array"] [dependencies] async-trait = "0.1.68" +cfg-if = "1.0.0" chrono = "0.4.24" database = { path = "../database" } derive_more = "0.99.17" @@ -14,7 +18,7 @@ jsonschema = "0.17.0" once_cell = "1.17.1" parse-display = "0.8.0" schemars = { version = "0.8.12", features = ["chrono"] } -sea-orm = { version = "0.11.3", features = ["postgres-array", "sqlx-postgres", "runtime-tokio-rustls"] } +sea-orm = { version = "0.11.3", features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls"] } serde = { version = "1.0.163", features = ["derive"] } serde_json = "1.0.96" thiserror = "1.0.40" diff --git a/packages/backend/native-utils/crates/model/src/entity/antenna.rs b/packages/backend/native-utils/crates/model/src/entity/antenna.rs index ccff0aab4f..0ebd1cf45b 100644 --- a/packages/backend/native-utils/crates/model/src/entity/antenna.rs +++ b/packages/backend/native-utils/crates/model/src/entity/antenna.rs @@ -28,7 +28,7 @@ pub struct Model { pub with_replies: bool, #[sea_orm(column_name = "userGroupJoiningId")] pub user_group_joining_id: Option, - pub users: Vec, + pub users: newtype::StringVec, #[sea_orm(column_name = "excludeKeywords", column_type = "JsonBinary")] pub exclude_keywords: newtype::JsonKeyword, #[sea_orm(column_type = "JsonBinary")] diff --git a/packages/backend/native-utils/crates/model/src/entity/newtype/macros.rs b/packages/backend/native-utils/crates/model/src/entity/newtype/macros.rs index 0266e80c85..4b05c2c99c 100644 --- a/packages/backend/native-utils/crates/model/src/entity/newtype/macros.rs +++ b/packages/backend/native-utils/crates/model/src/entity/newtype/macros.rs @@ -33,12 +33,12 @@ macro_rules! impl_json_newtype { stringify!($a).to_owned() } - fn array_type() -> sea_orm::sea_query::ArrayType { - sea_orm::sea_query::ArrayType::Json + fn array_type() -> sea_query::ArrayType { + sea_query::ArrayType::Json } fn column_type() -> sea_query::ColumnType { - sea_query::ColumnType::Json + sea_query::ColumnType::JsonBinary } } diff --git a/packages/backend/native-utils/crates/model/src/entity/newtype/mod.rs b/packages/backend/native-utils/crates/model/src/entity/newtype/mod.rs index f7e3c3d436..2b11c5f21a 100644 --- a/packages/backend/native-utils/crates/model/src/entity/newtype/mod.rs +++ b/packages/backend/native-utils/crates/model/src/entity/newtype/mod.rs @@ -1,17 +1,24 @@ mod macros; -use derive_more::From; -use schemars::JsonSchema; +use cfg_if::cfg_if; +use derive_more::{From, Into}; use sea_orm::{sea_query, DbErr, QueryResult, TryGetError, TryGetable, Value}; use serde::{Deserialize, Serialize}; use crate::impl_json_newtype; -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema, From)] +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, From, Into)] pub struct JsonKeyword(pub Vec>); impl_json_newtype!(JsonKeyword); -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema, From)] - +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, From, Into)] pub struct JsonStringVec(pub Vec); impl_json_newtype!(JsonStringVec); + +cfg_if! { + if #[cfg(feature = "compat")] { + pub type StringVec = Vec; + } else { + pub type StringVec = JsonStringVec; + } +} diff --git a/packages/backend/native-utils/crates/model/src/error.rs b/packages/backend/native-utils/crates/model/src/error.rs index 379ecb7842..7686312ec4 100644 --- a/packages/backend/native-utils/crates/model/src/error.rs +++ b/packages/backend/native-utils/crates/model/src/error.rs @@ -3,7 +3,7 @@ pub enum Error { #[error("Failed to parse string")] ParseError(#[from] parse_display::ParseError), #[error("Failed to get database connection")] - DatabaseConnectionError(#[from] database::error::Error), + DbConnError(#[from] database::error::Error), #[error("Database operation error: {0}")] DatabaseOperationError(#[from] sea_orm::DbErr), } diff --git a/packages/backend/native-utils/crates/model/src/repository/antenna.rs b/packages/backend/native-utils/crates/model/src/repository/antenna.rs index 9a6882b59a..f9ef012a6e 100644 --- a/packages/backend/native-utils/crates/model/src/repository/antenna.rs +++ b/packages/backend/native-utils/crates/model/src/repository/antenna.rs @@ -30,13 +30,13 @@ impl Repository for antenna::Model { id: self.id, created_at: self.created_at.into(), name: self.name, - keywords: self.keywords, - exclude_keywords: self.exclude_keywords, + keywords: self.keywords.into(), + exclude_keywords: self.exclude_keywords.into(), src: self.src.try_into()?, user_list_id: self.user_list_id, user_group_id, - users: self.users, - instances: self.instances, + users: self.users.into(), + instances: self.instances.into(), case_sensitive: self.case_sensitive, notify: self.notify, with_replies: self.with_replies, diff --git a/packages/backend/native-utils/crates/model/src/schema/antenna.rs b/packages/backend/native-utils/crates/model/src/schema/antenna.rs index 4a71982f7f..fa7902b924 100644 --- a/packages/backend/native-utils/crates/model/src/schema/antenna.rs +++ b/packages/backend/native-utils/crates/model/src/schema/antenna.rs @@ -5,7 +5,7 @@ use schemars::JsonSchema; use utoipa::ToSchema; use super::Schema; -use crate::entity::{newtype, sea_orm_active_enums::AntennaSrcEnum}; +use crate::entity::sea_orm_active_enums::AntennaSrcEnum; #[derive(Clone, Debug, PartialEq, Eq, JsonSchema, ToSchema)] #[serde(rename_all = "camelCase")] @@ -13,14 +13,14 @@ pub struct Antenna { pub id: String, pub created_at: chrono::DateTime, pub name: String, - pub keywords: newtype::JsonKeyword, - pub exclude_keywords: newtype::JsonKeyword, + pub keywords: Vec>, + pub exclude_keywords: Vec>, #[schema(inline)] pub src: AntennaSrc, pub user_list_id: Option, pub user_group_id: Option, pub users: Vec, - pub instances: newtype::JsonStringVec, + pub instances: Vec, #[serde(default)] pub case_sensitive: bool, #[serde(default)] diff --git a/packages/backend/native-utils/crates/model/tests/common.rs b/packages/backend/native-utils/crates/model/tests/common.rs index d459056f3f..f05c4432dc 100644 --- a/packages/backend/native-utils/crates/model/tests/common.rs +++ b/packages/backend/native-utils/crates/model/tests/common.rs @@ -5,7 +5,8 @@ mod repository; use chrono::Utc; use model::entity::{antenna, sea_orm_active_enums::AntennaSrcEnum, user}; use sea_orm::{ - ActiveModelTrait, ActiveValue::Set, DatabaseConnection, DbErr, EntityTrait, TransactionTrait, + ActiveModelTrait, ActiveValue::Set, ConnectionTrait, DbBackend, DbConn, DbErr, EntityTrait, + TransactionTrait, }; use std::env; use util::{ @@ -24,6 +25,15 @@ async fn prepare() { setup_model(db).await; } +async fn setup_schema(db: DbConn) { + // Setup Schema helper + let schema = sea_orm::Schema::new(DbBackend::Sqlite); + let stmt = schema.create_table_from_entity(antenna::Entity); + db.execute(db.get_database_backend().build(&stmt)) + .await + .expect("Unable to initialize in-memoty sqlite"); +} + /// Delete all entries in the database. async fn cleanup() { let db = database::get_database().unwrap(); @@ -39,7 +49,7 @@ async fn cleanup() { .expect("Unable to delete predefined models"); } -async fn setup_model(db: &DatabaseConnection) { +async fn setup_model(db: &DbConn) { init_id(12); db.transaction::<_, (), DbErr>(|txn| { @@ -89,6 +99,10 @@ async fn setup_model(db: &DatabaseConnection) { } mod int_test { + use sea_orm::Database; + + use crate::setup_schema; + use super::{cleanup, prepare}; #[tokio::test] @@ -96,4 +110,10 @@ mod int_test { prepare().await; cleanup().await; } + + #[tokio::test] + async fn can_setup_sqlite_schema() { + let db = Database::connect("sqlite::memory:").await.unwrap(); + setup_schema(db).await; + } }