add abstraction of string array type

packages/backend/native-utils/crates/database/src/lib.rs

@@ -1,10 +1,10 @@
 pub mod error;
 
-use sea_orm::{Database, DatabaseConnection};
+use sea_orm::{Database, DbConn};
 
 use crate::error::Error;
 
-static DB_CONN: once_cell::sync::OnceCell<DatabaseConnection> = once_cell::sync::OnceCell::new();
+static DB_CONN: once_cell::sync::OnceCell<DbConn> = once_cell::sync::OnceCell::new();
 
 pub async fn init_database(connection_uri: impl Into<String>) -> Result<(), Error> {
     let conn = Database::connect(connection_uri.into()).await?;
@@ -12,7 +12,7 @@ pub async fn init_database(connection_uri: impl Into<String>) -> Result<(), Erro
     Ok(())
 }
 
-pub fn get_database() -> Result<&'static DatabaseConnection, Error> {
+pub fn get_database() -> Result<&'static DbConn, Error> {
     DB_CONN.get().ok_or(Error::Uninitialized)
 }
 

packages/backend/native-utils/crates/migration/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "migration"
+version = "0.1.0"
+edition = "2021"
+publish = false
+
+[lib]
+name = "migration"
+path = "src/lib.rs"
+
+[dependencies]
+async-std = { version = "1", features = ["attributes", "tokio1"] }
+serde_json = "1.0.96"
+model = { path = "../model" }
+
+[dependencies.sea-orm-migration]
+version = "0.11.0"
+features = [
+  # Enable at least one `ASYNC_RUNTIME` and `DATABASE_DRIVER` feature if you want to run migration via CLI.
+  # View the list of supported features at https://www.sea-ql.org/SeaORM/docs/install-and-config/database-and-async-runtime.
+  # e.g.
+  "runtime-tokio-rustls",  # `ASYNC_RUNTIME` feature
+  "sqlx-postgres",         # `DATABASE_DRIVER` feature
+	"sqlx-sqlite",
+]

packages/backend/native-utils/crates/migration/README.md

@@ -0,0 +1,41 @@
+# Running Migrator CLI
+
+- Generate a new migration file
+    ```sh
+    cargo run -- migrate generate MIGRATION_NAME
+    ```
+- Apply all pending migrations
+    ```sh
+    cargo run
+    ```
+    ```sh
+    cargo run -- up
+    ```
+- Apply first 10 pending migrations
+    ```sh
+    cargo run -- up -n 10
+    ```
+- Rollback last applied migrations
+    ```sh
+    cargo run -- down
+    ```
+- Rollback last 10 applied migrations
+    ```sh
+    cargo run -- down -n 10
+    ```
+- Drop all tables from the database, then reapply all migrations
+    ```sh
+    cargo run -- fresh
+    ```
+- Rollback all applied migrations, then reapply all migrations
+    ```sh
+    cargo run -- refresh
+    ```
+- Rollback all applied migrations
+    ```sh
+    cargo run -- reset
+    ```
+- Check the status of all migrations
+    ```sh
+    cargo run -- status
+    ```

packages/backend/native-utils/crates/migration/src/lib.rs

@@ -0,0 +1,12 @@
+pub use sea_orm_migration::prelude::*;
+
+mod m20230531_180824_stringvec;
+
+pub struct Migrator;
+
+#[async_trait::async_trait]
+impl MigratorTrait for Migrator {
+    fn migrations() -> Vec<Box<dyn MigrationTrait>> {
+        vec![Box::new(m20230531_180824_stringvec::Migration)]
+    }
+}

packages/backend/native-utils/crates/migration/src/m20230531_180824_stringvec.rs

@@ -0,0 +1,74 @@
+use model::entity::{antenna, newtype::StringVec};
+use sea_orm_migration::{
+    prelude::*,
+    sea_orm::{DbBackend, EntityTrait, Statement},
+};
+use serde_json::json;
+
+#[derive(DeriveMigrationName)]
+pub struct Migration;
+
+#[async_trait::async_trait]
+impl MigrationTrait for Migration {
+    async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
+        if manager.get_database_backend() == DbBackend::Sqlite {
+            return Ok(());
+        }
+
+        let db = manager.get_connection();
+        let query = Query::select()
+            .column(Antenna::Id)
+            .column(Antenna::Users)
+            .from(Antenna::Table)
+            .to_string(PostgresQueryBuilder);
+        let res: Vec<(String, Vec<String>)> = db
+            .query_all(Statement::from_string(DbBackend::Postgres, query))
+            .await?
+            .iter()
+            .filter_map(|r| r.try_get_many_by_index().ok())
+            .collect();
+
+        manager
+            .alter_table(
+                Table::alter()
+                    .table(Antenna::Table)
+                    .drop_column(Antenna::Users)
+                    .add_column(
+                        ColumnDef::new(Antenna::Users)
+                            .json_binary()
+                            .not_null()
+                            .default(json!([])),
+                    )
+                    .to_owned(),
+            )
+            .await?;
+
+        let models: Vec<antenna::ActiveModel> = res
+            .iter()
+            .map(|(id, users)| antenna::ActiveModel {
+                id: sea_orm::Set(id.to_owned()),
+                users: sea_orm::Set(StringVec::from(users.to_owned())),
+                ..Default::default()
+            })
+            .collect();
+
+        for model in models {
+            antenna::Entity::update(model).exec(db).await?;
+        }
+
+        Ok(())
+    }
+
+    async fn down(&self, _manager: &SchemaManager) -> Result<(), DbErr> {
+        // Replace the sample below with your own migration scripts
+        Ok(())
+    }
+}
+
+/// Learn more at https://docs.rs/sea-query#iden
+#[derive(Iden)]
+enum Antenna {
+    Table,
+    Id,
+    Users,
+}

packages/backend/native-utils/crates/migration/src/main.rs

@@ -0,0 +1,6 @@
+use sea_orm_migration::prelude::*;
+
+#[async_std::main]
+async fn main() {
+    cli::run_cli(migration::Migrator).await;
+}

packages/backend/native-utils/crates/model/Cargo.toml

@@ -4,9 +4,13 @@ version = "0.1.0"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[features]
+default = ["compat"]
+compat = ["sea-orm/postgres-array"]
 
 [dependencies]
 async-trait = "0.1.68"
+cfg-if = "1.0.0"
 chrono = "0.4.24"
 database = { path = "../database" }
 derive_more = "0.99.17"
@@ -14,7 +18,7 @@ jsonschema = "0.17.0"
 once_cell = "1.17.1"
 parse-display = "0.8.0"
 schemars = { version = "0.8.12", features = ["chrono"] }
-sea-orm = { version = "0.11.3", features = ["postgres-array", "sqlx-postgres", "runtime-tokio-rustls"] }
+sea-orm = { version = "0.11.3", features = ["sqlx-postgres", "sqlx-sqlite", "runtime-tokio-rustls"] }
 serde = { version = "1.0.163", features = ["derive"] }
 serde_json = "1.0.96"
 thiserror = "1.0.40"

packages/backend/native-utils/crates/model/src/entity/antenna.rs

@@ -28,7 +28,7 @@ pub struct Model {
     pub with_replies: bool,
     #[sea_orm(column_name = "userGroupJoiningId")]
     pub user_group_joining_id: Option<String>,
-    pub users: Vec<String>,
+    pub users: newtype::StringVec,
     #[sea_orm(column_name = "excludeKeywords", column_type = "JsonBinary")]
     pub exclude_keywords: newtype::JsonKeyword,
     #[sea_orm(column_type = "JsonBinary")]

packages/backend/native-utils/crates/model/src/entity/newtype/macros.rs

@@ -33,12 +33,12 @@ macro_rules! impl_json_newtype {
                 stringify!($a).to_owned()
             }
 
-            fn array_type() -> sea_orm::sea_query::ArrayType {
+            fn array_type() -> sea_query::ArrayType {
-                sea_orm::sea_query::ArrayType::Json
+                sea_query::ArrayType::Json
             }
 
             fn column_type() -> sea_query::ColumnType {
-                sea_query::ColumnType::Json
+                sea_query::ColumnType::JsonBinary
             }
         }
 

packages/backend/native-utils/crates/model/src/entity/newtype/mod.rs

@@ -1,17 +1,24 @@
 mod macros;
 
-use derive_more::From;
+use cfg_if::cfg_if;
-use schemars::JsonSchema;
+use derive_more::{From, Into};
 use sea_orm::{sea_query, DbErr, QueryResult, TryGetError, TryGetable, Value};
 use serde::{Deserialize, Serialize};
 
 use crate::impl_json_newtype;
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema, From)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, From, Into)]
 pub struct JsonKeyword(pub Vec<Vec<String>>);
 impl_json_newtype!(JsonKeyword);
 
-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema, From)]
+#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, From, Into)]
-
 pub struct JsonStringVec(pub Vec<String>);
 impl_json_newtype!(JsonStringVec);
+
+cfg_if! {
+    if #[cfg(feature = "compat")] {
+        pub type StringVec = Vec<String>;
+    } else {
+        pub type StringVec = JsonStringVec;
+    }
+}

packages/backend/native-utils/crates/model/src/error.rs

@@ -3,7 +3,7 @@ pub enum Error {
     #[error("Failed to parse string")]
     ParseError(#[from] parse_display::ParseError),
     #[error("Failed to get database connection")]
-    DatabaseConnectionError(#[from] database::error::Error),
+    DbConnError(#[from] database::error::Error),
     #[error("Database operation error: {0}")]
     DatabaseOperationError(#[from] sea_orm::DbErr),
 }

packages/backend/native-utils/crates/model/src/repository/antenna.rs

@@ -30,13 +30,13 @@ impl Repository<Antenna> for antenna::Model {
             id: self.id,
             created_at: self.created_at.into(),
             name: self.name,
-            keywords: self.keywords,
+            keywords: self.keywords.into(),
-            exclude_keywords: self.exclude_keywords,
+            exclude_keywords: self.exclude_keywords.into(),
             src: self.src.try_into()?,
             user_list_id: self.user_list_id,
             user_group_id,
-            users: self.users,
+            users: self.users.into(),
-            instances: self.instances,
+            instances: self.instances.into(),
             case_sensitive: self.case_sensitive,
             notify: self.notify,
             with_replies: self.with_replies,

packages/backend/native-utils/crates/model/src/schema/antenna.rs

@@ -5,7 +5,7 @@ use schemars::JsonSchema;
 use utoipa::ToSchema;
 
 use super::Schema;
-use crate::entity::{newtype, sea_orm_active_enums::AntennaSrcEnum};
+use crate::entity::sea_orm_active_enums::AntennaSrcEnum;
 
 #[derive(Clone, Debug, PartialEq, Eq, JsonSchema, ToSchema)]
 #[serde(rename_all = "camelCase")]
@@ -13,14 +13,14 @@ pub struct Antenna {
     pub id: String,
     pub created_at: chrono::DateTime<chrono::Utc>,
     pub name: String,
-    pub keywords: newtype::JsonKeyword,
+    pub keywords: Vec<Vec<String>>,
-    pub exclude_keywords: newtype::JsonKeyword,
+    pub exclude_keywords: Vec<Vec<String>>,
     #[schema(inline)]
     pub src: AntennaSrc,
     pub user_list_id: Option<String>,
     pub user_group_id: Option<String>,
     pub users: Vec<String>,
-    pub instances: newtype::JsonStringVec,
+    pub instances: Vec<String>,
     #[serde(default)]
     pub case_sensitive: bool,
     #[serde(default)]

packages/backend/native-utils/crates/model/tests/common.rs

@@ -5,7 +5,8 @@ mod repository;
 use chrono::Utc;
 use model::entity::{antenna, sea_orm_active_enums::AntennaSrcEnum, user};
 use sea_orm::{
-    ActiveModelTrait, ActiveValue::Set, DatabaseConnection, DbErr, EntityTrait, TransactionTrait,
+    ActiveModelTrait, ActiveValue::Set, ConnectionTrait, DbBackend, DbConn, DbErr, EntityTrait,
+    TransactionTrait,
 };
 use std::env;
 use util::{
@@ -24,6 +25,15 @@ async fn prepare() {
     setup_model(db).await;
 }
 
+async fn setup_schema(db: DbConn) {
+    // Setup Schema helper
+    let schema = sea_orm::Schema::new(DbBackend::Sqlite);
+    let stmt = schema.create_table_from_entity(antenna::Entity);
+    db.execute(db.get_database_backend().build(&stmt))
+        .await
+        .expect("Unable to initialize in-memoty sqlite");
+}
+
 /// Delete all entries in the database.
 async fn cleanup() {
     let db = database::get_database().unwrap();
@@ -39,7 +49,7 @@ async fn cleanup() {
     .expect("Unable to delete predefined models");
 }
 
-async fn setup_model(db: &DatabaseConnection) {
+async fn setup_model(db: &DbConn) {
     init_id(12);
 
     db.transaction::<_, (), DbErr>(|txn| {
@@ -89,6 +99,10 @@ async fn setup_model(db: &DatabaseConnection) {
 }
 
 mod int_test {
+    use sea_orm::Database;
+
+    use crate::setup_schema;
+
     use super::{cleanup, prepare};
 
     #[tokio::test]
@@ -96,4 +110,10 @@ mod int_test {
         prepare().await;
         cleanup().await;
     }
+
+    #[tokio::test]
+    async fn can_setup_sqlite_schema() {
+        let db = Database::connect("sqlite::memory:").await.unwrap();
+        setup_schema(db).await;
+    }
 }