copy existing posts in antenna to redis at migration

This commit is contained in:
Namekuji 2023-07-10 01:39:33 -04:00
parent 6d7acfb7d3
commit 61f0f52d42
No known key found for this signature in database
GPG Key ID: 1D62332C07FBA532
14 changed files with 516 additions and 86 deletions

View File

@ -1,9 +1,7 @@
export class cleanup1680491187535 { export class cleanup1680491187535 {
name = "cleanup1680491187535"; name = "cleanup1680491187535";
async up(queryRunner) { async up(queryRunner) {}
await queryRunner.query(`DROP TABLE "antenna_note" `);
}
async down(queryRunner) {} async down(queryRunner) {}
} }

View File

@ -458,6 +458,20 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7"
[[package]]
name = "combine"
version = "4.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35ed6e9d84f0b51a7f52daf1c7d71dd136fd7a3f41a8462b8cdb8c78d920fad4"
dependencies = [
"bytes",
"futures-core",
"memchr",
"pin-project-lite",
"tokio",
"tokio-util",
]
[[package]] [[package]]
name = "console" name = "console"
version = "0.15.7" version = "0.15.7"
@ -486,6 +500,16 @@ dependencies = [
"unicode-segmentation", "unicode-segmentation",
] ]
[[package]]
name = "core-foundation"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]] [[package]]
name = "core-foundation-sys" name = "core-foundation-sys"
version = "0.8.4" version = "0.8.4"
@ -1278,11 +1302,14 @@ dependencies = [
"futures", "futures",
"indicatif", "indicatif",
"native-utils", "native-utils",
"redis",
"sea-orm",
"sea-orm-migration", "sea-orm-migration",
"serde", "serde",
"serde_json", "serde_json",
"serde_yaml", "serde_yaml",
"tokio", "tokio",
"url",
"urlencoding", "urlencoding",
] ]
@ -1509,6 +1536,12 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d"
[[package]]
name = "openssl-probe"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
[[package]] [[package]]
name = "os_str_bytes" name = "os_str_bytes"
version = "6.5.0" version = "6.5.0"
@ -1843,6 +1876,29 @@ dependencies = [
"rand_core", "rand_core",
] ]
[[package]]
name = "redis"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3ea8c51b5dc1d8e5fd3350ec8167f464ec0995e79f2e90a075b63371500d557f"
dependencies = [
"async-trait",
"bytes",
"combine",
"futures-util",
"itoa",
"percent-encoding",
"pin-project-lite",
"rustls 0.21.3",
"rustls-native-certs",
"ryu",
"sha1_smol",
"tokio",
"tokio-rustls 0.24.1",
"tokio-util",
"url",
]
[[package]] [[package]]
name = "redox_syscall" name = "redox_syscall"
version = "0.2.16" version = "0.2.16"
@ -2043,6 +2099,30 @@ dependencies = [
"webpki", "webpki",
] ]
[[package]]
name = "rustls"
version = "0.21.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b19faa85ecb5197342b54f987b142fb3e30d0c90da40f80ef4fa9a726e6676ed"
dependencies = [
"log",
"ring",
"rustls-webpki",
"sct",
]
[[package]]
name = "rustls-native-certs"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00"
dependencies = [
"openssl-probe",
"rustls-pemfile",
"schannel",
"security-framework",
]
[[package]] [[package]]
name = "rustls-pemfile" name = "rustls-pemfile"
version = "1.0.2" version = "1.0.2"
@ -2052,6 +2132,16 @@ dependencies = [
"base64 0.21.2", "base64 0.21.2",
] ]
[[package]]
name = "rustls-webpki"
version = "0.101.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15f36a6828982f422756984e47912a7a51dcbc2a197aa791158f8ca61cd8204e"
dependencies = [
"ring",
"untrusted",
]
[[package]] [[package]]
name = "rustversion" name = "rustversion"
version = "1.0.12" version = "1.0.12"
@ -2076,6 +2166,15 @@ version = "1.0.13"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
[[package]]
name = "schannel"
version = "0.1.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88"
dependencies = [
"windows-sys 0.48.0",
]
[[package]] [[package]]
name = "schemars" name = "schemars"
version = "0.8.12" version = "0.8.12"
@ -2286,6 +2385,29 @@ version = "4.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"
[[package]]
name = "security-framework"
version = "2.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1fc758eb7bffce5b308734e9b0c1468893cae9ff70ebf13e7090be8dcbcc83a8"
dependencies = [
"bitflags 1.3.2",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f51d0c0d83bec45f16480d0ce0058397a69e48fcdc52d1dc8855fb68acbd31a7"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]] [[package]]
name = "semver" name = "semver"
version = "1.0.17" version = "1.0.17"
@ -2370,6 +2492,12 @@ dependencies = [
"digest", "digest",
] ]
[[package]]
name = "sha1_smol"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012"
[[package]] [[package]]
name = "sha2" name = "sha2"
version = "0.10.6" version = "0.10.6"
@ -2518,7 +2646,7 @@ dependencies = [
"percent-encoding", "percent-encoding",
"rand", "rand",
"rust_decimal", "rust_decimal",
"rustls", "rustls 0.20.8",
"rustls-pemfile", "rustls-pemfile",
"serde", "serde",
"serde_json", "serde_json",
@ -2564,7 +2692,7 @@ checksum = "804d3f245f894e61b1e6263c84b23ca675d96753b5abfd5cc8597d86806e8024"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"tokio", "tokio",
"tokio-rustls", "tokio-rustls 0.23.4",
] ]
[[package]] [[package]]
@ -2778,11 +2906,21 @@ version = "0.23.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59"
dependencies = [ dependencies = [
"rustls", "rustls 0.20.8",
"tokio", "tokio",
"webpki", "webpki",
] ]
[[package]]
name = "tokio-rustls"
version = "0.24.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081"
dependencies = [
"rustls 0.21.3",
"tokio",
]
[[package]] [[package]]
name = "tokio-stream" name = "tokio-stream"
version = "0.1.14" version = "0.1.14"
@ -2949,6 +3087,7 @@ dependencies = [
"form_urlencoded", "form_urlencoded",
"idna", "idna",
"percent-encoding", "percent-encoding",
"serde",
] ]
[[package]] [[package]]

View File

@ -9,7 +9,7 @@ members = ["migration"]
[features] [features]
default = [] default = []
noarray = [] noarray = []
napi = ["dep:napi", "dep:napi-derive", "dep:radix_fmt"] napi = ["dep:napi", "dep:napi-derive"]
[lib] [lib]
crate-type = ["cdylib", "lib"] crate-type = ["cdylib", "lib"]
@ -31,11 +31,11 @@ serde_json = "1.0.96"
thiserror = "1.0.40" thiserror = "1.0.40"
tokio = { version = "1.28.1", features = ["full"] } tokio = { version = "1.28.1", features = ["full"] }
utoipa = "3.3.0" utoipa = "3.3.0"
radix_fmt = "1.0.0"
# Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix # Default enable napi4 feature, see https://nodejs.org/api/n-api.html#node-api-version-matrix
napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true } napi = { version = "2.13.1", default-features = false, features = ["napi6", "tokio_rt"], optional = true }
napi-derive = { version = "2.12.0", optional = true } napi-derive = { version = "2.12.0", optional = true }
radix_fmt = { version = "1.0.0", optional = true }
[dev-dependencies] [dev-dependencies]
pretty_assertions = "1.3.0" pretty_assertions = "1.3.0"

View File

@ -12,18 +12,18 @@ test("convert to mastodon id", (t) => {
t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219"); t.is(convertId("9gf61ehcxv", IdConvertType.MastodonId), "960365976481219");
t.is( t.is(
convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId), convertId("9fbr9z0wbrjqyd3u", IdConvertType.MastodonId),
"3954607381600562394", "2083785058661759970208986",
); );
t.is( t.is(
convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId), convertId("9fbs680oyviiqrol9md73p8g", IdConvertType.MastodonId),
"3494513243013053824", "5878598648988104013828532260828151168",
); );
}); });
test("create cuid2 with timestamp prefix", (t) => { test("create cuid2 with timestamp prefix", (t) => {
nativeInitIdGenerator(16, ""); nativeInitIdGenerator(16, "");
t.not(nativeCreateId(BigInt(Date.now())), nativeCreateId(BigInt(Date.now()))); t.not(nativeCreateId(Date.now()), nativeCreateId(Date.now()));
t.is(nativeCreateId(BigInt(Date.now())).length, 16); t.is(nativeCreateId(Date.now()).length, 16);
}); });
test("create random string", (t) => { test("create random string", (t) => {

View File

@ -10,17 +10,20 @@ path = "src/lib.rs"
[features] [features]
default = [] default = []
convert = ["dep:native-utils", "dep:indicatif", "dep:futures"] convert = ["dep:indicatif"]
[dependencies] [dependencies]
serde_json = "1.0.96" serde_json = "1.0.96"
native-utils = { path = "../", optional = true } native-utils = { path = "../" }
indicatif = { version = "0.17.4", features = ["tokio"], optional = true } indicatif = { version = "0.17.4", features = ["tokio"], optional = true }
tokio = { version = "1.28.2", features = ["full"] } tokio = { version = "1.28.2", features = ["full"] }
futures = { version = "0.3.28", optional = true } futures = "0.3.28"
serde_yaml = "0.9.21" serde_yaml = "0.9.21"
serde = { version = "1.0.163", features = ["derive"] } serde = { version = "1.0.163", features = ["derive"] }
urlencoding = "2.1.2" urlencoding = "2.1.2"
redis = { version = "0.23.0", features = ["tokio-rustls-comp"] }
sea-orm = "0.11.3"
url = { version = "2.4.0", features = ["serde"] }
[dependencies.sea-orm-migration] [dependencies.sea-orm-migration]
version = "0.11.0" version = "0.11.0"

View File

@ -2,6 +2,7 @@ pub use sea_orm_migration::prelude::*;
mod m20230531_180824_drop_reversi; mod m20230531_180824_drop_reversi;
mod m20230627_185451_index_note_url; mod m20230627_185451_index_note_url;
mod m20230709_000510_move_antenna_to_cache;
pub struct Migrator; pub struct Migrator;
@ -11,6 +12,7 @@ impl MigratorTrait for Migrator {
vec![ vec![
Box::new(m20230531_180824_drop_reversi::Migration), Box::new(m20230531_180824_drop_reversi::Migration),
Box::new(m20230627_185451_index_note_url::Migration), Box::new(m20230627_185451_index_note_url::Migration),
Box::new(m20230709_000510_move_antenna_to_cache::Migration),
] ]
} }
} }

View File

@ -0,0 +1,240 @@
use redis::streams::StreamMaxlen;
use sea_orm::Statement;
use sea_orm_migration::prelude::*;
use std::env;
#[derive(DeriveMigrationName)]
pub struct Migration;
#[async_trait::async_trait]
impl MigrationTrait for Migration {
async fn up(&self, manager: &SchemaManager) -> Result<(), DbErr> {
let cache_url = env::var("CACHE_URL").unwrap();
let copy_limit = env::var("ANTENNA_MIGRATION_LIMIT").unwrap_or_default();
let copy_limit: i64 = match copy_limit.parse() {
Ok(limit) => limit,
Err(_) => 0,
};
if cache_url != "no" {
let prefix = env::var("CACHE_PREFIX").unwrap();
let db = manager.get_connection();
let bk = manager.get_database_backend();
let count_stmt =
Statement::from_string(bk, "SELECT COUNT(1) FROM antenna_note".to_owned());
let total_num = db
.query_one(count_stmt)
.await?
.unwrap()
.try_get_by_index::<i64>(0)?;
let copy_limit = if copy_limit == 0 {
total_num
} else {
copy_limit
};
println!(
"Copying {} out of {} entries in antenna_note.",
copy_limit, total_num
);
let stmt_base = Query::select()
.column((AntennaNote::Table, AntennaNote::Id))
.column(AntennaNote::AntennaId)
.column(AntennaNote::NoteId)
.from(AntennaNote::Table)
.order_by((AntennaNote::Table, AntennaNote::Id), Order::Asc)
.limit(1000)
.to_owned();
let mut stmt = stmt_base.clone();
let client = redis::Client::open(cache_url).unwrap();
let mut redis_conn = client.get_connection().unwrap();
let mut remaining = total_num;
let mut pagination: i64 = 0;
loop {
let res = db.query_all(bk.build(&stmt)).await?;
if res.len() == 0 {
break;
}
let val: Vec<(String, String, String)> = res
.iter()
.filter_map(|q| q.try_get_many_by_index().ok())
.collect();
remaining -= val.len() as i64;
if remaining <= copy_limit {
let mut pipe = redis::pipe();
for v in &val {
pipe.xadd_maxlen(
format!("{}:antennaTimeline:{}", prefix, v.1),
StreamMaxlen::Approx(200),
"*",
&[("note", v.2.to_owned())],
)
.ignore();
}
pipe.query::<()>(&mut redis_conn).unwrap();
}
let copied = total_num - remaining;
let copied = std::cmp::min(copied, total_num);
pagination += 1;
if pagination % 100 == 0 {
println!(
"Migrating antenna [{:.2}%]",
(copied as f64 / total_num as f64) * 100_f64,
);
}
if let Some((last_id, _, _)) = val.last() {
stmt = stmt_base
.clone()
.and_where(
Expr::col((AntennaNote::Table, AntennaNote::Id)).gt(last_id.to_owned()),
)
.to_owned();
} else {
break;
}
}
}
println!("Migrating antenna [100.00%]");
manager
.drop_table(
Table::drop()
.table(AntennaNote::Table)
.if_exists()
.to_owned(),
)
.await?;
Ok(())
}
async fn down(&self, manager: &SchemaManager) -> Result<(), DbErr> {
manager
.create_table(
Table::create()
.table(AntennaNote::Table)
.if_not_exists()
.col(
ColumnDef::new(AntennaNote::Id)
.string_len(32)
.not_null()
.primary_key(),
)
.col(
ColumnDef::new(AntennaNote::NoteId)
.string_len(32)
.not_null(),
)
.col(
ColumnDef::new(AntennaNote::AntennaId)
.string_len(32)
.not_null(),
)
.col(
ColumnDef::new(AntennaNote::Read)
.boolean()
.default(false)
.not_null(),
)
.to_owned(),
)
.await?;
manager
.create_index(
Index::create()
.name("IDX_0d775946662d2575dfd2068a5f")
.table(AntennaNote::Table)
.col(AntennaNote::AntennaId)
.if_not_exists()
.to_owned(),
)
.await?;
manager
.create_index(
Index::create()
.name("IDX_bd0397be22147e17210940e125")
.table(AntennaNote::Table)
.col(AntennaNote::NoteId)
.if_not_exists()
.to_owned(),
)
.await?;
manager
.create_index(
Index::create()
.name("IDX_335a0bf3f904406f9ef3dd51c2")
.table(AntennaNote::Table)
.col(AntennaNote::NoteId)
.col(AntennaNote::AntennaId)
.unique()
.if_not_exists()
.to_owned(),
)
.await?;
manager
.create_index(
Index::create()
.name("IDX_9937ea48d7ae97ffb4f3f063a4")
.table(AntennaNote::Table)
.col(AntennaNote::Read)
.if_not_exists()
.to_owned(),
)
.await?;
manager
.create_foreign_key(
ForeignKey::create()
.name("FK_0d775946662d2575dfd2068a5f5")
.from(AntennaNote::Table, AntennaNote::AntennaId)
.to(Antenna::Table, Antenna::Id)
.on_delete(ForeignKeyAction::Cascade)
.to_owned(),
)
.await?;
manager
.create_foreign_key(
ForeignKey::create()
.name("FK_bd0397be22147e17210940e125b")
.from(AntennaNote::Table, AntennaNote::NoteId)
.to(Note::Table, Note::Id)
.on_delete(ForeignKeyAction::Cascade)
.to_owned(),
)
.await?;
Ok(())
}
}
/// Learn more at https://docs.rs/sea-query#iden
#[derive(Iden)]
enum AntennaNote {
Table,
Id,
#[iden = "noteId"]
NoteId,
#[iden = "antennaId"]
AntennaId,
Read,
}
#[derive(Iden)]
enum Antenna {
Table,
Id,
}
#[derive(Iden)]
enum Note {
Table,
Id,
}

View File

@ -5,6 +5,10 @@ use urlencoding::encode;
use sea_orm_migration::prelude::*; use sea_orm_migration::prelude::*;
const DB_URL_ENV: &str = "DATABASE_URL";
const CACHE_URL_ENV: &str = "CACHE_URL";
const CACHE_PREFIX_ENV: &str = "CACHE_PREFIX";
#[cfg(feature = "convert")] #[cfg(feature = "convert")]
mod vec_to_json; mod vec_to_json;
@ -15,17 +19,48 @@ async fn main() {
.expect("Failed to open '.config/default.yml'"); .expect("Failed to open '.config/default.yml'");
let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml"); let config: Config = serde_yaml::from_reader(yml).expect("Failed to parse yaml");
env::set_var( if env::var_os(DB_URL_ENV).is_none() {
"DATABASE_URL", env::set_var(
format!( DB_URL_ENV,
"postgres://{}:{}@{}:{}/{}", format!(
config.db.user, "postgres://{}:{}@{}:{}/{}",
encode(&config.db.pass), config.db.user,
config.db.host, encode(&config.db.pass),
config.db.port, config.db.host,
config.db.db, config.db.port,
), config.db.db,
); ),
);
};
if env::var_os(CACHE_URL_ENV).is_none() {
let redis_conf = match config.cache_server {
None => config.redis,
Some(conf) => conf,
};
let redis_proto = match redis_conf.tls {
None => "redis",
Some(_) => "rediss",
};
let redis_uri_userpass = match redis_conf.user {
None => "".to_string(),
Some(user) => format!("{}:{}@", user, redis_conf.pass.unwrap_or_default()),
};
let redis_uri_hostport = format!("{}:{}", redis_conf.host, redis_conf.port);
let redis_uri = format!(
"{}://{}{}",
redis_proto, redis_uri_userpass, redis_uri_hostport
);
env::set_var(CACHE_URL_ENV, redis_uri);
env::set_var(
CACHE_PREFIX_ENV,
if redis_conf.prefix.is_empty() {
config.url.host_str().unwrap()
} else {
&redis_conf.prefix
},
);
}
cli::run_cli(migration::Migrator).await; cli::run_cli(migration::Migrator).await;
@ -36,7 +71,10 @@ async fn main() {
#[derive(Debug, PartialEq, Deserialize)] #[derive(Debug, PartialEq, Deserialize)]
#[serde(rename = "camelCase")] #[serde(rename = "camelCase")]
pub struct Config { pub struct Config {
pub url: url::Url,
pub db: DbConfig, pub db: DbConfig,
pub redis: RedisConfig,
pub cache_server: Option<RedisConfig>,
} }
#[derive(Debug, PartialEq, Deserialize)] #[derive(Debug, PartialEq, Deserialize)]
@ -48,3 +86,24 @@ pub struct DbConfig {
pub user: String, pub user: String,
pub pass: String, pub pass: String,
} }
#[derive(Debug, PartialEq, Deserialize)]
#[serde(rename = "camelCase")]
pub struct RedisConfig {
pub host: String,
pub port: u32,
pub user: Option<String>,
pub pass: Option<String>,
pub tls: Option<TlsConfig>,
#[serde(default)]
pub db: u32,
#[serde(default)]
pub prefix: String,
}
#[derive(Debug, PartialEq, Deserialize)]
#[serde(rename = "camelCase")]
pub struct TlsConfig {
pub host: String,
pub reject_unauthorized: bool,
}

View File

@ -105,9 +105,9 @@ mod unit_test {
#[test] #[test]
fn app_valid() { fn app_valid() {
init_id(12, ""); init_id(16, "");
let instance = json!({ let instance = json!({
"id": create_id().unwrap(), "id": create_id(0).unwrap(),
"name": "Test App", "name": "Test App",
"secret": gen_string(24), "secret": gen_string(24),
"callbackUrl": "urn:ietf:wg:oauth:2.0:oob", "callbackUrl": "urn:ietf:wg:oauth:2.0:oob",
@ -119,9 +119,9 @@ mod unit_test {
#[test] #[test]
fn app_invalid() { fn app_invalid() {
init_id(12, ""); init_id(16, "");
let instance = json!({ let instance = json!({
"id": create_id().unwrap(), "id": create_id(0).unwrap(),
// "name" is required // "name" is required
"name": null, "name": null,
// "permission" must be one of the app permissions // "permission" must be one of the app permissions

View File

@ -1,7 +1,10 @@
//! ID generation utility based on [cuid2] //! ID generation utility based on [cuid2]
use cfg_if::cfg_if; use cfg_if::cfg_if;
use chrono::Utc;
use once_cell::sync::OnceCell; use once_cell::sync::OnceCell;
use radix_fmt::radix_36;
use std::cmp;
use crate::impl_into_napi_error; use crate::impl_into_napi_error;
@ -14,47 +17,56 @@ impl_into_napi_error!(ErrorUninitialized);
static FINGERPRINT: OnceCell<String> = OnceCell::new(); static FINGERPRINT: OnceCell<String> = OnceCell::new();
static GENERATOR: OnceCell<cuid2::CuidConstructor> = OnceCell::new(); static GENERATOR: OnceCell<cuid2::CuidConstructor> = OnceCell::new();
const TIME_2000: i64 = 946_684_800_000;
const TIMESTAMP_LENGTH: u16 = 8;
/// Initializes Cuid2 generator. Must be called before any [create_id]. /// Initializes Cuid2 generator. Must be called before any [create_id].
pub fn init_id(length: u16, fingerprint: impl Into<String>) { pub fn init_id<'a>(length: u16, fingerprint: &'a str) {
FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint.into(), cuid2::create_id())); FINGERPRINT.get_or_init(move || format!("{}{}", fingerprint, cuid2::create_id()));
GENERATOR.get_or_init(move || { GENERATOR.get_or_init(move || {
cuid2::CuidConstructor::new() cuid2::CuidConstructor::new()
.with_length(length) // length to pass shoule be greater than or equal to 8.
.with_length(cmp::max(length - TIMESTAMP_LENGTH, 8))
.with_fingerprinter(|| FINGERPRINT.get().unwrap().clone()) .with_fingerprinter(|| FINGERPRINT.get().unwrap().clone())
}); });
} }
/// Returns Cuid2 with the length specified by [init_id]. Must be called after /// Returns Cuid2 with the length specified by [init_id]. Must be called after
/// [init_id], otherwise returns [ErrorUninitialized]. /// [init_id], otherwise returns [ErrorUninitialized].
pub fn create_id() -> Result<String, ErrorUninitialized> { /// The current timestamp via [chrono::Utc] is used if `date_num` is `0`.
pub fn create_id(date_num: i64) -> Result<String, ErrorUninitialized> {
match GENERATOR.get() { match GENERATOR.get() {
None => Err(ErrorUninitialized), None => Err(ErrorUninitialized),
Some(gen) => Ok(gen.create_id()), Some(gen) => {
let date_num = if date_num > 0 {
date_num
} else {
Utc::now().timestamp_millis()
};
let time = cmp::max(date_num - TIME_2000, 0);
Ok(format!(
"{:0>8}{}",
radix_36(time).to_string(),
gen.create_id()
))
}
} }
} }
cfg_if! { cfg_if! {
if #[cfg(feature = "napi")] { if #[cfg(feature = "napi")] {
use radix_fmt::radix_36;
use std::cmp;
use napi::bindgen_prelude::BigInt;
use napi_derive::napi; use napi_derive::napi;
const TIME_2000: u64 = 946_684_800_000;
const TIMESTAMP_LENGTH: u16 = 8;
/// Calls [init_id] inside. Must be called before [native_create_id]. /// Calls [init_id] inside. Must be called before [native_create_id].
#[napi] #[napi]
pub fn native_init_id_generator(length: u16, fingerprint: String) { pub fn native_init_id_generator(length: u16, fingerprint: String) {
// length to pass init_id shoule be greater than or equal to 8. init_id(length, &fingerprint);
init_id(cmp::max(length - TIMESTAMP_LENGTH, 8), fingerprint);
} }
/// Generates /// Generates
#[napi] #[napi]
pub fn native_create_id(date_num: BigInt) -> String { pub fn native_create_id(date_num: i64) -> String {
let time = cmp::max(date_num.get_u64().1 - TIME_2000, 0); create_id(date_num).unwrap()
format!("{:0>8}{}", radix_36(time).to_string(), create_id().unwrap())
} }
} }
} }
@ -62,37 +74,17 @@ cfg_if! {
#[cfg(test)] #[cfg(test)]
mod unit_test { mod unit_test {
use crate::util::id; use crate::util::id;
use cfg_if::cfg_if;
use pretty_assertions::{assert_eq, assert_ne}; use pretty_assertions::{assert_eq, assert_ne};
use std::thread; use std::thread;
cfg_if! { #[test]
if #[cfg(feature = "napi")] { fn can_generate_unique_ids() {
use chrono::Utc; assert_eq!(id::create_id(0), Err(id::ErrorUninitialized));
id::init_id(16, "");
#[test] assert_eq!(id::create_id(0).unwrap().len(), 16);
fn can_generate_aid_compat_ids() { assert_ne!(id::create_id(0).unwrap(), id::create_id(0).unwrap());
id::native_init_id_generator(20, "".to_string()); let id1 = thread::spawn(|| id::create_id(0).unwrap());
let id1 = id::native_create_id(Utc::now().timestamp_millis().into()); let id2 = thread::spawn(|| id::create_id(0).unwrap());
assert_eq!(id1.len(), 20); assert_ne!(id1.join().unwrap(), id2.join().unwrap());
let id1 = id::native_create_id(Utc::now().timestamp_millis().into());
let id2 = id::native_create_id(Utc::now().timestamp_millis().into());
assert_ne!(id1, id2);
let id1 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into()));
let id2 = thread::spawn(|| id::native_create_id(Utc::now().timestamp_millis().into()));
assert_ne!(id1.join().unwrap(), id2.join().unwrap());
}
} else {
#[test]
fn can_generate_unique_ids() {
assert_eq!(id::create_id(), Err(id::ErrorUninitialized));
id::init_id(12, "");
assert_eq!(id::create_id().unwrap().len(), 12);
assert_ne!(id::create_id().unwrap(), id::create_id().unwrap());
let id1 = thread::spawn(|| id::create_id().unwrap());
let id2 = thread::spawn(|| id::create_id().unwrap());
assert_ne!(id1.join().unwrap(), id2.join().unwrap());
}
}
} }
} }

View File

@ -139,11 +139,11 @@ async fn cleanup() {
} }
async fn setup_model(db: &DbConn) { async fn setup_model(db: &DbConn) {
init_id(12, ""); init_id(16, "");
db.transaction::<_, (), DbErr>(|txn| { db.transaction::<_, (), DbErr>(|txn| {
Box::pin(async move { Box::pin(async move {
let user_id = create_id().unwrap(); let user_id = create_id(0).unwrap();
let name = "Alice"; let name = "Alice";
let user_model = entity::user::Model { let user_model = entity::user::Model {
id: user_id.to_owned(), id: user_id.to_owned(),
@ -161,7 +161,7 @@ async fn setup_model(db: &DbConn) {
.insert(txn) .insert(txn)
.await?; .await?;
let antenna_model = entity::antenna::Model { let antenna_model = entity::antenna::Model {
id: create_id().unwrap(), id: create_id(0).unwrap(),
created_at: Utc::now().into(), created_at: Utc::now().into(),
user_id: user_id.to_owned(), user_id: user_id.to_owned(),
name: "Alice Antenna".to_string(), name: "Alice Antenna".to_string(),
@ -186,7 +186,7 @@ async fn setup_model(db: &DbConn) {
.insert(txn) .insert(txn)
.await?; .await?;
let note_model = entity::note::Model { let note_model = entity::note::Model {
id: create_id().unwrap(), id: create_id(0).unwrap(),
created_at: Utc::now().into(), created_at: Utc::now().into(),
text: Some("Testing 123".to_string()), text: Some("Testing 123".to_string()),
user_id: user_id.to_owned(), user_id: user_id.to_owned(),

View File

@ -95,7 +95,7 @@ mod int_test {
.unwrap() .unwrap()
.expect("note not found"); .expect("note not found");
let antenna_note = antenna_note::Model { let antenna_note = antenna_note::Model {
id: util::id::create_id().unwrap(), id: util::id::create_id(0).unwrap(),
antenna_id: alice_antenna.id.to_owned(), antenna_id: alice_antenna.id.to_owned(),
note_id: note_model.id.to_owned(), note_id: note_model.id.to_owned(),
read: false, read: false,

View File

@ -17,5 +17,5 @@ nativeInitIdGenerator(length, fingerprint);
* Ref: https://github.com/paralleldrive/cuid2#parameterized-length * Ref: https://github.com/paralleldrive/cuid2#parameterized-length
*/ */
export function genId(date?: Date): string { export function genId(date?: Date): string {
return nativeCreateId(BigInt((date ?? new Date()).getTime())); return nativeCreateId((date ?? new Date()).getTime());
} }

View File

@ -8,17 +8,14 @@ import type { User } from "@/models/entities/user.js";
export async function addNoteToAntenna( export async function addNoteToAntenna(
antenna: Antenna, antenna: Antenna,
note: Note, note: Note,
noteUser: { id: User["id"] }, _noteUser: { id: User["id"] },
) { ) {
// 通知しない設定になっているか、自分自身の投稿なら既読にする
const read = !antenna.notify || antenna.userId === noteUser.id;
redisClient.xadd( redisClient.xadd(
`antennaTimeline:${antenna.id}`, `antennaTimeline:${antenna.id}`,
"MAXLEN", "MAXLEN",
"~", "~",
"200", "200",
`${genId(note.createdAt)}-*`, "*",
"note", "note",
note.id, note.id,
); );