diff --git a/Cargo.lock b/Cargo.lock index 91a74f1..c8158f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -263,6 +263,15 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + [[package]] name = "base64" version = "0.13.1" @@ -1333,6 +1342,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "is_ci" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb" + [[package]] name = "itertools" version = "0.10.5" @@ -1342,6 +1357,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -1411,6 +1435,15 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +[[package]] +name = "lru" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" +dependencies = [ + "hashbrown 0.14.0", +] + [[package]] name = "magnetar" version = "0.2.1-alpha" @@ -1422,6 +1455,8 @@ dependencies = [ "dotenvy", "headers", "hyper", + "itertools 0.11.0", + "lru", "magnetar_calckey_model", "magnetar_common", "magnetar_core", @@ -1599,8 +1634,17 @@ version = "5.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" dependencies = [ + "backtrace", + "backtrace-ext", + "is-terminal", "miette-derive", "once_cell", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", "thiserror", "unicode-width", ] @@ -1812,6 +1856,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2702,6 +2752,12 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +[[package]] +name = "smawk" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" + [[package]] name = "socket2" version = "0.4.9" @@ -2743,7 +2799,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" dependencies = [ - "itertools", + "itertools 0.10.5", "nom", "unicode_categories", ] @@ -3015,6 +3071,34 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "supports-color" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4950e7174bffabe99455511c39707310e7e9b440364a2fcb1cc21521be57b354" +dependencies = [ + "is-terminal", + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84231692eb0d4d41e4cdd0cabfdd2e6cd9e255e65f80c9aa7c98dd502b4233d" +dependencies = [ + "is-terminal", +] + +[[package]] +name = "supports-unicode" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6c2cb240ab5dd21ed4906895ee23fe5a48acdbd15a3ce388e7b62a9b66baf7" +dependencies = [ + "is-terminal", +] + [[package]] name = "syn" version = "1.0.109" @@ -3093,6 +3177,27 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "textwrap" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.44" @@ -3514,6 +3619,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-linebreak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5faade31a542b8b35855fff6e8def199853b2da8da256da52f52f1316ee3137" +dependencies = [ + "hashbrown 0.12.3", + "regex", +] + [[package]] name = "unicode-normalization" version = "0.1.22" diff --git a/Cargo.toml b/Cargo.toml index 05886ae..172b55b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,8 +37,10 @@ futures-util = "0.3" headers = "0.3" http = "0.2" hyper = "0.14" +itertools = "0.11" js-sys = "0.3" log = "0.4" +lru = "0.12" miette = "5.9" nom = "7" nom_locate = "4" @@ -78,6 +80,8 @@ magnetar_calckey_model = { path = "./ext_calckey_model" } magnetar_sdk = { path = "./magnetar_sdk" } cached = { workspace = true } +lru = { workspace = true } + chrono = { workspace = true } dotenvy = { workspace = true } @@ -93,9 +97,11 @@ tracing = { workspace = true } cfg-if = { workspace = true } +itertools = { workspace = true } + strum = { workspace = true, features = ["derive"] } thiserror = { workspace = true } -miette = { workspace = true } +miette = { workspace = true, features = ["fancy"] } percent-encoding = { workspace = true } diff --git a/ext_calckey_model/src/lib.rs b/ext_calckey_model/src/lib.rs index 1fb0704..6f8fdc3 100644 --- a/ext_calckey_model/src/lib.rs +++ b/ext_calckey_model/src/lib.rs @@ -1,5 +1,6 @@ pub use ck; use ck::*; +pub use sea_orm; use chrono::Utc; use ext_calckey_model_migration::{Migrator, MigratorTrait}; @@ -124,6 +125,39 @@ impl CalckeyModel { .await?) } + pub async fn fetch_emoji( + &self, + shortcode: &str, + host: Option<&str>, + ) -> Result, CalckeyDbError> { + let host_filter = if let Some(host) = host { + emoji::Column::Host.eq(host) + } else { + emoji::Column::Host.is_null() + }; + + let name_filter = emoji::Column::Name.eq(shortcode); + let filter = host_filter.and(name_filter); + + Ok(emoji::Entity::find().filter(filter).one(&self.0).await?) + } + pub async fn fetch_many_emojis( + &self, + shortcodes: &[String], + host: Option<&str>, + ) -> Result, CalckeyDbError> { + let host_filter = if let Some(host) = host { + emoji::Column::Host.eq(host) + } else { + emoji::Column::Host.is_null() + }; + + let name_filter = emoji::Column::Name.is_in(shortcodes); + let filter = host_filter.and(name_filter); + + Ok(emoji::Entity::find().filter(filter).all(&self.0).await?) + } + pub async fn get_access_token( &self, token: &str, diff --git a/magnetar_mmm_parser/src/lib.rs b/magnetar_mmm_parser/src/lib.rs index 25dfbc2..613f269 100644 --- a/magnetar_mmm_parser/src/lib.rs +++ b/magnetar_mmm_parser/src/lib.rs @@ -29,6 +29,7 @@ use unicode_segmentation::UnicodeSegmentation; pub enum MentionType { Community, User, + MatrixUser, } impl MentionType { @@ -36,6 +37,14 @@ impl MentionType { match self { MentionType::Community => '!', MentionType::User => '@', + MentionType::MatrixUser => ':', + } + } + + pub fn separator(&self) -> char { + match self { + MentionType::Community | MentionType::User => '@', + MentionType::MatrixUser => ':', } } } @@ -224,6 +233,28 @@ impl Token { } } + pub fn walk_map_collect(&self, func: &impl Fn(&Token) -> Option, out: &mut Vec) { + if let Some(v) = func(self) { + out.push(v) + } + + match self { + Token::Sequence(items) => { + items.iter().for_each(|tok| tok.walk_map_collect(func, out)); + } + Token::Quote(inner) + | Token::Small(inner) + | Token::BoldItalic(inner) + | Token::Bold(inner) + | Token::Italic(inner) + | Token::Center(inner) + | Token::Function { inner, .. } + | Token::Link { label: inner, .. } + | Token::Strikethrough(inner) => inner.walk_map_collect(func, out), + _ => {} + } + } + fn write(&self, writer: &mut quick_xml::Writer) -> quick_xml::Result<()> { match self { Token::PlainText(plain) => { @@ -635,6 +666,16 @@ impl Context { } } + pub fn parse_profile_fields(&self, input: &str) -> Token { + match self.inline_profile_fields(Span::new_extra(input, SpanMeta::default())) { + Ok((_, t)) => t.merged(), + Err(e) => { + trace!(input = input, "Profile field parser fail: {:?}", e); + Token::PlainText(e.to_compact_string()) + } + } + } + #[inline] fn partial( &self, @@ -666,6 +707,19 @@ impl Context { )(input) } + fn inline_profile_fields<'a>(&self, input: Span<'a>) -> IResult, Token> { + map( + many1(alt(( + self.partial(Self::unicode_emoji), + self.partial(Self::tag_mention), + self.partial(Self::tag_hashtag), + self.partial(Self::raw_url), + self.partial(Self::tag_raw_text), + ))), + Token::Sequence, + )(input) + } + fn inline_ui<'a>(&self, input: Span<'a>) -> IResult, Token> { map( many1(alt(( @@ -1450,21 +1504,30 @@ impl Context { )(input)?; let before = input; - let (_, host) = map( - opt(tuple(( - tag("@"), - map( - recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))), - Span::into_fragment, - ), - ))), - |maybe_tag_host| maybe_tag_host.map(|(_, host)| host), - )(input)?; + let (_, host_opt) = opt(tuple(( + one_of(if matches!(mention_type, MentionType::User) { + "@:" + } else { + "@" + }), + map( + recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))), + Span::into_fragment, + ), + )))(input)?; - let host = host.map(|h| h.trim_end_matches(|c| matches!(c, '.' | '-' | '_'))); + // Promote tags with a colon separator to Matrix handles + let mention_type = if let Some((':', _)) = host_opt { + MentionType::MatrixUser + } else { + mention_type + }; + let host = + host_opt.map(|(_, name)| name.trim_end_matches(|c| matches!(c, '.' | '-' | '_'))); + let input = host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before); Ok(( - host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before), + input, Token::Mention { mention_type, name: name.into(), @@ -2145,6 +2208,15 @@ text"# Token::PlainText(" test".into()) ]) ); + + assert_eq!( + parse_full("@tag:domain.com"), + Token::Mention { + mention_type: crate::MentionType::MatrixUser, + name: "tag".into(), + host: Some("domain.com".into()) + }, + ); } #[test] diff --git a/magnetar_sdk/src/types/mod.rs b/magnetar_sdk/src/types/mod.rs index a3d7508..215c045 100644 --- a/magnetar_sdk/src/types/mod.rs +++ b/magnetar_sdk/src/types/mod.rs @@ -23,12 +23,19 @@ pub struct Id { pub id: String, } -impl From<&str> for Id { - fn from(id: &str) -> Self { - Self { id: id.to_string() } +impl> From for Id { + fn from(id: T) -> Self { + Self { + id: id.as_ref().to_string(), + } } } +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[ts(export)] +#[repr(transparent)] +pub struct MmXml(pub String); + #[derive(Copy, Clone, Debug, Deserialize, Serialize, TS)] #[ts(export)] pub enum NotificationType { diff --git a/magnetar_sdk/src/types/note.rs b/magnetar_sdk/src/types/note.rs index e5e5dbb..2942600 100644 --- a/magnetar_sdk/src/types/note.rs +++ b/magnetar_sdk/src/types/note.rs @@ -1,6 +1,6 @@ use crate::types::emoji::EmojiContext; use crate::types::user::PackUserBase; -use crate::types::Id; +use crate::types::{Id, MmXml}; use crate::{Packed, Required}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -52,9 +52,11 @@ pack!(PackPollBase, Required as id & Required as poll); pub struct NoteBase { pub created_at: DateTime, pub cw: Option, + pub cw_mm: Option, pub uri: Option, pub url: Option, pub text: String, + pub text_mm: MmXml, pub visibility: NoteVisibility, pub user: Box, pub parent_note_id: Option, @@ -63,10 +65,10 @@ pub struct NoteBase { pub renote_count: u64, pub hashtags: Vec, pub reactions: Vec, - pub emojis: EmojiContext, pub local_only: bool, pub has_poll: bool, pub file_ids: Vec, + pub emojis: EmojiContext, } pack!(PackNoteBase, Required as id & Required as note); diff --git a/magnetar_sdk/src/types/user.rs b/magnetar_sdk/src/types/user.rs index 6ff7e53..a515716 100644 --- a/magnetar_sdk/src/types/user.rs +++ b/magnetar_sdk/src/types/user.rs @@ -1,5 +1,5 @@ use crate::types::emoji::EmojiContext; -use crate::types::{Id, NotificationSettings}; +use crate::types::{Id, MmXml, NotificationSettings}; use crate::{Packed, Required}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -29,6 +29,7 @@ pub enum SpeechTransform { pub struct ProfileField { name: String, value: String, + value_mm: Option, verified_at: Option>, } @@ -38,6 +39,7 @@ pub struct UserBase { pub acct: String, pub username: String, pub display_name: String, + pub display_name_mm: Option, pub host: Option, pub speech_transform: SpeechTransform, pub created_at: DateTime, @@ -61,6 +63,7 @@ pub struct UserProfileExt { pub is_suspended: bool, pub description: Option, + pub description_mm: Option, pub location: Option, pub birthday: Option>, pub fields: Vec, @@ -79,6 +82,7 @@ pub struct UserProfileExt { pub banner_blurhash: Option, pub has_public_reactions: bool, + pub emojis: EmojiContext, } #[derive(Clone, Debug, Deserialize, Serialize, TS)] @@ -167,6 +171,12 @@ pack!( & Option as auth ); +impl From for PackUserMaybeAll { + fn from(value: PackUserBase) -> Self { + Self::pack_from((value.id, value.user, None, None, None, None, None)) + } +} + pack!( PackUserSelfMaybeAll, Required as id @@ -176,3 +186,9 @@ pack!( & Option as detail & Option as secrets ); + +impl From for PackUserSelfMaybeAll { + fn from(value: PackUserBase) -> Self { + Self::pack_from((value.id, value.user, None, None, None, None)) + } +} diff --git a/src/api_v1/user.rs b/src/api_v1/user.rs index f2b7c9b..74cdb18 100644 --- a/src/api_v1/user.rs +++ b/src/api_v1/user.rs @@ -1,8 +1,11 @@ +use crate::model::processing::user::UserModel; +use crate::model::PackingContext; use crate::service::MagnetarService; use crate::web::auth::{AuthenticatedUser, MaybeUser}; -use crate::web::ApiError; +use crate::web::{ApiError, ObjectNotFound}; use axum::extract::{Path, Query, State}; use axum::Json; +use magnetar_calckey_model::ck; use magnetar_sdk::endpoints::user::{GetUserById, GetUserSelf, UserByIdReq, UserSelfReq}; use magnetar_sdk::endpoints::{Req, Res}; use std::sync::Arc; @@ -14,14 +17,18 @@ pub async fn handle_user_info_self( profile: _, secrets: _, }): Query>, - State(_service): State>, - AuthenticatedUser(_user): AuthenticatedUser, + State(service): State>, + AuthenticatedUser(user): AuthenticatedUser, ) -> Result>, ApiError> { - todo!() + // TODO: Extended properties! + + let ctx = PackingContext::new(service, Some(user.clone())).await?; + let user = UserModel.base_from_existing(&ctx, user.as_ref()).await?; + Ok(Json(user.into())) } pub async fn handle_user_info( - Path(_id): Path, + Path(id): Path, Query(UserByIdReq { detail: _, pins: _, @@ -29,8 +36,18 @@ pub async fn handle_user_info( relation: _, auth: _, }): Query>, - State(_service): State>, - MaybeUser(_user): MaybeUser, + State(service): State>, + MaybeUser(self_user): MaybeUser, ) -> Result>, ApiError> { - todo!() + // TODO: Extended properties! + + let ctx = PackingContext::new(service.clone(), self_user).await?; + let user_model = service + .db + .get_user_by_id(&id) + .await? + .ok_or_else(|| ObjectNotFound(id))?; + + let user = UserModel.base_from_existing(&ctx, &user_model).await?; + Ok(Json(user.into())) } diff --git a/src/model/data/note.rs b/src/model/data/note.rs index f2fcf83..147bb78 100644 --- a/src/model/data/note.rs +++ b/src/model/data/note.rs @@ -6,6 +6,7 @@ use magnetar_sdk::types::{ emoji::EmojiContext, note::{NoteAttachmentExt, NoteBase, NoteVisibility, PackReactionBase, ReactionBase}, user::UserBase, + MmXml, }; use magnetar_sdk::{Packed, Required}; @@ -18,7 +19,7 @@ impl PackType<&ck::note_reaction::Model> for ReactionBase { user_id: reaction.user_id.clone(), reaction: Reaction::guess_from( &reaction.reaction, - &context.instance_info.default_reaction, + &context.instance_meta.default_reaction, ) .unwrap_or_else(|| /* Shouldn't happen */ Reaction::Unicode("👍".to_string())), } @@ -27,9 +28,11 @@ impl PackType<&ck::note_reaction::Model> for ReactionBase { pub struct NoteBaseSource<'a> { pub note: &'a ck::note::Model, + pub cw_mm: Option<&'a MmXml>, + pub text_mm: &'a MmXml, pub reactions: &'a Vec, - pub emojis: &'a EmojiContext, pub user: &'a UserBase, + pub emoji_context: &'a EmojiContext, } impl PackType> for NoteBase { @@ -37,18 +40,22 @@ impl PackType> for NoteBase { _context: &PackingContext, NoteBaseSource { note, + text_mm, + cw_mm, reactions, - emojis, user, + emoji_context, }: NoteBaseSource<'_>, ) -> Self { use ck::sea_orm_active_enums::NoteVisibilityEnum as NVE; NoteBase { created_at: note.created_at.into(), cw: note.cw.clone(), + cw_mm: cw_mm.cloned(), uri: note.uri.clone(), url: note.url.clone(), text: note.text.clone().unwrap_or_default(), + text_mm: text_mm.clone(), visibility: match note.visibility { NVE::Followers => NoteVisibility::Followers, NVE::Hidden => NoteVisibility::Direct, @@ -66,10 +73,10 @@ impl PackType> for NoteBase { renote_count: note.renote_count as u64, hashtags: note.tags.clone(), reactions: reactions.clone(), - emojis: emojis.clone(), local_only: note.local_only, has_poll: note.has_poll, file_ids: note.file_ids.clone(), + emojis: emoji_context.clone(), } } } diff --git a/src/model/data/user.rs b/src/model/data/user.rs index 5975976..4ec64bc 100644 --- a/src/model/data/user.rs +++ b/src/model/data/user.rs @@ -3,9 +3,10 @@ use magnetar_calckey_model::ck::sea_orm_active_enums::UserProfileFfvisibilityEnu use magnetar_sdk::types::emoji::{EmojiContext, PackEmojiBase}; use magnetar_sdk::types::note::PackNoteFull; use magnetar_sdk::types::user::{ - AvatarDecoration, PackSecurityKeyBase, SecurityKeyBase, SpeechTransform, UserBase, - UserDetailExt, UserProfileExt, UserProfilePinsEx, UserRelationExt, UserSecretsExt, + AvatarDecoration, PackSecurityKeyBase, ProfileField, SecurityKeyBase, SpeechTransform, + UserBase, UserDetailExt, UserProfileExt, UserProfilePinsEx, UserRelationExt, UserSecretsExt, }; +use magnetar_sdk::types::MmXml; use crate::model::{PackType, PackingContext}; @@ -15,14 +16,23 @@ impl PackType<&[PackEmojiBase]> for EmojiContext { } } -type UserBaseSource<'a> = ( - &'a ck::user::Model, - &'a Option, - &'a EmojiContext, -); +pub struct UserBaseSource<'a> { + pub user: &'a ck::user::Model, + pub username_mm: Option<&'a MmXml>, + pub avatar: Option<&'a ck::drive_file::Model>, + pub emoji_context: &'a EmojiContext, +} impl PackType> for UserBase { - fn extract(_context: &PackingContext, (user, avatar, emoji_context): UserBaseSource) -> Self { + fn extract( + _context: &PackingContext, + UserBaseSource { + user, + username_mm, + avatar, + emoji_context, + }: UserBaseSource, + ) -> Self { UserBase { acct: user .host @@ -31,6 +41,7 @@ impl PackType> for UserBase { .unwrap_or_else(|| format!("@{}", user.username)), username: user.username.clone(), display_name: user.name.clone().unwrap_or_else(|| user.username.clone()), + display_name_mm: username_mm.cloned(), host: user.host.clone(), speech_transform: if user.is_cat && user.speak_as_cat { SpeechTransform::Cat @@ -38,8 +49,8 @@ impl PackType> for UserBase { SpeechTransform::None }, created_at: user.created_at.into(), - avatar_url: avatar.as_ref().map(|v| v.url.clone()), - avatar_blurhash: avatar.as_ref().and_then(|v| v.blurhash.clone()), + avatar_url: avatar.map(|v| v.url.clone()), + avatar_blurhash: avatar.and_then(|v| v.blurhash.clone()), avatar_color: None, avatar_decoration: if user.is_cat { AvatarDecoration::CatEars @@ -54,14 +65,27 @@ impl PackType> for UserBase { } } -type UserProfileExtSource<'a> = ( - &'a ck::user::Model, - &'a ck::user_profile::Model, - Option<&'a UserRelationExt>, -); +pub struct UserProfileExtSource<'a> { + pub user: &'a ck::user::Model, + pub profile: &'a ck::user_profile::Model, + pub profile_fields: &'a Vec, + pub description_mm: Option<&'a MmXml>, + pub relation: Option<&'a UserRelationExt>, + pub emoji_context: &'a EmojiContext, +} impl PackType> for UserProfileExt { - fn extract(context: &PackingContext, (user, profile, relation): UserProfileExtSource) -> Self { + fn extract( + context: &PackingContext, + UserProfileExtSource { + user, + profile, + profile_fields, + description_mm, + relation, + emoji_context, + }: UserProfileExtSource, + ) -> Self { let follow_visibility = match profile.ff_visibility { UserProfileFfvisibilityEnum::Public => true, UserProfileFfvisibilityEnum::Followers => relation.is_some_and(|r| r.follows_you), @@ -73,12 +97,13 @@ impl PackType> for UserProfileExt { is_silenced: user.is_silenced, is_suspended: user.is_suspended, description: profile.description.clone(), + description_mm: description_mm.cloned(), location: profile.location.clone(), birthday: profile .birthday .clone() .and_then(|b| b.parse().map_or_else(|_| None, Some)), - fields: serde_json::from_value(profile.fields.clone()).unwrap_or_else(|_| Vec::new()), + fields: profile_fields.clone(), follower_count: follow_visibility.then_some(user.followers_count as u64), following_count: follow_visibility.then_some(user.following_count as u64), note_count: Some(user.notes_count as u64), @@ -89,6 +114,7 @@ impl PackType> for UserProfileExt { banner_color: None, banner_blurhash: None, has_public_reactions: profile.public_reactions, + emojis: emoji_context.clone(), } } } @@ -103,19 +129,26 @@ impl PackType<&ck::user::Model> for UserDetailExt { } } -type UserRelationExtSource<'a> = ( - Option<&'a ck::following::Model>, - Option<&'a ck::following::Model>, - Option<&'a ck::blocking::Model>, - Option<&'a ck::blocking::Model>, - Option<&'a ck::muting::Model>, - Option<&'a ck::renote_muting::Model>, -); +struct UserRelationExtSource<'a> { + pub follow_out: Option<&'a ck::following::Model>, + pub follow_in: Option<&'a ck::following::Model>, + pub block_out: Option<&'a ck::blocking::Model>, + pub block_in: Option<&'a ck::blocking::Model>, + pub mute: Option<&'a ck::muting::Model>, + pub renote_mute: Option<&'a ck::renote_muting::Model>, +} impl PackType> for UserRelationExt { fn extract( context: &PackingContext, - (follow_out, follow_in, block_out, block_in, mute, renote_mute): UserRelationExtSource, + UserRelationExtSource { + follow_out, + follow_in, + block_out, + block_in, + mute, + renote_mute, + }: UserRelationExtSource, ) -> Self { let self_user = context.self_user(); diff --git a/src/model/mod.rs b/src/model/mod.rs index 031dbaf..a1c5e2d 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,12 +1,28 @@ +use crate::model::processing::PackResult; +use crate::service::MagnetarService; use magnetar_calckey_model::ck; use std::sync::Arc; pub mod data; +pub mod processing; + +#[derive(Clone, Debug)] +pub struct ProcessingLimits { + max_emojis: usize, +} + +impl Default for ProcessingLimits { + fn default() -> Self { + ProcessingLimits { max_emojis: 500 } + } +} #[derive(Clone, Debug)] pub struct PackingContext { - instance_info: Arc, + instance_meta: Arc, self_user: Option>, + service: Arc, + limits: ProcessingLimits, } pub trait PackType: 'static { @@ -14,6 +30,18 @@ pub trait PackType: 'static { } impl PackingContext { + pub async fn new( + service: Arc, + self_user: Option>, + ) -> PackResult { + Ok(Self { + instance_meta: service.instance_meta_cache.get().await?, + self_user, + service, + limits: Default::default(), + }) + } + fn self_user(&self) -> Option<&ck::user::Model> { self.self_user.as_deref() } diff --git a/src/model/processing/drive.rs b/src/model/processing/drive.rs new file mode 100644 index 0000000..2e383d0 --- /dev/null +++ b/src/model/processing/drive.rs @@ -0,0 +1,33 @@ +use crate::model::processing::PackResult; +use crate::model::{PackType, PackingContext}; +use magnetar_calckey_model::ck; +use magnetar_sdk::types::drive::{DriveFileBase, PackDriveFileBase}; +use magnetar_sdk::types::Id; +use magnetar_sdk::{Packed, Required}; + +pub struct DriveModel; + +impl DriveModel { + pub fn pack_existing( + &self, + ctx: &PackingContext, + file: &ck::drive_file::Model, + ) -> PackDriveFileBase { + PackDriveFileBase::pack_from(( + Required(Id::from(&file.id)), + Required(DriveFileBase::extract(ctx, &file)), + )) + } + + pub async fn get_cached_base( + &self, + ctx: &PackingContext, + id: &str, + ) -> PackResult> { + let Some(file) = ctx.service.drive_file_cache.get(id).await? else { + return Ok(None); + }; + + Ok(Some(self.pack_existing(ctx, file.as_ref()))) + } +} diff --git a/src/model/processing/emoji.rs b/src/model/processing/emoji.rs new file mode 100644 index 0000000..8bb7855 --- /dev/null +++ b/src/model/processing/emoji.rs @@ -0,0 +1,39 @@ +use crate::model::processing::PackResult; +use crate::model::{PackType, PackingContext}; +use itertools::Itertools; +use magnetar_calckey_model::ck; +use magnetar_sdk::types::emoji::{EmojiBase, PackEmojiBase}; +use magnetar_sdk::types::Id; +use magnetar_sdk::{Packed, Required}; + +pub struct EmojiModel; + +impl EmojiModel { + pub fn pack_existing(&self, ctx: &PackingContext, emoji: &ck::emoji::Model) -> PackEmojiBase { + PackEmojiBase::pack_from(( + Required(Id::from(&emoji.id)), + Required(EmojiBase::extract(ctx, &emoji)), + )) + } + + pub async fn fetch_many_emojis( + &self, + ctx: &PackingContext, + shortcodes: &[String], + host: Option<&str>, + ) -> PackResult> { + let emojis = ctx.service.emoji_cache.get_many(shortcodes, host).await?; + let packed_emojis = emojis.iter().map(|e| self.pack_existing(ctx, &e)).collect(); + + Ok(packed_emojis) + } + + pub fn deduplicate_emoji(&self, ctx: &PackingContext, emoji_list: Vec) -> Vec { + emoji_list + .into_iter() + .sorted() + .dedup() + .take(ctx.limits.max_emojis) + .collect::>() + } +} diff --git a/src/model/processing/mod.rs b/src/model/processing/mod.rs new file mode 100644 index 0000000..8d7af81 --- /dev/null +++ b/src/model/processing/mod.rs @@ -0,0 +1,42 @@ +use crate::service::emoji_cache::EmojiCacheError; +use crate::service::generic_id_cache::GenericIdCacheError; +use crate::service::instance_meta_cache::InstanceMetaCacheError; +use magnetar_calckey_model::sea_orm::DbErr; +use magnetar_calckey_model::CalckeyDbError; +use magnetar_sdk::mmm::Token; +use thiserror::Error; + +pub mod drive; +pub mod emoji; +pub mod user; + +#[derive(Debug, Error, strum::IntoStaticStr)] +pub enum PackError { + #[error("Database error: {0}")] + DbError(#[from] DbErr), + #[error("Calckey database wrapper error: {0}")] + CalckeyDbError(#[from] CalckeyDbError), + #[error("Emoji cache error: {0}")] + EmojiCacheError(#[from] EmojiCacheError), + #[error("Instance cache error: {0}")] + InstanceMetaCacheError(#[from] InstanceMetaCacheError), + #[error("Generic cache error: {0}")] + GenericCacheError(#[from] GenericIdCacheError), +} + +pub type PackResult = Result; + +fn get_mm_token_emoji(token: &Token) -> Vec { + let mut v = Vec::new(); + token.walk_map_collect( + &|t| { + if let Token::ShortcodeEmoji(e) = t { + Some(e.to_owned()) + } else { + None + } + }, + &mut v, + ); + v +} diff --git a/src/model/processing/user.rs b/src/model/processing/user.rs new file mode 100644 index 0000000..88ff5b5 --- /dev/null +++ b/src/model/processing/user.rs @@ -0,0 +1,55 @@ +use crate::model::data::user::UserBaseSource; +use crate::model::processing::emoji::EmojiModel; +use crate::model::processing::{get_mm_token_emoji, PackResult}; +use crate::model::{PackType, PackingContext}; +use magnetar_calckey_model::ck; +use magnetar_calckey_model::sea_orm::EntityTrait; +use magnetar_sdk::mmm::Token; +use magnetar_sdk::types::emoji::EmojiContext; +use magnetar_sdk::types::user::{PackUserBase, UserBase}; +use magnetar_sdk::types::{Id, MmXml}; +use magnetar_sdk::{mmm, Packed, Required}; +use std::sync::Arc; + +pub struct UserModel; + +impl UserModel { + pub fn tokenize_username(&self, user: &ck::user::Model) -> Token { + mmm::Context::default().parse_ui(user.name.as_deref().unwrap_or(&user.username)) + } + + pub async fn base_from_existing( + &self, + ctx: &PackingContext, + user: &ck::user::Model, + ) -> PackResult { + let avatar = match &user.avatar_id { + Some(av_id) => ctx.service.drive_file_cache.get(av_id).await?, + None => None, + }; + + let username_mm = self.tokenize_username(&user); + + let emoji_model = EmojiModel; + let shortcodes = emoji_model.deduplicate_emoji(ctx, get_mm_token_emoji(&username_mm)); + let emojis = emoji_model + .fetch_many_emojis(ctx, &shortcodes, user.host.as_deref()) + .await?; + let emoji_context = EmojiContext(emojis); + + let base = UserBase::extract( + ctx, + UserBaseSource { + user, + username_mm: mmm::to_xml_string(&username_mm).map(MmXml).as_ref().ok(), + avatar: avatar.as_deref(), + emoji_context: &emoji_context, + }, + ); + + Ok(PackUserBase::pack_from(( + Required(Id::from(&user.id)), + Required(base), + ))) + } +} diff --git a/src/service/emoji_cache.rs b/src/service/emoji_cache.rs new file mode 100644 index 0000000..772ea54 --- /dev/null +++ b/src/service/emoji_cache.rs @@ -0,0 +1,122 @@ +use crate::web::ApiError; +use lru::LruCache; +use magnetar_calckey_model::{ck, CalckeyDbError, CalckeyModel}; +use std::collections::HashSet; +use std::sync::Arc; +use strum::EnumVariantNames; +use thiserror::Error; +use tokio::sync::Mutex; + +#[derive(Debug, Error, EnumVariantNames)] +pub enum EmojiCacheError { + #[error("Database error: {0}")] + DbError(#[from] CalckeyDbError), +} + +impl From for ApiError { + fn from(err: EmojiCacheError) -> Self { + let mut api_error: ApiError = match err { + EmojiCacheError::DbError(err) => err.into(), + }; + + api_error.message = format!("Emoji cache error: {}", api_error.message); + + api_error + } +} + +#[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Hash)] +struct EmojiLocator { + name: String, + host: Option, +} + +pub struct EmojiCacheService { + cache: Mutex>>, + db: CalckeyModel, +} + +impl EmojiCacheService { + pub(super) fn new(db: CalckeyModel) -> Self { + const CACHE_SIZE: usize = 4096; + Self { + cache: Mutex::new(LruCache::new(CACHE_SIZE.try_into().unwrap())), + db, + } + } + + pub async fn get( + &self, + name: &str, + host: Option<&str>, + ) -> Result>, EmojiCacheError> { + let loc = EmojiLocator { + name: name.to_string(), + host: host.map(str::to_string), + }; + let mut read = self.cache.lock().await; + if let Some(emoji) = read.get(&loc) { + return Ok(Some(emoji.clone())); + } + drop(read); + + let emoji = self.db.fetch_emoji(name, host).await?; + + if emoji.is_none() { + return Ok(None); + } + + let mut write = self.cache.lock().await; + let emoji = Arc::new(emoji.unwrap()); + write.put(loc, emoji.clone()); + Ok(Some(emoji)) + } + + pub async fn get_many( + &self, + names: &[String], + host: Option<&str>, + ) -> Result>, EmojiCacheError> { + let locs = names + .into_iter() + .map(|n| EmojiLocator { + name: n.clone(), + host: host.map(str::to_string), + }) + .collect::>(); + let mut to_resolve = Vec::new(); + let mut resolved = Vec::new(); + + let mut read = self.cache.lock().await; + for loc in locs { + if let Some(emoji) = read.get(&loc) { + resolved.push(emoji.clone()); + } else { + to_resolve.push(loc.name); + } + } + drop(read); + + let emoji = self + .db + .fetch_many_emojis(&to_resolve, host) + .await? + .into_iter() + .map(Arc::new) + .collect::>(); + resolved.extend(emoji.iter().cloned()); + + let mut write = self.cache.lock().await; + emoji.iter().for_each(|e| { + write.put( + EmojiLocator { + name: e.name.clone(), + host: e.host.clone(), + }, + e.clone(), + ); + }); + + Ok(resolved) + } +} diff --git a/src/service/generic_id_cache.rs b/src/service/generic_id_cache.rs new file mode 100644 index 0000000..c4df02b --- /dev/null +++ b/src/service/generic_id_cache.rs @@ -0,0 +1,96 @@ +use crate::web::ApiError; +use lru::LruCache; +use magnetar_calckey_model::sea_orm::{EntityTrait, PrimaryKeyTrait}; +use magnetar_calckey_model::{CalckeyDbError, CalckeyModel}; +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use strum::EnumVariantNames; +use thiserror::Error; +use tokio::sync::Mutex; + +#[derive(Debug, Error, EnumVariantNames)] +pub enum GenericIdCacheError { + #[error("Database error: {0}")] + DbError(#[from] CalckeyDbError), +} + +impl From for ApiError { + fn from(err: GenericIdCacheError) -> Self { + let mut api_error: ApiError = match err { + GenericIdCacheError::DbError(err) => err.into(), + }; + + api_error.message = format!("Generic ID cache error: {}", api_error.message); + + api_error + } +} + +#[derive(Debug)] +struct CacheEntry { + created: Instant, + data: Arc, +} + +impl CacheEntry { + fn new(data: Arc) -> Self { + Self { + created: Instant::now(), + data, + } + } +} + +pub struct GenericIdCacheService { + cache: Mutex>>, + lifetime_max: Duration, + db: CalckeyModel, + _entity_type: PhantomData, +} + +impl GenericIdCacheService { + pub(super) fn new(db: CalckeyModel, cache_size: usize, entry_lifetime: Duration) -> Self { + const CACHE_SIZE: usize = 4096; + + Self { + cache: Mutex::new(LruCache::new( + cache_size + .try_into() + .unwrap_or(CACHE_SIZE.try_into().unwrap()), + )), + lifetime_max: entry_lifetime, + db, + _entity_type: PhantomData, + } + } + + pub async fn get<'a>(&self, id: &'a str) -> Result>, GenericIdCacheError> + where + <::PrimaryKey as PrimaryKeyTrait>::ValueType: From<&'a str>, + { + let mut read = self.cache.lock().await; + if let Some(item) = read.peek(id) { + if item.created + self.lifetime_max >= Instant::now() { + let data = item.data.clone(); + read.promote(id); + return Ok(Some(data)); + } + } + drop(read); + + let val = E::find_by_id(id) + .one(self.db.inner()) + .await + .map_err(CalckeyDbError::from)?; + + if val.is_none() { + return Ok(None); + } + + let mut write = self.cache.lock().await; + let data = Arc::new(val.unwrap()); + write.put(id.to_string(), CacheEntry::new(data.clone())); + Ok(Some(data)) + } +} diff --git a/src/service/instance_meta_cache.rs b/src/service/instance_meta_cache.rs new file mode 100644 index 0000000..437b4dd --- /dev/null +++ b/src/service/instance_meta_cache.rs @@ -0,0 +1,115 @@ +use crate::web::ApiError; +use magnetar_calckey_model::{ck, CalckeyDbError, CalckeyModel}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use strum::EnumVariantNames; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot}; +use tracing::error; + +#[derive(Debug, Error, EnumVariantNames)] +pub enum InstanceMetaCacheError { + #[error("Database error: {0}")] + DbError(#[from] CalckeyDbError), + #[error("Cache channel closed")] + ChannelClosed, +} + +impl From for ApiError { + fn from(err: InstanceMetaCacheError) -> Self { + let mut api_error: ApiError = match err { + InstanceMetaCacheError::DbError(err) => err.into(), + InstanceMetaCacheError::ChannelClosed => err.into(), + }; + + api_error.message = format!("Instance meta cache error: {}", api_error.message); + + api_error + } +} + +struct Entry { + value: Option>, + last_fetched: Option, +} + +#[derive(Copy, Clone)] +enum CacheRequest { + Get, + Fetch, +} + +type Callback = oneshot::Sender, InstanceMetaCacheError>>; + +struct InstanceMetaCache { + sender: mpsc::UnboundedSender<(CacheRequest, Callback)>, +} + +impl InstanceMetaCache { + fn new(db: CalckeyModel) -> Self { + const REFRESH_INTERVAL_SEC: u64 = 10; + let stale_threshold = Duration::from_secs(REFRESH_INTERVAL_SEC); + + let mut state = Entry { + value: None, + last_fetched: None, + }; + + let (req_tx, mut req_rx) = mpsc::unbounded_channel::<(CacheRequest, Callback)>(); + + tokio::spawn(async move { + while let Some((req, res_tx)) = req_rx.recv().await { + if let Some(val) = &state.value { + if state + .last_fetched + .is_some_and(|i| Instant::now() - i < stale_threshold) + && !matches!(req, CacheRequest::Fetch) + { + res_tx.send(Ok(val.clone())).ok(); + continue; + } + } + + let res = db.get_instance_meta().await.map(Arc::new); + + if let Ok(ref data) = res { + state.value = Some(data.clone()); + state.last_fetched = Some(Instant::now()); + } + + res_tx.send(res.map_err(CalckeyDbError::into)).ok(); + } + }); + + Self { sender: req_tx } + } + + async fn get(&self, req: CacheRequest) -> Result, InstanceMetaCacheError> { + let (tx, rx) = oneshot::channel(); + self.sender + .send((req, tx)) + .map_err(|_| InstanceMetaCacheError::ChannelClosed)?; + rx.await + .map_err(|_| InstanceMetaCacheError::ChannelClosed)? + } +} + +pub struct InstanceMetaCacheService { + cache: InstanceMetaCache, +} + +impl InstanceMetaCacheService { + pub(super) fn new(db: CalckeyModel) -> Self { + Self { + cache: InstanceMetaCache::new(db), + } + } + + pub async fn fetch(&self) -> Result, InstanceMetaCacheError> { + self.cache.get(CacheRequest::Fetch).await + } + + pub async fn get(&self) -> Result, InstanceMetaCacheError> { + self.cache.get(CacheRequest::Get).await + } +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 7a88653..e1f6670 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,7 +1,12 @@ -use magnetar_calckey_model::{CalckeyCache, CalckeyModel}; +use magnetar_calckey_model::{ck, CalckeyCache, CalckeyModel}; use magnetar_common::config::MagnetarConfig; +use std::fmt::{Debug, Formatter}; +use std::time::Duration; use thiserror::Error; +pub mod emoji_cache; +pub mod generic_id_cache; +pub mod instance_meta_cache; pub mod user_cache; pub struct MagnetarService { @@ -9,6 +14,19 @@ pub struct MagnetarService { pub cache: CalckeyCache, pub config: &'static MagnetarConfig, pub auth_cache: user_cache::UserCacheService, + pub instance_meta_cache: instance_meta_cache::InstanceMetaCacheService, + pub emoji_cache: emoji_cache::EmojiCacheService, + pub drive_file_cache: generic_id_cache::GenericIdCacheService, +} + +impl Debug for MagnetarService { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MagnetarService") + .field("db", &self.db) + .field("cache", &self.cache) + .field("config", &self.config) + .finish_non_exhaustive() + } } #[derive(Debug, Error)] @@ -25,12 +43,19 @@ impl MagnetarService { ) -> Result { let auth_cache = user_cache::UserCacheService::new(config, db.clone(), cache.clone()).await?; + let instance_meta_cache = instance_meta_cache::InstanceMetaCacheService::new(db.clone()); + let emoji_cache = emoji_cache::EmojiCacheService::new(db.clone()); + let drive_file_cache = + generic_id_cache::GenericIdCacheService::new(db.clone(), 128, Duration::from_secs(10)); Ok(Self { db, cache, config, auth_cache, + instance_meta_cache, + emoji_cache, + drive_file_cache, }) } } diff --git a/src/web/mod.rs b/src/web/mod.rs index bfc834e..a3df24d 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -1,3 +1,4 @@ +use crate::model::processing::PackError; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::Json; @@ -81,3 +82,40 @@ impl From for ApiError { } } } + +impl From for ApiError { + fn from(err: PackError) -> Self { + Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + code: err.error_code(), + message: if cfg!(debug_assertions) { + format!("Data transformation error: {}", err) + } else { + "Data transformation error".to_string() + }, + } + } +} + +#[derive(Debug)] +pub struct ObjectNotFound(pub String); + +impl From<&ObjectNotFound> for &str { + fn from(_: &ObjectNotFound) -> Self { + "ObjectNotFound" + } +} + +impl From for ApiError { + fn from(err: ObjectNotFound) -> Self { + Self { + status: StatusCode::NOT_FOUND, + code: err.error_code(), + message: if cfg!(debug_assertions) { + format!("Object not found: {}", err.0) + } else { + "Object not found".to_string() + }, + } + } +}