From f34be3a1048d35498041bebd819b3ffda205445d Mon Sep 17 00:00:00 2001 From: Natty Date: Thu, 26 Oct 2023 21:08:51 +0200 Subject: [PATCH 1/6] MMM: Profile field parsing and skipping Matrix handles --- magnetar_mmm_parser/src/lib.rs | 41 +++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/magnetar_mmm_parser/src/lib.rs b/magnetar_mmm_parser/src/lib.rs index 25dfbc2..52c5e84 100644 --- a/magnetar_mmm_parser/src/lib.rs +++ b/magnetar_mmm_parser/src/lib.rs @@ -635,6 +635,16 @@ impl Context { } } + pub fn parse_profile_fields(&self, input: &str) -> Token { + match self.inline_profile_fields(Span::new_extra(input, SpanMeta::default())) { + Ok((_, t)) => t.merged(), + Err(e) => { + trace!(input = input, "Profile field parser fail: {:?}", e); + Token::PlainText(e.to_compact_string()) + } + } + } + #[inline] fn partial( &self, @@ -666,6 +676,19 @@ impl Context { )(input) } + fn inline_profile_fields<'a>(&self, input: Span<'a>) -> IResult, Token> { + map( + many1(alt(( + self.partial(Self::unicode_emoji), + self.partial(Self::tag_mention), + self.partial(Self::tag_hashtag), + self.partial(Self::raw_url), + self.partial(Self::tag_raw_text), + ))), + Token::Sequence, + )(input) + } + fn inline_ui<'a>(&self, input: Span<'a>) -> IResult, Token> { map( many1(alt(( @@ -1437,6 +1460,7 @@ impl Context { return Ok((plain_out, Token::PlainText(plain.into()))); } + let start = input; let tags = one_of("@!"); let (input, mention_type) = map(tags, |c| match c { '@' => MentionType::User, @@ -1462,9 +1486,19 @@ impl Context { )(input)?; let host = host.map(|h| h.trim_end_matches(|c| matches!(c, '.' | '-' | '_'))); + let input = host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before); + + if let (input, Some(_)) = + peek(opt(tuple((recognize(tag(":")), alphanumeric1_unicode))))(input)? + { + return Ok(( + input, + Token::PlainText(start.up_to(&input).into_fragment().into()), + )); + } Ok(( - host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before), + input, Token::Mention { mention_type, name: name.into(), @@ -2145,6 +2179,11 @@ text"# Token::PlainText(" test".into()) ]) ); + + assert_eq!( + parse_full("@tag:domain.com"), + Token::PlainText("@tag:domain.com".into()) + ); } #[test] From 42e68fffcd713ef1a0fa2debaafe7d0906f71c87 Mon Sep 17 00:00:00 2001 From: Natty Date: Thu, 26 Oct 2023 21:23:59 +0200 Subject: [PATCH 2/6] MMM: Matrix handle parsing --- magnetar_mmm_parser/src/lib.rs | 55 ++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/magnetar_mmm_parser/src/lib.rs b/magnetar_mmm_parser/src/lib.rs index 52c5e84..fc9b06d 100644 --- a/magnetar_mmm_parser/src/lib.rs +++ b/magnetar_mmm_parser/src/lib.rs @@ -29,6 +29,7 @@ use unicode_segmentation::UnicodeSegmentation; pub enum MentionType { Community, User, + MatrixUser, } impl MentionType { @@ -36,6 +37,14 @@ impl MentionType { match self { MentionType::Community => '!', MentionType::User => '@', + MentionType::MatrixUser => ':', + } + } + + pub fn separator(&self) -> char { + match self { + MentionType::Community | MentionType::User => '@', + MentionType::MatrixUser => ':', } } } @@ -1460,7 +1469,6 @@ impl Context { return Ok((plain_out, Token::PlainText(plain.into()))); } - let start = input; let tags = one_of("@!"); let (input, mention_type) = map(tags, |c| match c { '@' => MentionType::User, @@ -1474,29 +1482,28 @@ impl Context { )(input)?; let before = input; - let (_, host) = map( - opt(tuple(( - tag("@"), - map( - recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))), - Span::into_fragment, - ), - ))), - |maybe_tag_host| maybe_tag_host.map(|(_, host)| host), - )(input)?; + let (_, host_opt) = opt(tuple(( + one_of(if matches!(mention_type, MentionType::User) { + "@:" + } else { + "@" + }), + map( + recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))), + Span::into_fragment, + ), + )))(input)?; - let host = host.map(|h| h.trim_end_matches(|c| matches!(c, '.' | '-' | '_'))); + // Promote tags with a colon separator to Matrix handles + let mention_type = if let Some((':', _)) = host_opt { + MentionType::MatrixUser + } else { + mention_type + }; + let host = + host_opt.map(|(_, name)| name.trim_end_matches(|c| matches!(c, '.' | '-' | '_'))); let input = host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before); - if let (input, Some(_)) = - peek(opt(tuple((recognize(tag(":")), alphanumeric1_unicode))))(input)? - { - return Ok(( - input, - Token::PlainText(start.up_to(&input).into_fragment().into()), - )); - } - Ok(( input, Token::Mention { @@ -2182,7 +2189,11 @@ text"# assert_eq!( parse_full("@tag:domain.com"), - Token::PlainText("@tag:domain.com".into()) + Token::Mention { + mention_type: crate::MentionType::MatrixUser, + name: "tag".into(), + host: Some("domain.com".into()) + }, ); } From fc86f0e29c06fdeb087d2aa762d5d1f06fe6cd26 Mon Sep 17 00:00:00 2001 From: Natty Date: Thu, 26 Oct 2023 21:37:15 +0200 Subject: [PATCH 3/6] Backend: Prep for MMM parser processing --- magnetar_sdk/src/types/mod.rs | 5 +++ magnetar_sdk/src/types/note.rs | 6 ++- magnetar_sdk/src/types/user.rs | 6 ++- src/model/data/note.rs | 13 ++++-- src/model/data/user.rs | 81 ++++++++++++++++++++++++---------- 5 files changed, 81 insertions(+), 30 deletions(-) diff --git a/magnetar_sdk/src/types/mod.rs b/magnetar_sdk/src/types/mod.rs index a3d7508..6920a5b 100644 --- a/magnetar_sdk/src/types/mod.rs +++ b/magnetar_sdk/src/types/mod.rs @@ -29,6 +29,11 @@ impl From<&str> for Id { } } +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[ts(export)] +#[repr(transparent)] +pub struct MmXml(String); + #[derive(Copy, Clone, Debug, Deserialize, Serialize, TS)] #[ts(export)] pub enum NotificationType { diff --git a/magnetar_sdk/src/types/note.rs b/magnetar_sdk/src/types/note.rs index e5e5dbb..2942600 100644 --- a/magnetar_sdk/src/types/note.rs +++ b/magnetar_sdk/src/types/note.rs @@ -1,6 +1,6 @@ use crate::types::emoji::EmojiContext; use crate::types::user::PackUserBase; -use crate::types::Id; +use crate::types::{Id, MmXml}; use crate::{Packed, Required}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -52,9 +52,11 @@ pack!(PackPollBase, Required as id & Required as poll); pub struct NoteBase { pub created_at: DateTime, pub cw: Option, + pub cw_mm: Option, pub uri: Option, pub url: Option, pub text: String, + pub text_mm: MmXml, pub visibility: NoteVisibility, pub user: Box, pub parent_note_id: Option, @@ -63,10 +65,10 @@ pub struct NoteBase { pub renote_count: u64, pub hashtags: Vec, pub reactions: Vec, - pub emojis: EmojiContext, pub local_only: bool, pub has_poll: bool, pub file_ids: Vec, + pub emojis: EmojiContext, } pack!(PackNoteBase, Required as id & Required as note); diff --git a/magnetar_sdk/src/types/user.rs b/magnetar_sdk/src/types/user.rs index 6ff7e53..851a04a 100644 --- a/magnetar_sdk/src/types/user.rs +++ b/magnetar_sdk/src/types/user.rs @@ -1,5 +1,5 @@ use crate::types::emoji::EmojiContext; -use crate::types::{Id, NotificationSettings}; +use crate::types::{Id, MmXml, NotificationSettings}; use crate::{Packed, Required}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -29,6 +29,7 @@ pub enum SpeechTransform { pub struct ProfileField { name: String, value: String, + value_mm: Option, verified_at: Option>, } @@ -38,6 +39,7 @@ pub struct UserBase { pub acct: String, pub username: String, pub display_name: String, + pub display_name_mm: MmXml, pub host: Option, pub speech_transform: SpeechTransform, pub created_at: DateTime, @@ -61,6 +63,7 @@ pub struct UserProfileExt { pub is_suspended: bool, pub description: Option, + pub description_mm: Option, pub location: Option, pub birthday: Option>, pub fields: Vec, @@ -79,6 +82,7 @@ pub struct UserProfileExt { pub banner_blurhash: Option, pub has_public_reactions: bool, + pub emojis: EmojiContext, } #[derive(Clone, Debug, Deserialize, Serialize, TS)] diff --git a/src/model/data/note.rs b/src/model/data/note.rs index f2fcf83..987566b 100644 --- a/src/model/data/note.rs +++ b/src/model/data/note.rs @@ -6,6 +6,7 @@ use magnetar_sdk::types::{ emoji::EmojiContext, note::{NoteAttachmentExt, NoteBase, NoteVisibility, PackReactionBase, ReactionBase}, user::UserBase, + MmXml, }; use magnetar_sdk::{Packed, Required}; @@ -27,9 +28,11 @@ impl PackType<&ck::note_reaction::Model> for ReactionBase { pub struct NoteBaseSource<'a> { pub note: &'a ck::note::Model, + pub cw_mm: Option<&'a MmXml>, + pub text_mm: &'a MmXml, pub reactions: &'a Vec, - pub emojis: &'a EmojiContext, pub user: &'a UserBase, + pub emoji_context: &'a EmojiContext, } impl PackType> for NoteBase { @@ -37,18 +40,22 @@ impl PackType> for NoteBase { _context: &PackingContext, NoteBaseSource { note, + text_mm, + cw_mm, reactions, - emojis, user, + emoji_context, }: NoteBaseSource<'_>, ) -> Self { use ck::sea_orm_active_enums::NoteVisibilityEnum as NVE; NoteBase { created_at: note.created_at.into(), cw: note.cw.clone(), + cw_mm: cw_mm.cloned(), uri: note.uri.clone(), url: note.url.clone(), text: note.text.clone().unwrap_or_default(), + text_mm: text_mm.clone(), visibility: match note.visibility { NVE::Followers => NoteVisibility::Followers, NVE::Hidden => NoteVisibility::Direct, @@ -66,10 +73,10 @@ impl PackType> for NoteBase { renote_count: note.renote_count as u64, hashtags: note.tags.clone(), reactions: reactions.clone(), - emojis: emojis.clone(), local_only: note.local_only, has_poll: note.has_poll, file_ids: note.file_ids.clone(), + emojis: emoji_context.clone(), } } } diff --git a/src/model/data/user.rs b/src/model/data/user.rs index 5975976..03f033c 100644 --- a/src/model/data/user.rs +++ b/src/model/data/user.rs @@ -3,9 +3,10 @@ use magnetar_calckey_model::ck::sea_orm_active_enums::UserProfileFfvisibilityEnu use magnetar_sdk::types::emoji::{EmojiContext, PackEmojiBase}; use magnetar_sdk::types::note::PackNoteFull; use magnetar_sdk::types::user::{ - AvatarDecoration, PackSecurityKeyBase, SecurityKeyBase, SpeechTransform, UserBase, - UserDetailExt, UserProfileExt, UserProfilePinsEx, UserRelationExt, UserSecretsExt, + AvatarDecoration, PackSecurityKeyBase, ProfileField, SecurityKeyBase, SpeechTransform, + UserBase, UserDetailExt, UserProfileExt, UserProfilePinsEx, UserRelationExt, UserSecretsExt, }; +use magnetar_sdk::types::MmXml; use crate::model::{PackType, PackingContext}; @@ -15,14 +16,23 @@ impl PackType<&[PackEmojiBase]> for EmojiContext { } } -type UserBaseSource<'a> = ( - &'a ck::user::Model, - &'a Option, - &'a EmojiContext, -); +struct UserBaseSource<'a> { + user: &'a ck::user::Model, + username_mm: &'a MmXml, + avatar: &'a Option, + emoji_context: &'a EmojiContext, +} impl PackType> for UserBase { - fn extract(_context: &PackingContext, (user, avatar, emoji_context): UserBaseSource) -> Self { + fn extract( + _context: &PackingContext, + UserBaseSource { + user, + username_mm, + avatar, + emoji_context, + }: UserBaseSource, + ) -> Self { UserBase { acct: user .host @@ -31,6 +41,7 @@ impl PackType> for UserBase { .unwrap_or_else(|| format!("@{}", user.username)), username: user.username.clone(), display_name: user.name.clone().unwrap_or_else(|| user.username.clone()), + display_name_mm: username_mm.clone(), host: user.host.clone(), speech_transform: if user.is_cat && user.speak_as_cat { SpeechTransform::Cat @@ -54,14 +65,27 @@ impl PackType> for UserBase { } } -type UserProfileExtSource<'a> = ( - &'a ck::user::Model, - &'a ck::user_profile::Model, - Option<&'a UserRelationExt>, -); +struct UserProfileExtSource<'a> { + user: &'a ck::user::Model, + profile: &'a ck::user_profile::Model, + profile_fields: &'a Vec, + description_mm: Option<&'a MmXml>, + relation: Option<&'a UserRelationExt>, + emoji_context: &'a EmojiContext, +} impl PackType> for UserProfileExt { - fn extract(context: &PackingContext, (user, profile, relation): UserProfileExtSource) -> Self { + fn extract( + context: &PackingContext, + UserProfileExtSource { + user, + profile, + profile_fields, + description_mm, + relation, + emoji_context, + }: UserProfileExtSource, + ) -> Self { let follow_visibility = match profile.ff_visibility { UserProfileFfvisibilityEnum::Public => true, UserProfileFfvisibilityEnum::Followers => relation.is_some_and(|r| r.follows_you), @@ -73,12 +97,13 @@ impl PackType> for UserProfileExt { is_silenced: user.is_silenced, is_suspended: user.is_suspended, description: profile.description.clone(), + description_mm: description_mm.cloned(), location: profile.location.clone(), birthday: profile .birthday .clone() .and_then(|b| b.parse().map_or_else(|_| None, Some)), - fields: serde_json::from_value(profile.fields.clone()).unwrap_or_else(|_| Vec::new()), + fields: profile_fields.clone(), follower_count: follow_visibility.then_some(user.followers_count as u64), following_count: follow_visibility.then_some(user.following_count as u64), note_count: Some(user.notes_count as u64), @@ -89,6 +114,7 @@ impl PackType> for UserProfileExt { banner_color: None, banner_blurhash: None, has_public_reactions: profile.public_reactions, + emojis: emoji_context.clone(), } } } @@ -103,19 +129,26 @@ impl PackType<&ck::user::Model> for UserDetailExt { } } -type UserRelationExtSource<'a> = ( - Option<&'a ck::following::Model>, - Option<&'a ck::following::Model>, - Option<&'a ck::blocking::Model>, - Option<&'a ck::blocking::Model>, - Option<&'a ck::muting::Model>, - Option<&'a ck::renote_muting::Model>, -); +struct UserRelationExtSource<'a> { + follow_out: Option<&'a ck::following::Model>, + follow_in: Option<&'a ck::following::Model>, + block_out: Option<&'a ck::blocking::Model>, + block_in: Option<&'a ck::blocking::Model>, + mute: Option<&'a ck::muting::Model>, + renote_mute: Option<&'a ck::renote_muting::Model>, +} impl PackType> for UserRelationExt { fn extract( context: &PackingContext, - (follow_out, follow_in, block_out, block_in, mute, renote_mute): UserRelationExtSource, + UserRelationExtSource { + follow_out, + follow_in, + block_out, + block_in, + mute, + renote_mute, + }: UserRelationExtSource, ) -> Self { let self_user = context.self_user(); From 18d526cf8c85fef15da17d532f04fbfc5863e13a Mon Sep 17 00:00:00 2001 From: Natty Date: Fri, 27 Oct 2023 01:41:48 +0200 Subject: [PATCH 4/6] Instance meta cache and initial pack processing for users --- ext_calckey_model/src/lib.rs | 18 +++++ magnetar_mmm_parser/src/lib.rs | 24 ++++++ magnetar_sdk/src/types/mod.rs | 10 ++- magnetar_sdk/src/types/user.rs | 2 +- src/model/data/note.rs | 2 +- src/model/data/user.rs | 38 +++++----- src/model/mod.rs | 3 +- src/model/processing/emoji.rs | 33 +++++++++ src/model/processing/mod.rs | 32 ++++++++ src/model/processing/user.rs | 51 +++++++++++++ src/service/instance_meta_cache.rs | 115 +++++++++++++++++++++++++++++ src/service/mod.rs | 4 + 12 files changed, 306 insertions(+), 26 deletions(-) create mode 100644 src/model/processing/emoji.rs create mode 100644 src/model/processing/mod.rs create mode 100644 src/model/processing/user.rs create mode 100644 src/service/instance_meta_cache.rs diff --git a/ext_calckey_model/src/lib.rs b/ext_calckey_model/src/lib.rs index 1fb0704..40fdb1e 100644 --- a/ext_calckey_model/src/lib.rs +++ b/ext_calckey_model/src/lib.rs @@ -1,5 +1,6 @@ pub use ck; use ck::*; +pub use sea_orm; use chrono::Utc; use ext_calckey_model_migration::{Migrator, MigratorTrait}; @@ -124,6 +125,23 @@ impl CalckeyModel { .await?) } + pub async fn fetch_many_emojis( + &self, + shortcodes: &[String], + host: Option<&str>, + ) -> Result, CalckeyDbError> { + let host_filter = if let Some(host) = host { + emoji::Column::Host.eq(host) + } else { + emoji::Column::Host.is_null() + }; + + let name_filter = emoji::Column::Name.is_in(shortcodes); + let filter = host_filter.and(name_filter); + + Ok(emoji::Entity::find().filter(filter).all(&self.0).await?) + } + pub async fn get_access_token( &self, token: &str, diff --git a/magnetar_mmm_parser/src/lib.rs b/magnetar_mmm_parser/src/lib.rs index fc9b06d..3cf6da8 100644 --- a/magnetar_mmm_parser/src/lib.rs +++ b/magnetar_mmm_parser/src/lib.rs @@ -233,6 +233,30 @@ impl Token { } } + pub fn walk_map_collect(&self, func: impl Fn(&Token) -> Option, out: &mut Vec) { + if let Some(v) = func(self) { + out.push(v) + } + + match self { + Token::Sequence(items) => { + items + .iter() + .for_each(|tok| tok.walk_map_collect(&func, out)); + } + Token::Quote(inner) + | Token::Small(inner) + | Token::BoldItalic(inner) + | Token::Bold(inner) + | Token::Italic(inner) + | Token::Center(inner) + | Token::Function { inner, .. } + | Token::Link { label: inner, .. } + | Token::Strikethrough(inner) => inner.walk_map_collect(func, out), + _ => {} + } + } + fn write(&self, writer: &mut quick_xml::Writer) -> quick_xml::Result<()> { match self { Token::PlainText(plain) => { diff --git a/magnetar_sdk/src/types/mod.rs b/magnetar_sdk/src/types/mod.rs index 6920a5b..215c045 100644 --- a/magnetar_sdk/src/types/mod.rs +++ b/magnetar_sdk/src/types/mod.rs @@ -23,16 +23,18 @@ pub struct Id { pub id: String, } -impl From<&str> for Id { - fn from(id: &str) -> Self { - Self { id: id.to_string() } +impl> From for Id { + fn from(id: T) -> Self { + Self { + id: id.as_ref().to_string(), + } } } #[derive(Clone, Debug, Deserialize, Serialize, TS)] #[ts(export)] #[repr(transparent)] -pub struct MmXml(String); +pub struct MmXml(pub String); #[derive(Copy, Clone, Debug, Deserialize, Serialize, TS)] #[ts(export)] diff --git a/magnetar_sdk/src/types/user.rs b/magnetar_sdk/src/types/user.rs index 851a04a..5a4f8d4 100644 --- a/magnetar_sdk/src/types/user.rs +++ b/magnetar_sdk/src/types/user.rs @@ -39,7 +39,7 @@ pub struct UserBase { pub acct: String, pub username: String, pub display_name: String, - pub display_name_mm: MmXml, + pub display_name_mm: Option, pub host: Option, pub speech_transform: SpeechTransform, pub created_at: DateTime, diff --git a/src/model/data/note.rs b/src/model/data/note.rs index 987566b..147bb78 100644 --- a/src/model/data/note.rs +++ b/src/model/data/note.rs @@ -19,7 +19,7 @@ impl PackType<&ck::note_reaction::Model> for ReactionBase { user_id: reaction.user_id.clone(), reaction: Reaction::guess_from( &reaction.reaction, - &context.instance_info.default_reaction, + &context.instance_meta.default_reaction, ) .unwrap_or_else(|| /* Shouldn't happen */ Reaction::Unicode("👍".to_string())), } diff --git a/src/model/data/user.rs b/src/model/data/user.rs index 03f033c..2a9edcf 100644 --- a/src/model/data/user.rs +++ b/src/model/data/user.rs @@ -16,11 +16,11 @@ impl PackType<&[PackEmojiBase]> for EmojiContext { } } -struct UserBaseSource<'a> { - user: &'a ck::user::Model, - username_mm: &'a MmXml, - avatar: &'a Option, - emoji_context: &'a EmojiContext, +pub struct UserBaseSource<'a> { + pub user: &'a ck::user::Model, + pub username_mm: Option<&'a MmXml>, + pub avatar: &'a Option, + pub emoji_context: &'a EmojiContext, } impl PackType> for UserBase { @@ -41,7 +41,7 @@ impl PackType> for UserBase { .unwrap_or_else(|| format!("@{}", user.username)), username: user.username.clone(), display_name: user.name.clone().unwrap_or_else(|| user.username.clone()), - display_name_mm: username_mm.clone(), + display_name_mm: username_mm.cloned(), host: user.host.clone(), speech_transform: if user.is_cat && user.speak_as_cat { SpeechTransform::Cat @@ -65,13 +65,13 @@ impl PackType> for UserBase { } } -struct UserProfileExtSource<'a> { - user: &'a ck::user::Model, - profile: &'a ck::user_profile::Model, - profile_fields: &'a Vec, - description_mm: Option<&'a MmXml>, - relation: Option<&'a UserRelationExt>, - emoji_context: &'a EmojiContext, +pub struct UserProfileExtSource<'a> { + pub user: &'a ck::user::Model, + pub profile: &'a ck::user_profile::Model, + pub profile_fields: &'a Vec, + pub description_mm: Option<&'a MmXml>, + pub relation: Option<&'a UserRelationExt>, + pub emoji_context: &'a EmojiContext, } impl PackType> for UserProfileExt { @@ -130,12 +130,12 @@ impl PackType<&ck::user::Model> for UserDetailExt { } struct UserRelationExtSource<'a> { - follow_out: Option<&'a ck::following::Model>, - follow_in: Option<&'a ck::following::Model>, - block_out: Option<&'a ck::blocking::Model>, - block_in: Option<&'a ck::blocking::Model>, - mute: Option<&'a ck::muting::Model>, - renote_mute: Option<&'a ck::renote_muting::Model>, + pub follow_out: Option<&'a ck::following::Model>, + pub follow_in: Option<&'a ck::following::Model>, + pub block_out: Option<&'a ck::blocking::Model>, + pub block_in: Option<&'a ck::blocking::Model>, + pub mute: Option<&'a ck::muting::Model>, + pub renote_mute: Option<&'a ck::renote_muting::Model>, } impl PackType> for UserRelationExt { diff --git a/src/model/mod.rs b/src/model/mod.rs index 031dbaf..63f9486 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -2,10 +2,11 @@ use magnetar_calckey_model::ck; use std::sync::Arc; pub mod data; +pub mod processing; #[derive(Clone, Debug)] pub struct PackingContext { - instance_info: Arc, + instance_meta: Arc, self_user: Option>, } diff --git a/src/model/processing/emoji.rs b/src/model/processing/emoji.rs new file mode 100644 index 0000000..e99ad69 --- /dev/null +++ b/src/model/processing/emoji.rs @@ -0,0 +1,33 @@ +use crate::model::processing::PackResult; +use crate::model::{PackType, PackingContext}; +use magnetar_calckey_model::{ck, CalckeyModel}; +use magnetar_sdk::types::emoji::{EmojiBase, PackEmojiBase}; +use magnetar_sdk::types::Id; +use magnetar_sdk::{Packed, Required}; + +pub struct EmojiModel(CalckeyModel); + +impl EmojiModel { + pub fn new(model: CalckeyModel) -> Self { + EmojiModel(model) + } + + pub fn pack_existing(&self, ctx: &PackingContext, emoji: &ck::emoji::Model) -> PackEmojiBase { + PackEmojiBase::pack_from(( + Required(Id::from(&emoji.id)), + Required(EmojiBase::extract(ctx, &emoji)), + )) + } + + pub async fn fetch_many_emojis( + &self, + ctx: &PackingContext, + shortcodes: &[String], + host: Option<&str>, + ) -> PackResult> { + let emojis = self.0.fetch_many_emojis(shortcodes, host).await?; + let packed_emojis = emojis.iter().map(|e| self.pack_existing(ctx, &e)).collect(); + + Ok(packed_emojis) + } +} diff --git a/src/model/processing/mod.rs b/src/model/processing/mod.rs new file mode 100644 index 0000000..f43b528 --- /dev/null +++ b/src/model/processing/mod.rs @@ -0,0 +1,32 @@ +use magnetar_calckey_model::sea_orm::DbErr; +use magnetar_calckey_model::CalckeyDbError; +use magnetar_sdk::mmm::Token; +use thiserror::Error; + +pub mod emoji; +pub mod user; + +#[derive(Debug, Error)] +pub enum PackError { + #[error("Database error: {0}")] + DbError(#[from] DbErr), + #[error("Calckey database wrapper error: {0}")] + CalckeyDbError(#[from] CalckeyDbError), +} + +pub type PackResult = Result; + +fn get_mm_token_emoji(token: &Token) -> Vec { + let mut v = Vec::new(); + token.walk_map_collect( + |t| { + if let Token::ShortcodeEmoji(e) = t { + Some(e.to_owned()) + } else { + None + } + }, + &mut v, + ); + v +} diff --git a/src/model/processing/user.rs b/src/model/processing/user.rs new file mode 100644 index 0000000..f258005 --- /dev/null +++ b/src/model/processing/user.rs @@ -0,0 +1,51 @@ +use crate::model::data::user::UserBaseSource; +use crate::model::processing::emoji::EmojiModel; +use crate::model::processing::{get_mm_token_emoji, PackResult}; +use crate::model::{PackType, PackingContext}; +use magnetar_calckey_model::sea_orm::EntityTrait; +use magnetar_calckey_model::{ck, CalckeyModel}; +use magnetar_sdk::types::emoji::EmojiContext; +use magnetar_sdk::types::user::{PackUserBase, UserBase}; +use magnetar_sdk::types::{Id, MmXml}; +use magnetar_sdk::{mmm, Packed, Required}; + +pub struct UserModel(CalckeyModel); + +impl UserModel { + pub async fn base_from_existing( + &self, + ctx: &PackingContext, + user: &ck::user::Model, + ) -> PackResult { + let avatar = if let Some(avatar_id) = user.avatar_id.as_ref() { + ck::drive_file::Entity::find_by_id(avatar_id) + .one(self.0.inner()) + .await? + } else { + None + }; + + let username_mm = + mmm::Context::default().parse_ui(user.name.as_deref().unwrap_or(&user.username)); + let emoji_model = EmojiModel::new(self.0.clone()); + let emojis = emoji_model + .fetch_many_emojis(ctx, &get_mm_token_emoji(&username_mm), user.host.as_deref()) + .await?; + let emoji_context = EmojiContext(emojis); + + let base = UserBase::extract( + ctx, + UserBaseSource { + user, + username_mm: mmm::to_xml_string(&username_mm).map(MmXml).as_ref().ok(), + avatar: &avatar, + emoji_context: &emoji_context, + }, + ); + + Ok(PackUserBase::pack_from(( + Required(Id::from(&user.id)), + Required(base), + ))) + } +} diff --git a/src/service/instance_meta_cache.rs b/src/service/instance_meta_cache.rs new file mode 100644 index 0000000..437b4dd --- /dev/null +++ b/src/service/instance_meta_cache.rs @@ -0,0 +1,115 @@ +use crate::web::ApiError; +use magnetar_calckey_model::{ck, CalckeyDbError, CalckeyModel}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use strum::EnumVariantNames; +use thiserror::Error; +use tokio::sync::{mpsc, oneshot}; +use tracing::error; + +#[derive(Debug, Error, EnumVariantNames)] +pub enum InstanceMetaCacheError { + #[error("Database error: {0}")] + DbError(#[from] CalckeyDbError), + #[error("Cache channel closed")] + ChannelClosed, +} + +impl From for ApiError { + fn from(err: InstanceMetaCacheError) -> Self { + let mut api_error: ApiError = match err { + InstanceMetaCacheError::DbError(err) => err.into(), + InstanceMetaCacheError::ChannelClosed => err.into(), + }; + + api_error.message = format!("Instance meta cache error: {}", api_error.message); + + api_error + } +} + +struct Entry { + value: Option>, + last_fetched: Option, +} + +#[derive(Copy, Clone)] +enum CacheRequest { + Get, + Fetch, +} + +type Callback = oneshot::Sender, InstanceMetaCacheError>>; + +struct InstanceMetaCache { + sender: mpsc::UnboundedSender<(CacheRequest, Callback)>, +} + +impl InstanceMetaCache { + fn new(db: CalckeyModel) -> Self { + const REFRESH_INTERVAL_SEC: u64 = 10; + let stale_threshold = Duration::from_secs(REFRESH_INTERVAL_SEC); + + let mut state = Entry { + value: None, + last_fetched: None, + }; + + let (req_tx, mut req_rx) = mpsc::unbounded_channel::<(CacheRequest, Callback)>(); + + tokio::spawn(async move { + while let Some((req, res_tx)) = req_rx.recv().await { + if let Some(val) = &state.value { + if state + .last_fetched + .is_some_and(|i| Instant::now() - i < stale_threshold) + && !matches!(req, CacheRequest::Fetch) + { + res_tx.send(Ok(val.clone())).ok(); + continue; + } + } + + let res = db.get_instance_meta().await.map(Arc::new); + + if let Ok(ref data) = res { + state.value = Some(data.clone()); + state.last_fetched = Some(Instant::now()); + } + + res_tx.send(res.map_err(CalckeyDbError::into)).ok(); + } + }); + + Self { sender: req_tx } + } + + async fn get(&self, req: CacheRequest) -> Result, InstanceMetaCacheError> { + let (tx, rx) = oneshot::channel(); + self.sender + .send((req, tx)) + .map_err(|_| InstanceMetaCacheError::ChannelClosed)?; + rx.await + .map_err(|_| InstanceMetaCacheError::ChannelClosed)? + } +} + +pub struct InstanceMetaCacheService { + cache: InstanceMetaCache, +} + +impl InstanceMetaCacheService { + pub(super) fn new(db: CalckeyModel) -> Self { + Self { + cache: InstanceMetaCache::new(db), + } + } + + pub async fn fetch(&self) -> Result, InstanceMetaCacheError> { + self.cache.get(CacheRequest::Fetch).await + } + + pub async fn get(&self) -> Result, InstanceMetaCacheError> { + self.cache.get(CacheRequest::Get).await + } +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 7a88653..027cae4 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -2,6 +2,7 @@ use magnetar_calckey_model::{CalckeyCache, CalckeyModel}; use magnetar_common::config::MagnetarConfig; use thiserror::Error; +pub mod instance_meta_cache; pub mod user_cache; pub struct MagnetarService { @@ -9,6 +10,7 @@ pub struct MagnetarService { pub cache: CalckeyCache, pub config: &'static MagnetarConfig, pub auth_cache: user_cache::UserCacheService, + pub instance_meta_cache: instance_meta_cache::InstanceMetaCacheService, } #[derive(Debug, Error)] @@ -25,12 +27,14 @@ impl MagnetarService { ) -> Result { let auth_cache = user_cache::UserCacheService::new(config, db.clone(), cache.clone()).await?; + let instance_meta_cache = instance_meta_cache::InstanceMetaCacheService::new(db.clone()); Ok(Self { db, cache, config, auth_cache, + instance_meta_cache, }) } } From 557269551539cc5ba857979c6d4071e4482f0bc3 Mon Sep 17 00:00:00 2001 From: Natty Date: Fri, 27 Oct 2023 21:55:08 +0200 Subject: [PATCH 5/6] Cached emoji resolution --- Cargo.lock | 117 +++++++++++++++++++++++++++++++- Cargo.toml | 8 ++- ext_calckey_model/src/lib.rs | 16 +++++ src/model/mod.rs | 14 ++++ src/model/processing/emoji.rs | 10 +-- src/model/processing/mod.rs | 3 + src/model/processing/user.rs | 18 +++-- src/service/emoji_cache.rs | 123 ++++++++++++++++++++++++++++++++++ src/service/mod.rs | 15 +++++ 9 files changed, 309 insertions(+), 15 deletions(-) create mode 100644 src/service/emoji_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 91a74f1..c8158f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -263,6 +263,15 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "backtrace-ext" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "537beee3be4a18fb023b570f80e3ae28003db9167a751266b259926e25539d50" +dependencies = [ + "backtrace", +] + [[package]] name = "base64" version = "0.13.1" @@ -1333,6 +1342,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "is_ci" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "616cde7c720bb2bb5824a224687d8f77bfd38922027f01d825cd7453be5099fb" + [[package]] name = "itertools" version = "0.10.5" @@ -1342,6 +1357,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.9" @@ -1411,6 +1435,15 @@ version = "0.4.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +[[package]] +name = "lru" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efa59af2ddfad1854ae27d75009d538d0998b4b2fd47083e743ac1a10e46c60" +dependencies = [ + "hashbrown 0.14.0", +] + [[package]] name = "magnetar" version = "0.2.1-alpha" @@ -1422,6 +1455,8 @@ dependencies = [ "dotenvy", "headers", "hyper", + "itertools 0.11.0", + "lru", "magnetar_calckey_model", "magnetar_common", "magnetar_core", @@ -1599,8 +1634,17 @@ version = "5.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59bb584eaeeab6bd0226ccf3509a69d7936d148cf3d036ad350abe35e8c6856e" dependencies = [ + "backtrace", + "backtrace-ext", + "is-terminal", "miette-derive", "once_cell", + "owo-colors", + "supports-color", + "supports-hyperlinks", + "supports-unicode", + "terminal_size", + "textwrap", "thiserror", "unicode-width", ] @@ -1812,6 +1856,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + [[package]] name = "parking_lot" version = "0.12.1" @@ -2702,6 +2752,12 @@ version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +[[package]] +name = "smawk" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f67ad224767faa3c7d8b6d91985b78e70a1324408abcb1cfcc2be4c06bc06043" + [[package]] name = "socket2" version = "0.4.9" @@ -2743,7 +2799,7 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c12bc9199d1db8234678b7051747c07f517cdcf019262d1847b94ec8b1aee3e" dependencies = [ - "itertools", + "itertools 0.10.5", "nom", "unicode_categories", ] @@ -3015,6 +3071,34 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +[[package]] +name = "supports-color" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4950e7174bffabe99455511c39707310e7e9b440364a2fcb1cc21521be57b354" +dependencies = [ + "is-terminal", + "is_ci", +] + +[[package]] +name = "supports-hyperlinks" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f84231692eb0d4d41e4cdd0cabfdd2e6cd9e255e65f80c9aa7c98dd502b4233d" +dependencies = [ + "is-terminal", +] + +[[package]] +name = "supports-unicode" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6c2cb240ab5dd21ed4906895ee23fe5a48acdbd15a3ce388e7b62a9b66baf7" +dependencies = [ + "is-terminal", +] + [[package]] name = "syn" version = "1.0.109" @@ -3093,6 +3177,27 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "terminal_size" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633c1a546cee861a1a6d0dc69ebeca693bf4296661ba7852b9d21d159e0506df" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "textwrap" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7b3e525a49ec206798b40326a44121291b530c963cfb01018f63e135bac543d" +dependencies = [ + "smawk", + "unicode-linebreak", + "unicode-width", +] + [[package]] name = "thiserror" version = "1.0.44" @@ -3514,6 +3619,16 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" +[[package]] +name = "unicode-linebreak" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5faade31a542b8b35855fff6e8def199853b2da8da256da52f52f1316ee3137" +dependencies = [ + "hashbrown 0.12.3", + "regex", +] + [[package]] name = "unicode-normalization" version = "0.1.22" diff --git a/Cargo.toml b/Cargo.toml index 05886ae..172b55b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,8 +37,10 @@ futures-util = "0.3" headers = "0.3" http = "0.2" hyper = "0.14" +itertools = "0.11" js-sys = "0.3" log = "0.4" +lru = "0.12" miette = "5.9" nom = "7" nom_locate = "4" @@ -78,6 +80,8 @@ magnetar_calckey_model = { path = "./ext_calckey_model" } magnetar_sdk = { path = "./magnetar_sdk" } cached = { workspace = true } +lru = { workspace = true } + chrono = { workspace = true } dotenvy = { workspace = true } @@ -93,9 +97,11 @@ tracing = { workspace = true } cfg-if = { workspace = true } +itertools = { workspace = true } + strum = { workspace = true, features = ["derive"] } thiserror = { workspace = true } -miette = { workspace = true } +miette = { workspace = true, features = ["fancy"] } percent-encoding = { workspace = true } diff --git a/ext_calckey_model/src/lib.rs b/ext_calckey_model/src/lib.rs index 40fdb1e..6f8fdc3 100644 --- a/ext_calckey_model/src/lib.rs +++ b/ext_calckey_model/src/lib.rs @@ -125,6 +125,22 @@ impl CalckeyModel { .await?) } + pub async fn fetch_emoji( + &self, + shortcode: &str, + host: Option<&str>, + ) -> Result, CalckeyDbError> { + let host_filter = if let Some(host) = host { + emoji::Column::Host.eq(host) + } else { + emoji::Column::Host.is_null() + }; + + let name_filter = emoji::Column::Name.eq(shortcode); + let filter = host_filter.and(name_filter); + + Ok(emoji::Entity::find().filter(filter).one(&self.0).await?) + } pub async fn fetch_many_emojis( &self, shortcodes: &[String], diff --git a/src/model/mod.rs b/src/model/mod.rs index 63f9486..e54d704 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,13 +1,27 @@ +use crate::service::MagnetarService; use magnetar_calckey_model::ck; use std::sync::Arc; pub mod data; pub mod processing; +#[derive(Clone, Debug)] +pub struct ProcessingLimits { + max_emojis: usize, +} + +impl Default for ProcessingLimits { + fn default() -> Self { + ProcessingLimits { max_emojis: 500 } + } +} + #[derive(Clone, Debug)] pub struct PackingContext { instance_meta: Arc, self_user: Option>, + service: Arc, + limits: ProcessingLimits, } pub trait PackType: 'static { diff --git a/src/model/processing/emoji.rs b/src/model/processing/emoji.rs index e99ad69..9c47281 100644 --- a/src/model/processing/emoji.rs +++ b/src/model/processing/emoji.rs @@ -1,17 +1,13 @@ use crate::model::processing::PackResult; use crate::model::{PackType, PackingContext}; -use magnetar_calckey_model::{ck, CalckeyModel}; +use magnetar_calckey_model::ck; use magnetar_sdk::types::emoji::{EmojiBase, PackEmojiBase}; use magnetar_sdk::types::Id; use magnetar_sdk::{Packed, Required}; -pub struct EmojiModel(CalckeyModel); +pub struct EmojiModel; impl EmojiModel { - pub fn new(model: CalckeyModel) -> Self { - EmojiModel(model) - } - pub fn pack_existing(&self, ctx: &PackingContext, emoji: &ck::emoji::Model) -> PackEmojiBase { PackEmojiBase::pack_from(( Required(Id::from(&emoji.id)), @@ -25,7 +21,7 @@ impl EmojiModel { shortcodes: &[String], host: Option<&str>, ) -> PackResult> { - let emojis = self.0.fetch_many_emojis(shortcodes, host).await?; + let emojis = ctx.service.emoji_cache.get_many(shortcodes, host).await?; let packed_emojis = emojis.iter().map(|e| self.pack_existing(ctx, &e)).collect(); Ok(packed_emojis) diff --git a/src/model/processing/mod.rs b/src/model/processing/mod.rs index f43b528..35c9658 100644 --- a/src/model/processing/mod.rs +++ b/src/model/processing/mod.rs @@ -1,3 +1,4 @@ +use crate::service::emoji_cache::EmojiCacheError; use magnetar_calckey_model::sea_orm::DbErr; use magnetar_calckey_model::CalckeyDbError; use magnetar_sdk::mmm::Token; @@ -12,6 +13,8 @@ pub enum PackError { DbError(#[from] DbErr), #[error("Calckey database wrapper error: {0}")] CalckeyDbError(#[from] CalckeyDbError), + #[error("Emoji cache error: {0}")] + EmojiCacheError(#[from] EmojiCacheError), } pub type PackResult = Result; diff --git a/src/model/processing/user.rs b/src/model/processing/user.rs index f258005..c383fed 100644 --- a/src/model/processing/user.rs +++ b/src/model/processing/user.rs @@ -2,14 +2,15 @@ use crate::model::data::user::UserBaseSource; use crate::model::processing::emoji::EmojiModel; use crate::model::processing::{get_mm_token_emoji, PackResult}; use crate::model::{PackType, PackingContext}; +use itertools::Itertools; +use magnetar_calckey_model::ck; use magnetar_calckey_model::sea_orm::EntityTrait; -use magnetar_calckey_model::{ck, CalckeyModel}; use magnetar_sdk::types::emoji::EmojiContext; use magnetar_sdk::types::user::{PackUserBase, UserBase}; use magnetar_sdk::types::{Id, MmXml}; use magnetar_sdk::{mmm, Packed, Required}; -pub struct UserModel(CalckeyModel); +pub struct UserModel; impl UserModel { pub async fn base_from_existing( @@ -19,7 +20,7 @@ impl UserModel { ) -> PackResult { let avatar = if let Some(avatar_id) = user.avatar_id.as_ref() { ck::drive_file::Entity::find_by_id(avatar_id) - .one(self.0.inner()) + .one(ctx.service.db.inner()) .await? } else { None @@ -27,9 +28,14 @@ impl UserModel { let username_mm = mmm::Context::default().parse_ui(user.name.as_deref().unwrap_or(&user.username)); - let emoji_model = EmojiModel::new(self.0.clone()); - let emojis = emoji_model - .fetch_many_emojis(ctx, &get_mm_token_emoji(&username_mm), user.host.as_deref()) + let shortcodes = get_mm_token_emoji(&username_mm) + .into_iter() + .sorted() + .dedup() + .take(ctx.limits.max_emojis) + .collect::>(); + let emojis = EmojiModel + .fetch_many_emojis(ctx, &shortcodes, user.host.as_deref()) .await?; let emoji_context = EmojiContext(emojis); diff --git a/src/service/emoji_cache.rs b/src/service/emoji_cache.rs new file mode 100644 index 0000000..96c5c47 --- /dev/null +++ b/src/service/emoji_cache.rs @@ -0,0 +1,123 @@ +use crate::web::ApiError; +use lru::LruCache; +use magnetar_calckey_model::{ck, CalckeyDbError, CalckeyModel}; +use std::collections::HashSet; +use std::sync::Arc; +use strum::EnumVariantNames; +use thiserror::Error; +use tokio::io::AsyncWriteExt; +use tokio::sync::Mutex; + +#[derive(Debug, Error, EnumVariantNames)] +pub enum EmojiCacheError { + #[error("Database error: {0}")] + DbError(#[from] CalckeyDbError), +} + +impl From for ApiError { + fn from(err: EmojiCacheError) -> Self { + let mut api_error: ApiError = match err { + EmojiCacheError::DbError(err) => err.into(), + }; + + api_error.message = format!("Emoji cache error: {}", api_error.message); + + api_error + } +} + +#[derive(Clone, Debug, Ord, PartialOrd, Eq, PartialEq, Hash)] +struct EmojiLocator { + name: String, + host: Option, +} + +pub struct EmojiCacheService { + cache: Mutex>>, + db: CalckeyModel, +} + +impl EmojiCacheService { + pub(super) fn new(db: CalckeyModel) -> Self { + const CACHE_SIZE: usize = 4096; + Self { + cache: Mutex::new(LruCache::new(CACHE_SIZE.try_into().unwrap())), + db, + } + } + + pub async fn get( + &self, + name: &str, + host: Option<&str>, + ) -> Result>, EmojiCacheError> { + let loc = EmojiLocator { + name: name.to_string(), + host: host.map(str::to_string), + }; + let mut read = self.cache.lock().await; + if let Some(emoji) = read.get(&loc) { + return Ok(Some(emoji.clone())); + } + drop(read); + + let emoji = self.db.fetch_emoji(name, host).await?; + + if emoji.is_none() { + return Ok(None); + } + + let mut write = self.cache.lock().await; + let emoji = Arc::new(emoji.unwrap()); + write.put(loc, emoji.clone()); + Ok(Some(emoji)) + } + + pub async fn get_many( + &self, + names: &[String], + host: Option<&str>, + ) -> Result>, EmojiCacheError> { + let locs = names + .into_iter() + .map(|n| EmojiLocator { + name: n.clone(), + host: host.map(str::to_string), + }) + .collect::>(); + let mut to_resolve = Vec::new(); + let mut resolved = Vec::new(); + + let mut read = self.cache.lock().await; + for loc in locs { + if let Some(emoji) = read.get(&loc) { + resolved.push(emoji.clone()); + } else { + to_resolve.push(loc.name); + } + } + drop(read); + + let emoji = self + .db + .fetch_many_emojis(&to_resolve, host) + .await? + .into_iter() + .map(Arc::new) + .collect::>(); + resolved.extend(emoji.iter().cloned()); + + let mut write = self.cache.lock().await; + emoji.iter().for_each(|e| { + write.put( + EmojiLocator { + name: e.name.clone(), + host: e.host.clone(), + }, + e.clone(), + ); + }); + + Ok(resolved) + } +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 027cae4..dcbd11e 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,7 +1,9 @@ use magnetar_calckey_model::{CalckeyCache, CalckeyModel}; use magnetar_common::config::MagnetarConfig; +use std::fmt::{Debug, Formatter}; use thiserror::Error; +pub mod emoji_cache; pub mod instance_meta_cache; pub mod user_cache; @@ -11,6 +13,17 @@ pub struct MagnetarService { pub config: &'static MagnetarConfig, pub auth_cache: user_cache::UserCacheService, pub instance_meta_cache: instance_meta_cache::InstanceMetaCacheService, + pub emoji_cache: emoji_cache::EmojiCacheService, +} + +impl Debug for MagnetarService { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MagnetarService") + .field("db", &self.db) + .field("cache", &self.cache) + .field("config", &self.config) + .finish_non_exhaustive() + } } #[derive(Debug, Error)] @@ -28,6 +41,7 @@ impl MagnetarService { let auth_cache = user_cache::UserCacheService::new(config, db.clone(), cache.clone()).await?; let instance_meta_cache = instance_meta_cache::InstanceMetaCacheService::new(db.clone()); + let emoji_cache = emoji_cache::EmojiCacheService::new(db.clone()); Ok(Self { db, @@ -35,6 +49,7 @@ impl MagnetarService { config, auth_cache, instance_meta_cache, + emoji_cache, }) } } From f0e56deca9d138564819a7c0c66a9d9b9f881805 Mon Sep 17 00:00:00 2001 From: Natty Date: Sat, 28 Oct 2023 00:33:09 +0200 Subject: [PATCH 6/6] Generic caching and basic user fetching in the backend --- magnetar_mmm_parser/src/lib.rs | 6 +-- magnetar_sdk/src/types/user.rs | 12 +++++ src/api_v1/user.rs | 33 +++++++++--- src/model/data/user.rs | 6 +-- src/model/mod.rs | 13 +++++ src/model/processing/drive.rs | 33 ++++++++++++ src/model/processing/emoji.rs | 10 ++++ src/model/processing/mod.rs | 11 +++- src/model/processing/user.rs | 32 ++++++----- src/service/emoji_cache.rs | 1 - src/service/generic_id_cache.rs | 96 +++++++++++++++++++++++++++++++++ src/service/mod.rs | 8 ++- src/web/mod.rs | 38 +++++++++++++ 13 files changed, 263 insertions(+), 36 deletions(-) create mode 100644 src/model/processing/drive.rs create mode 100644 src/service/generic_id_cache.rs diff --git a/magnetar_mmm_parser/src/lib.rs b/magnetar_mmm_parser/src/lib.rs index 3cf6da8..613f269 100644 --- a/magnetar_mmm_parser/src/lib.rs +++ b/magnetar_mmm_parser/src/lib.rs @@ -233,16 +233,14 @@ impl Token { } } - pub fn walk_map_collect(&self, func: impl Fn(&Token) -> Option, out: &mut Vec) { + pub fn walk_map_collect(&self, func: &impl Fn(&Token) -> Option, out: &mut Vec) { if let Some(v) = func(self) { out.push(v) } match self { Token::Sequence(items) => { - items - .iter() - .for_each(|tok| tok.walk_map_collect(&func, out)); + items.iter().for_each(|tok| tok.walk_map_collect(func, out)); } Token::Quote(inner) | Token::Small(inner) diff --git a/magnetar_sdk/src/types/user.rs b/magnetar_sdk/src/types/user.rs index 5a4f8d4..a515716 100644 --- a/magnetar_sdk/src/types/user.rs +++ b/magnetar_sdk/src/types/user.rs @@ -171,6 +171,12 @@ pack!( & Option as auth ); +impl From for PackUserMaybeAll { + fn from(value: PackUserBase) -> Self { + Self::pack_from((value.id, value.user, None, None, None, None, None)) + } +} + pack!( PackUserSelfMaybeAll, Required as id @@ -180,3 +186,9 @@ pack!( & Option as detail & Option as secrets ); + +impl From for PackUserSelfMaybeAll { + fn from(value: PackUserBase) -> Self { + Self::pack_from((value.id, value.user, None, None, None, None)) + } +} diff --git a/src/api_v1/user.rs b/src/api_v1/user.rs index f2b7c9b..74cdb18 100644 --- a/src/api_v1/user.rs +++ b/src/api_v1/user.rs @@ -1,8 +1,11 @@ +use crate::model::processing::user::UserModel; +use crate::model::PackingContext; use crate::service::MagnetarService; use crate::web::auth::{AuthenticatedUser, MaybeUser}; -use crate::web::ApiError; +use crate::web::{ApiError, ObjectNotFound}; use axum::extract::{Path, Query, State}; use axum::Json; +use magnetar_calckey_model::ck; use magnetar_sdk::endpoints::user::{GetUserById, GetUserSelf, UserByIdReq, UserSelfReq}; use magnetar_sdk::endpoints::{Req, Res}; use std::sync::Arc; @@ -14,14 +17,18 @@ pub async fn handle_user_info_self( profile: _, secrets: _, }): Query>, - State(_service): State>, - AuthenticatedUser(_user): AuthenticatedUser, + State(service): State>, + AuthenticatedUser(user): AuthenticatedUser, ) -> Result>, ApiError> { - todo!() + // TODO: Extended properties! + + let ctx = PackingContext::new(service, Some(user.clone())).await?; + let user = UserModel.base_from_existing(&ctx, user.as_ref()).await?; + Ok(Json(user.into())) } pub async fn handle_user_info( - Path(_id): Path, + Path(id): Path, Query(UserByIdReq { detail: _, pins: _, @@ -29,8 +36,18 @@ pub async fn handle_user_info( relation: _, auth: _, }): Query>, - State(_service): State>, - MaybeUser(_user): MaybeUser, + State(service): State>, + MaybeUser(self_user): MaybeUser, ) -> Result>, ApiError> { - todo!() + // TODO: Extended properties! + + let ctx = PackingContext::new(service.clone(), self_user).await?; + let user_model = service + .db + .get_user_by_id(&id) + .await? + .ok_or_else(|| ObjectNotFound(id))?; + + let user = UserModel.base_from_existing(&ctx, &user_model).await?; + Ok(Json(user.into())) } diff --git a/src/model/data/user.rs b/src/model/data/user.rs index 2a9edcf..4ec64bc 100644 --- a/src/model/data/user.rs +++ b/src/model/data/user.rs @@ -19,7 +19,7 @@ impl PackType<&[PackEmojiBase]> for EmojiContext { pub struct UserBaseSource<'a> { pub user: &'a ck::user::Model, pub username_mm: Option<&'a MmXml>, - pub avatar: &'a Option, + pub avatar: Option<&'a ck::drive_file::Model>, pub emoji_context: &'a EmojiContext, } @@ -49,8 +49,8 @@ impl PackType> for UserBase { SpeechTransform::None }, created_at: user.created_at.into(), - avatar_url: avatar.as_ref().map(|v| v.url.clone()), - avatar_blurhash: avatar.as_ref().and_then(|v| v.blurhash.clone()), + avatar_url: avatar.map(|v| v.url.clone()), + avatar_blurhash: avatar.and_then(|v| v.blurhash.clone()), avatar_color: None, avatar_decoration: if user.is_cat { AvatarDecoration::CatEars diff --git a/src/model/mod.rs b/src/model/mod.rs index e54d704..a1c5e2d 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -1,3 +1,4 @@ +use crate::model::processing::PackResult; use crate::service::MagnetarService; use magnetar_calckey_model::ck; use std::sync::Arc; @@ -29,6 +30,18 @@ pub trait PackType: 'static { } impl PackingContext { + pub async fn new( + service: Arc, + self_user: Option>, + ) -> PackResult { + Ok(Self { + instance_meta: service.instance_meta_cache.get().await?, + self_user, + service, + limits: Default::default(), + }) + } + fn self_user(&self) -> Option<&ck::user::Model> { self.self_user.as_deref() } diff --git a/src/model/processing/drive.rs b/src/model/processing/drive.rs new file mode 100644 index 0000000..2e383d0 --- /dev/null +++ b/src/model/processing/drive.rs @@ -0,0 +1,33 @@ +use crate::model::processing::PackResult; +use crate::model::{PackType, PackingContext}; +use magnetar_calckey_model::ck; +use magnetar_sdk::types::drive::{DriveFileBase, PackDriveFileBase}; +use magnetar_sdk::types::Id; +use magnetar_sdk::{Packed, Required}; + +pub struct DriveModel; + +impl DriveModel { + pub fn pack_existing( + &self, + ctx: &PackingContext, + file: &ck::drive_file::Model, + ) -> PackDriveFileBase { + PackDriveFileBase::pack_from(( + Required(Id::from(&file.id)), + Required(DriveFileBase::extract(ctx, &file)), + )) + } + + pub async fn get_cached_base( + &self, + ctx: &PackingContext, + id: &str, + ) -> PackResult> { + let Some(file) = ctx.service.drive_file_cache.get(id).await? else { + return Ok(None); + }; + + Ok(Some(self.pack_existing(ctx, file.as_ref()))) + } +} diff --git a/src/model/processing/emoji.rs b/src/model/processing/emoji.rs index 9c47281..8bb7855 100644 --- a/src/model/processing/emoji.rs +++ b/src/model/processing/emoji.rs @@ -1,5 +1,6 @@ use crate::model::processing::PackResult; use crate::model::{PackType, PackingContext}; +use itertools::Itertools; use magnetar_calckey_model::ck; use magnetar_sdk::types::emoji::{EmojiBase, PackEmojiBase}; use magnetar_sdk::types::Id; @@ -26,4 +27,13 @@ impl EmojiModel { Ok(packed_emojis) } + + pub fn deduplicate_emoji(&self, ctx: &PackingContext, emoji_list: Vec) -> Vec { + emoji_list + .into_iter() + .sorted() + .dedup() + .take(ctx.limits.max_emojis) + .collect::>() + } } diff --git a/src/model/processing/mod.rs b/src/model/processing/mod.rs index 35c9658..8d7af81 100644 --- a/src/model/processing/mod.rs +++ b/src/model/processing/mod.rs @@ -1,13 +1,16 @@ use crate::service::emoji_cache::EmojiCacheError; +use crate::service::generic_id_cache::GenericIdCacheError; +use crate::service::instance_meta_cache::InstanceMetaCacheError; use magnetar_calckey_model::sea_orm::DbErr; use magnetar_calckey_model::CalckeyDbError; use magnetar_sdk::mmm::Token; use thiserror::Error; +pub mod drive; pub mod emoji; pub mod user; -#[derive(Debug, Error)] +#[derive(Debug, Error, strum::IntoStaticStr)] pub enum PackError { #[error("Database error: {0}")] DbError(#[from] DbErr), @@ -15,6 +18,10 @@ pub enum PackError { CalckeyDbError(#[from] CalckeyDbError), #[error("Emoji cache error: {0}")] EmojiCacheError(#[from] EmojiCacheError), + #[error("Instance cache error: {0}")] + InstanceMetaCacheError(#[from] InstanceMetaCacheError), + #[error("Generic cache error: {0}")] + GenericCacheError(#[from] GenericIdCacheError), } pub type PackResult = Result; @@ -22,7 +29,7 @@ pub type PackResult = Result; fn get_mm_token_emoji(token: &Token) -> Vec { let mut v = Vec::new(); token.walk_map_collect( - |t| { + &|t| { if let Token::ShortcodeEmoji(e) = t { Some(e.to_owned()) } else { diff --git a/src/model/processing/user.rs b/src/model/processing/user.rs index c383fed..88ff5b5 100644 --- a/src/model/processing/user.rs +++ b/src/model/processing/user.rs @@ -2,39 +2,37 @@ use crate::model::data::user::UserBaseSource; use crate::model::processing::emoji::EmojiModel; use crate::model::processing::{get_mm_token_emoji, PackResult}; use crate::model::{PackType, PackingContext}; -use itertools::Itertools; use magnetar_calckey_model::ck; use magnetar_calckey_model::sea_orm::EntityTrait; +use magnetar_sdk::mmm::Token; use magnetar_sdk::types::emoji::EmojiContext; use magnetar_sdk::types::user::{PackUserBase, UserBase}; use magnetar_sdk::types::{Id, MmXml}; use magnetar_sdk::{mmm, Packed, Required}; +use std::sync::Arc; pub struct UserModel; impl UserModel { + pub fn tokenize_username(&self, user: &ck::user::Model) -> Token { + mmm::Context::default().parse_ui(user.name.as_deref().unwrap_or(&user.username)) + } + pub async fn base_from_existing( &self, ctx: &PackingContext, user: &ck::user::Model, ) -> PackResult { - let avatar = if let Some(avatar_id) = user.avatar_id.as_ref() { - ck::drive_file::Entity::find_by_id(avatar_id) - .one(ctx.service.db.inner()) - .await? - } else { - None + let avatar = match &user.avatar_id { + Some(av_id) => ctx.service.drive_file_cache.get(av_id).await?, + None => None, }; - let username_mm = - mmm::Context::default().parse_ui(user.name.as_deref().unwrap_or(&user.username)); - let shortcodes = get_mm_token_emoji(&username_mm) - .into_iter() - .sorted() - .dedup() - .take(ctx.limits.max_emojis) - .collect::>(); - let emojis = EmojiModel + let username_mm = self.tokenize_username(&user); + + let emoji_model = EmojiModel; + let shortcodes = emoji_model.deduplicate_emoji(ctx, get_mm_token_emoji(&username_mm)); + let emojis = emoji_model .fetch_many_emojis(ctx, &shortcodes, user.host.as_deref()) .await?; let emoji_context = EmojiContext(emojis); @@ -44,7 +42,7 @@ impl UserModel { UserBaseSource { user, username_mm: mmm::to_xml_string(&username_mm).map(MmXml).as_ref().ok(), - avatar: &avatar, + avatar: avatar.as_deref(), emoji_context: &emoji_context, }, ); diff --git a/src/service/emoji_cache.rs b/src/service/emoji_cache.rs index 96c5c47..772ea54 100644 --- a/src/service/emoji_cache.rs +++ b/src/service/emoji_cache.rs @@ -5,7 +5,6 @@ use std::collections::HashSet; use std::sync::Arc; use strum::EnumVariantNames; use thiserror::Error; -use tokio::io::AsyncWriteExt; use tokio::sync::Mutex; #[derive(Debug, Error, EnumVariantNames)] diff --git a/src/service/generic_id_cache.rs b/src/service/generic_id_cache.rs new file mode 100644 index 0000000..c4df02b --- /dev/null +++ b/src/service/generic_id_cache.rs @@ -0,0 +1,96 @@ +use crate::web::ApiError; +use lru::LruCache; +use magnetar_calckey_model::sea_orm::{EntityTrait, PrimaryKeyTrait}; +use magnetar_calckey_model::{CalckeyDbError, CalckeyModel}; +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use strum::EnumVariantNames; +use thiserror::Error; +use tokio::sync::Mutex; + +#[derive(Debug, Error, EnumVariantNames)] +pub enum GenericIdCacheError { + #[error("Database error: {0}")] + DbError(#[from] CalckeyDbError), +} + +impl From for ApiError { + fn from(err: GenericIdCacheError) -> Self { + let mut api_error: ApiError = match err { + GenericIdCacheError::DbError(err) => err.into(), + }; + + api_error.message = format!("Generic ID cache error: {}", api_error.message); + + api_error + } +} + +#[derive(Debug)] +struct CacheEntry { + created: Instant, + data: Arc, +} + +impl CacheEntry { + fn new(data: Arc) -> Self { + Self { + created: Instant::now(), + data, + } + } +} + +pub struct GenericIdCacheService { + cache: Mutex>>, + lifetime_max: Duration, + db: CalckeyModel, + _entity_type: PhantomData, +} + +impl GenericIdCacheService { + pub(super) fn new(db: CalckeyModel, cache_size: usize, entry_lifetime: Duration) -> Self { + const CACHE_SIZE: usize = 4096; + + Self { + cache: Mutex::new(LruCache::new( + cache_size + .try_into() + .unwrap_or(CACHE_SIZE.try_into().unwrap()), + )), + lifetime_max: entry_lifetime, + db, + _entity_type: PhantomData, + } + } + + pub async fn get<'a>(&self, id: &'a str) -> Result>, GenericIdCacheError> + where + <::PrimaryKey as PrimaryKeyTrait>::ValueType: From<&'a str>, + { + let mut read = self.cache.lock().await; + if let Some(item) = read.peek(id) { + if item.created + self.lifetime_max >= Instant::now() { + let data = item.data.clone(); + read.promote(id); + return Ok(Some(data)); + } + } + drop(read); + + let val = E::find_by_id(id) + .one(self.db.inner()) + .await + .map_err(CalckeyDbError::from)?; + + if val.is_none() { + return Ok(None); + } + + let mut write = self.cache.lock().await; + let data = Arc::new(val.unwrap()); + write.put(id.to_string(), CacheEntry::new(data.clone())); + Ok(Some(data)) + } +} diff --git a/src/service/mod.rs b/src/service/mod.rs index dcbd11e..e1f6670 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -1,9 +1,11 @@ -use magnetar_calckey_model::{CalckeyCache, CalckeyModel}; +use magnetar_calckey_model::{ck, CalckeyCache, CalckeyModel}; use magnetar_common::config::MagnetarConfig; use std::fmt::{Debug, Formatter}; +use std::time::Duration; use thiserror::Error; pub mod emoji_cache; +pub mod generic_id_cache; pub mod instance_meta_cache; pub mod user_cache; @@ -14,6 +16,7 @@ pub struct MagnetarService { pub auth_cache: user_cache::UserCacheService, pub instance_meta_cache: instance_meta_cache::InstanceMetaCacheService, pub emoji_cache: emoji_cache::EmojiCacheService, + pub drive_file_cache: generic_id_cache::GenericIdCacheService, } impl Debug for MagnetarService { @@ -42,6 +45,8 @@ impl MagnetarService { user_cache::UserCacheService::new(config, db.clone(), cache.clone()).await?; let instance_meta_cache = instance_meta_cache::InstanceMetaCacheService::new(db.clone()); let emoji_cache = emoji_cache::EmojiCacheService::new(db.clone()); + let drive_file_cache = + generic_id_cache::GenericIdCacheService::new(db.clone(), 128, Duration::from_secs(10)); Ok(Self { db, @@ -50,6 +55,7 @@ impl MagnetarService { auth_cache, instance_meta_cache, emoji_cache, + drive_file_cache, }) } } diff --git a/src/web/mod.rs b/src/web/mod.rs index bfc834e..a3df24d 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -1,3 +1,4 @@ +use crate::model::processing::PackError; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::Json; @@ -81,3 +82,40 @@ impl From for ApiError { } } } + +impl From for ApiError { + fn from(err: PackError) -> Self { + Self { + status: StatusCode::INTERNAL_SERVER_ERROR, + code: err.error_code(), + message: if cfg!(debug_assertions) { + format!("Data transformation error: {}", err) + } else { + "Data transformation error".to_string() + }, + } + } +} + +#[derive(Debug)] +pub struct ObjectNotFound(pub String); + +impl From<&ObjectNotFound> for &str { + fn from(_: &ObjectNotFound) -> Self { + "ObjectNotFound" + } +} + +impl From for ApiError { + fn from(err: ObjectNotFound) -> Self { + Self { + status: StatusCode::NOT_FOUND, + code: err.error_code(), + message: if cfg!(debug_assertions) { + format!("Object not found: {}", err.0) + } else { + "Object not found".to_string() + }, + } + } +}