From 8e02e46be5c5850164e125b98ccb985b2da6dcf4 Mon Sep 17 00:00:00 2001 From: Natty Date: Mon, 30 Oct 2023 23:00:46 +0100 Subject: [PATCH] User fetching with reactions and a user by tag endpoint --- Cargo.lock | 2 + Cargo.toml | 3 + ext_calckey_model/src/emoji.rs | 103 +++++++++++++++++++++++++++++ ext_calckey_model/src/lib.rs | 43 +----------- magnetar_common/Cargo.toml | 3 +- magnetar_common/src/util.rs | 29 ++++++++ magnetar_mmm_parser/src/lib.rs | 61 ++++++++++++++--- magnetar_sdk/src/endpoints/user.rs | 11 +++ magnetar_sdk/src/types/note.rs | 43 ++++-------- src/api_v1/mod.rs | 3 +- src/api_v1/user.rs | 38 ++++++++++- src/model/data/note.rs | 21 ++---- src/model/processing/emoji.rs | 12 ++++ src/model/processing/mod.rs | 7 +- src/model/processing/note.rs | 88 ++++++++++++++++++++++-- src/service/emoji_cache.rs | 57 +++++++++++++++- src/web/mod.rs | 15 +++++ 17 files changed, 421 insertions(+), 118 deletions(-) create mode 100644 ext_calckey_model/src/emoji.rs diff --git a/Cargo.lock b/Cargo.lock index ceefc1f..6f533a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1458,6 +1458,7 @@ dependencies = [ "futures-util", "headers", "hyper", + "idna", "itertools 0.11.0", "lru", "magnetar_calckey_model", @@ -1535,6 +1536,7 @@ name = "magnetar_common" version = "0.2.1-alpha" dependencies = [ "magnetar_core", + "magnetar_sdk", "percent-encoding", "serde", "thiserror", diff --git a/Cargo.toml b/Cargo.toml index 3c0994d..69b9cca 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ futures-util = "0.3" headers = "0.3" http = "0.2" hyper = "0.14" +idna = "0.4" itertools = "0.11" lru = "0.12" miette = "5.9" @@ -87,6 +88,8 @@ tokio = { workspace = true, features = ["full"] } tower = { workspace = true } tower-http = { workspace = true, features = ["cors", "trace", "fs"] } +idna = { workspace = true } + tracing-subscriber = { workspace = true, features = ["env-filter"] } tracing = { workspace = true } diff --git a/ext_calckey_model/src/emoji.rs b/ext_calckey_model/src/emoji.rs new file mode 100644 index 0000000..bde0d14 --- /dev/null +++ b/ext_calckey_model/src/emoji.rs @@ -0,0 +1,103 @@ +use crate::{CalckeyDbError, CalckeyModel}; +use ck::emoji; +use sea_orm::prelude::Expr; +use sea_orm::{ColumnTrait, EntityTrait, IntoSimpleExpr, QueryFilter, QueryOrder}; + +#[derive(Debug, Copy, Clone)] +pub struct EmojiTag<'a> { + pub name: &'a str, + pub host: Option<&'a str>, +} + +pub struct EmojiResolver(CalckeyModel); + +impl EmojiResolver { + pub fn new(db: CalckeyModel) -> Self { + EmojiResolver(db) + } + + pub async fn get_local_emoji(&self) -> Result, CalckeyDbError> { + Ok(emoji::Entity::find() + .filter(emoji::Column::Host.is_null()) + .order_by_asc(emoji::Column::Category) + .order_by_asc(emoji::Column::Name) + .all(self.0.inner()) + .await?) + } + + pub async fn fetch_emoji( + &self, + shortcode: &str, + host: Option<&str>, + ) -> Result, CalckeyDbError> { + let host_filter = if let Some(host) = host { + emoji::Column::Host.eq(host) + } else { + emoji::Column::Host.is_null() + }; + + let name_filter = emoji::Column::Name.eq(shortcode); + let filter = host_filter.and(name_filter); + + Ok(emoji::Entity::find() + .filter(filter) + .one(self.0.inner()) + .await?) + } + + pub async fn fetch_many_emojis( + &self, + shortcodes: &[String], + host: Option<&str>, + ) -> Result, CalckeyDbError> { + let host_filter = if let Some(host) = host { + emoji::Column::Host.eq(host) + } else { + emoji::Column::Host.is_null() + }; + + let name_filter = emoji::Column::Name.is_in(shortcodes); + let filter = host_filter.and(name_filter); + + Ok(emoji::Entity::find() + .filter(filter) + .all(self.0.inner()) + .await?) + } + + pub async fn fetch_many_tagged_emojis( + &self, + shortcodes: &[EmojiTag<'_>], + ) -> Result, CalckeyDbError> { + let remote_shortcode_host_pairs = shortcodes + .iter() + .filter_map(|s| { + s.host.map(|host| { + Expr::tuple([ + Expr::value(s.name.to_string()), + Expr::value(host.to_string()), + ]) + }) + }) + .collect::>(); + let local_shortcodes = shortcodes + .iter() + .filter_map(|s| s.host.is_none().then_some(s.name.to_string())) + .collect::>(); + + let remote_filter = Expr::tuple([ + emoji::Column::Name.into_simple_expr(), + emoji::Column::Host.into_simple_expr(), + ]) + .is_in(remote_shortcode_host_pairs); + let local_filter = emoji::Column::Name + .is_in(local_shortcodes) + .and(emoji::Column::Host.is_null()); + let filter = remote_filter.or(local_filter); + + Ok(emoji::Entity::find() + .filter(filter) + .all(self.0.inner()) + .await?) + } +} diff --git a/ext_calckey_model/src/lib.rs b/ext_calckey_model/src/lib.rs index 29ff648..b7e6232 100644 --- a/ext_calckey_model/src/lib.rs +++ b/ext_calckey_model/src/lib.rs @@ -1,3 +1,4 @@ +pub mod emoji; pub mod note_model; pub use ck; @@ -192,48 +193,6 @@ impl CalckeyModel { .await?) } - pub async fn get_local_emoji(&self) -> Result, CalckeyDbError> { - Ok(emoji::Entity::find() - .filter(emoji::Column::Host.is_null()) - .order_by_asc(emoji::Column::Category) - .order_by_asc(emoji::Column::Name) - .all(&self.0) - .await?) - } - - pub async fn fetch_emoji( - &self, - shortcode: &str, - host: Option<&str>, - ) -> Result, CalckeyDbError> { - let host_filter = if let Some(host) = host { - emoji::Column::Host.eq(host) - } else { - emoji::Column::Host.is_null() - }; - - let name_filter = emoji::Column::Name.eq(shortcode); - let filter = host_filter.and(name_filter); - - Ok(emoji::Entity::find().filter(filter).one(&self.0).await?) - } - pub async fn fetch_many_emojis( - &self, - shortcodes: &[String], - host: Option<&str>, - ) -> Result, CalckeyDbError> { - let host_filter = if let Some(host) = host { - emoji::Column::Host.eq(host) - } else { - emoji::Column::Host.is_null() - }; - - let name_filter = emoji::Column::Name.is_in(shortcodes); - let filter = host_filter.and(name_filter); - - Ok(emoji::Entity::find().filter(filter).all(&self.0).await?) - } - pub async fn get_access_token( &self, token: &str, diff --git a/magnetar_common/Cargo.toml b/magnetar_common/Cargo.toml index 642fe78..0aa0726 100644 --- a/magnetar_common/Cargo.toml +++ b/magnetar_common/Cargo.toml @@ -8,8 +8,9 @@ crate-type = ["rlib"] [dependencies] magnetar_core = { path = "../core" } +magnetar_sdk = { path = "../magnetar_sdk" } percent-encoding = { workspace = true } serde = { workspace = true, features = ["derive"] } toml = { workspace = true } -thiserror = { workspace = true } \ No newline at end of file +thiserror = { workspace = true } diff --git a/magnetar_common/src/util.rs b/magnetar_common/src/util.rs index 7eb2248..927ea2f 100644 --- a/magnetar_common/src/util.rs +++ b/magnetar_common/src/util.rs @@ -1,4 +1,6 @@ use magnetar_core::web_model::acct::Acct; +use magnetar_sdk::mmm; +use magnetar_sdk::mmm::Token; use percent_encoding::percent_decode_str; use std::borrow::Cow; use thiserror::Error; @@ -17,6 +19,12 @@ pub enum FediverseTagParseError { InvalidUtf8(#[from] std::str::Utf8Error), } +impl From<&FediverseTagParseError> for &str { + fn from(_: &FediverseTagParseError) -> Self { + "FediverseTagParseError" + } +} + impl, S2: AsRef> From<(S1, Option)> for FediverseTag { fn from((name, host): (S1, Option)) -> Self { Self { @@ -106,3 +114,24 @@ pub fn lenient_parse_tag_decode( host: host_decoded, }) } + +#[derive(Debug)] +pub enum RawReaction { + Unicode(String), + Shortcode { + shortcode: String, + host: Option, + }, +} + +pub fn parse_reaction(tag: impl AsRef) -> Option { + let tok = mmm::Context::default().parse_ui(tag.as_ref()); + + match tok { + Token::UnicodeEmoji(text) => Some(RawReaction::Unicode(text)), + Token::ShortcodeEmoji { shortcode, host } => { + Some(RawReaction::Shortcode { shortcode, host }) + } + _ => None, + } +} diff --git a/magnetar_mmm_parser/src/lib.rs b/magnetar_mmm_parser/src/lib.rs index 02f170c..c21572c 100644 --- a/magnetar_mmm_parser/src/lib.rs +++ b/magnetar_mmm_parser/src/lib.rs @@ -86,7 +86,10 @@ pub enum Token { mention_type: MentionType, }, UnicodeEmoji(String), - ShortcodeEmoji(String), + ShortcodeEmoji { + shortcode: String, + host: Option, + }, Hashtag(String), } @@ -109,7 +112,6 @@ impl Token { Token::Function { inner, .. } => inner.str_content_left(), Token::Mention { name, .. } => Some(name.as_ref()), Token::UnicodeEmoji(code) => Some(code.as_ref()), - Token::ShortcodeEmoji(_) => None, Token::Hashtag(tag) => Some(tag.as_ref()), _ => None, } @@ -160,7 +162,7 @@ impl Token { Token::Function { inner, .. } => inner.inner(), Token::Mention { name, .. } => Token::PlainText(name.clone().into()), Token::UnicodeEmoji(code) => Token::PlainText(code.clone().into()), - Token::ShortcodeEmoji(shortcode) => Token::PlainText(shortcode.clone().into()), + Token::ShortcodeEmoji { shortcode, .. } => Token::PlainText(shortcode.clone().into()), Token::Hashtag(tag) => Token::PlainText(tag.clone().into()), } } @@ -406,10 +408,14 @@ impl Token { .create_element("ue") .write_text_content(BytesText::new(text))?; } - Token::ShortcodeEmoji(shortcode) => { - writer - .create_element("ee") - .write_text_content(BytesText::new(shortcode))?; + Token::ShortcodeEmoji { shortcode, host } => { + let mut ew = writer.create_element("ee"); + + if let Some(host) = host { + ew = ew.with_attribute(("host", host.as_str())); + } + + ew.write_text_content(BytesText::new(shortcode))?; } Token::Hashtag(tag) => { writer @@ -1492,10 +1498,26 @@ impl Context { )))), Span::into_fragment, )(input)?; + let (input, host) = opt(map( + tuple(( + tag("@"), + map( + recognize(many1(alt((alphanumeric1, recognize(one_of("-.")))))), + Span::into_fragment, + ), + )), + |(_at, host)| host, + ))(input)?; let (input, _) = tag(":")(input)?; let (input, _) = not(alphanumeric1_unicode)(input)?; - Ok((input, Token::ShortcodeEmoji(shortcode.into()))) + Ok(( + input, + Token::ShortcodeEmoji { + shortcode: shortcode.into(), + host: host.map(str::to_string), + }, + )) } fn tag_mention<'a>(&self, input: Span<'a>) -> IResult, Token> { @@ -2242,17 +2264,34 @@ text"# fn parse_shortcodes() { assert_eq!( parse_full(":bottom:"), - Token::ShortcodeEmoji("bottom".into()) + Token::ShortcodeEmoji { + shortcode: "bottom".into(), + host: None + } ); assert_eq!( parse_full(":bottom::blobfox:"), Token::Sequence(vec![ - Token::ShortcodeEmoji("bottom".into()), - Token::ShortcodeEmoji("blobfox".into()) + Token::ShortcodeEmoji { + shortcode: "bottom".into(), + host: None + }, + Token::ShortcodeEmoji { + shortcode: "blobfox".into(), + host: None + } ]) ); + assert_eq!( + parse_full(":bottom@magnetar.social:"), + Token::ShortcodeEmoji { + shortcode: "bottom".into(), + host: Some("magnetar.social".into()) + } + ); + assert_eq!( parse_full(":bottom:blobfox"), Token::PlainText(":bottom:blobfox".into()) diff --git a/magnetar_sdk/src/endpoints/user.rs b/magnetar_sdk/src/endpoints/user.rs index d4efd81..540b3c6 100644 --- a/magnetar_sdk/src/endpoints/user.rs +++ b/magnetar_sdk/src/endpoints/user.rs @@ -53,3 +53,14 @@ pub struct UserByIdReq { response = PackUserMaybeAll )] pub struct GetUserById; + +// Get user by fedi tag + +#[derive(Endpoint)] +#[endpoint( + endpoint = "/users/by-acct/:user_id", + method = Method::GET, + request = UserByIdReq, + response = PackUserMaybeAll +)] +pub struct GetUserByAcct; diff --git a/magnetar_sdk/src/types/note.rs b/magnetar_sdk/src/types/note.rs index 48d17da..bfc949d 100644 --- a/magnetar_sdk/src/types/note.rs +++ b/magnetar_sdk/src/types/note.rs @@ -63,8 +63,9 @@ pub struct NoteBase { pub renoted_note_id: Option, pub reply_count: u64, pub renote_count: u64, + pub mentions: Vec, pub hashtags: Vec, - pub reactions: Vec, + pub reactions: Vec, pub local_only: bool, pub has_poll: bool, pub file_ids: Vec, @@ -97,37 +98,17 @@ pack!( ); #[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[serde(untagged)] pub enum Reaction { Unicode(String), - Shortcode(String), + Shortcode { + name: String, + host: Option, + url: String, + }, + Unknown { + raw: String, + }, } -impl Reaction { - pub fn guess_from(s: &str, default_emote: &str) -> Option { - let code = s.trim(); - match code.chars().next() { - Some(':') => Some(Reaction::Shortcode(code.to_string())), - Some(_) => Some(Reaction::Unicode(code.to_string())), - None => Self::guess_from(default_emote, "👍"), - } - } -} - -#[derive(Clone, Debug, Deserialize, Serialize, TS)] -pub struct ReactionBase { - pub created_at: DateTime, - pub user_id: String, - pub reaction: Reaction, -} - -pack!(PackReactionBase, Required as id & Required as reaction); - -#[derive(Clone, Debug, Deserialize, Serialize, TS)] -pub struct ReactionUserExt { - pub user: Box, -} - -pack!( - PackReactionWithUser, - Required as id & Required as reaction & Required as user -); +pub type ReactionPair = (Reaction, usize); diff --git a/src/api_v1/mod.rs b/src/api_v1/mod.rs index 79a49db..1c188cd 100644 --- a/src/api_v1/mod.rs +++ b/src/api_v1/mod.rs @@ -2,7 +2,7 @@ mod note; mod user; use crate::api_v1::note::handle_note; -use crate::api_v1::user::{handle_user_info, handle_user_info_self}; +use crate::api_v1::user::{handle_user_info, handle_user_info_by_acct, handle_user_info_self}; use crate::service::MagnetarService; use crate::web::auth; use crate::web::auth::AuthState; @@ -14,6 +14,7 @@ use std::sync::Arc; pub fn create_api_router(service: Arc) -> Router { Router::new() .route("/users/@self", get(handle_user_info_self)) + .route("/users/by-acct/:id", get(handle_user_info_by_acct)) .route("/users/:id", get(handle_user_info)) .route("/notes/:id", get(handle_note)) .layer(from_fn_with_state( diff --git a/src/api_v1/user.rs b/src/api_v1/user.rs index 10db333..8ffcab0 100644 --- a/src/api_v1/user.rs +++ b/src/api_v1/user.rs @@ -5,10 +5,12 @@ use crate::web::auth::{AuthenticatedUser, MaybeUser}; use crate::web::{ApiError, ObjectNotFound}; use axum::extract::{Path, Query, State}; use axum::Json; -use magnetar_sdk::endpoints::user::{GetUserById, GetUserSelf, UserByIdReq, UserSelfReq}; +use magnetar_common::util::lenient_parse_tag; +use magnetar_sdk::endpoints::user::{ + GetUserByAcct, GetUserById, GetUserSelf, UserByIdReq, UserSelfReq, +}; use magnetar_sdk::endpoints::{Req, Res}; use std::sync::Arc; - pub async fn handle_user_info_self( Query(UserSelfReq { detail: _, @@ -45,7 +47,37 @@ pub async fn handle_user_info( .db .get_user_by_id(&id) .await? - .ok_or_else(|| ObjectNotFound(id))?; + .ok_or(ObjectNotFound(id))?; + + let user = UserModel.base_from_existing(&ctx, &user_model).await?; + Ok(Json(user.into())) +} + +pub async fn handle_user_info_by_acct( + Path(tag_str): Path, + Query(UserByIdReq { + detail: _, + pins: _, + profile: _, + relation: _, + auth: _, + }): Query>, + State(service): State>, + MaybeUser(self_user): MaybeUser, +) -> Result>, ApiError> { + // TODO: Extended properties! + + let mut tag = lenient_parse_tag(&tag_str)?; + if matches!(&tag.host, Some(host) if host == &service.config.networking.host) { + tag.host = None; + } + + let ctx = PackingContext::new(service.clone(), self_user).await?; + let user_model = service + .db + .get_user_by_tag(&tag.name, tag.host.as_deref()) + .await? + .ok_or(ObjectNotFound(tag_str))?; let user = UserModel.base_from_existing(&ctx, &user_model).await?; Ok(Json(user.into())) diff --git a/src/model/data/note.rs b/src/model/data/note.rs index 1a7be1f..4b69573 100644 --- a/src/model/data/note.rs +++ b/src/model/data/note.rs @@ -1,12 +1,12 @@ use magnetar_calckey_model::ck; use magnetar_sdk::types::note::{ - NoteDetailExt, PackNoteWithMaybeAttachments, PackPollBase, Reaction, + NoteDetailExt, PackNoteWithMaybeAttachments, PackPollBase, ReactionPair, }; use magnetar_sdk::types::user::PackUserBase; use magnetar_sdk::types::{ drive::PackDriveFileBase, emoji::EmojiContext, - note::{NoteAttachmentExt, NoteBase, NoteVisibility, PackReactionBase, ReactionBase}, + note::{NoteAttachmentExt, NoteBase, NoteVisibility}, user::UserBase, MmXml, }; @@ -14,25 +14,11 @@ use magnetar_sdk::{Packed, Required}; use crate::model::{PackType, PackingContext}; -impl PackType<&ck::note_reaction::Model> for ReactionBase { - fn extract(context: &PackingContext, reaction: &ck::note_reaction::Model) -> Self { - ReactionBase { - created_at: reaction.created_at.into(), - user_id: reaction.user_id.clone(), - reaction: Reaction::guess_from( - &reaction.reaction, - &context.instance_meta.default_reaction, - ) - .unwrap_or_else(|| /* Shouldn't happen */ Reaction::Unicode("👍".to_string())), - } - } -} - pub struct NoteBaseSource<'a> { pub note: &'a ck::note::Model, pub cw_mm: Option<&'a MmXml>, pub text_mm: Option<&'a MmXml>, - pub reactions: &'a Vec, + pub reactions: &'a Vec, pub user: &'a UserBase, pub emoji_context: &'a EmojiContext, } @@ -73,6 +59,7 @@ impl PackType> for NoteBase { renoted_note_id: note.renote_id.clone(), reply_count: note.replies_count as u64, renote_count: note.renote_count as u64, + mentions: note.mentions.clone(), hashtags: note.tags.clone(), reactions: reactions.clone(), local_only: note.local_only, diff --git a/src/model/processing/emoji.rs b/src/model/processing/emoji.rs index 8bb7855..50e19fa 100644 --- a/src/model/processing/emoji.rs +++ b/src/model/processing/emoji.rs @@ -2,6 +2,7 @@ use crate::model::processing::PackResult; use crate::model::{PackType, PackingContext}; use itertools::Itertools; use magnetar_calckey_model::ck; +use magnetar_calckey_model::emoji::EmojiTag; use magnetar_sdk::types::emoji::{EmojiBase, PackEmojiBase}; use magnetar_sdk::types::Id; use magnetar_sdk::{Packed, Required}; @@ -28,6 +29,17 @@ impl EmojiModel { Ok(packed_emojis) } + pub async fn fetch_many_tag_emojis( + &self, + ctx: &PackingContext, + tags: &[EmojiTag<'_>], + ) -> PackResult> { + let emojis = ctx.service.emoji_cache.get_many_tagged(tags).await?; + let packed_emojis = emojis.iter().map(|e| self.pack_existing(ctx, &e)).collect(); + + Ok(packed_emojis) + } + pub fn deduplicate_emoji(&self, ctx: &PackingContext, emoji_list: Vec) -> Vec { emoji_list .into_iter() diff --git a/src/model/processing/mod.rs b/src/model/processing/mod.rs index 59153ed..33d2fc3 100644 --- a/src/model/processing/mod.rs +++ b/src/model/processing/mod.rs @@ -23,6 +23,8 @@ pub enum PackError { InstanceMetaCacheError(#[from] InstanceMetaCacheError), #[error("Generic cache error: {0}")] GenericCacheError(#[from] GenericIdCacheError), + #[error("Deserializer error: {0}")] + DeserializerError(#[from] serde_json::Error), } pub type PackResult = Result; @@ -31,8 +33,9 @@ fn get_mm_token_emoji(token: &Token) -> Vec { let mut v = Vec::new(); token.walk_map_collect( &|t| { - if let Token::ShortcodeEmoji(e) = t { - Some(e.to_owned()) + // TODO: Remote emoji + if let Token::ShortcodeEmoji { shortcode, .. } = t { + Some(shortcode.to_owned()) } else { None } diff --git a/src/model/processing/note.rs b/src/model/processing/note.rs index edbf039..0a4d1aa 100644 --- a/src/model/processing/note.rs +++ b/src/model/processing/note.rs @@ -2,12 +2,14 @@ use crate::model::data::id::BaseId; use crate::model::data::note::{NoteAttachmentSource, NoteBaseSource, NoteDetailSource}; use crate::model::processing::emoji::EmojiModel; use crate::model::processing::user::UserModel; -use crate::model::processing::{get_mm_token_emoji, PackResult}; +use crate::model::processing::{get_mm_token_emoji, PackError, PackResult}; use crate::model::{PackType, PackingContext, UserRelationship}; use compact_str::CompactString; use either::Either; use futures_util::future::try_join_all; +use futures_util::TryFutureExt; use magnetar_calckey_model::ck::sea_orm_active_enums::NoteVisibilityEnum; +use magnetar_calckey_model::emoji::EmojiTag; use magnetar_calckey_model::note_model::{ NoteData, NoteResolveOptions, NoteVisibilityFilterFactory, }; @@ -15,15 +17,17 @@ use magnetar_calckey_model::sea_orm::prelude::Expr; use magnetar_calckey_model::sea_orm::sea_query::{Alias, IntoIden, PgFunc, Query, SimpleExpr}; use magnetar_calckey_model::sea_orm::{ActiveEnum, ColumnTrait, IntoSimpleExpr}; use magnetar_calckey_model::{ck, CalckeyDbError}; +use magnetar_common::util::{parse_reaction, RawReaction}; use magnetar_sdk::mmm::Token; use magnetar_sdk::types::drive::PackDriveFileBase; use magnetar_sdk::types::emoji::EmojiContext; use magnetar_sdk::types::note::{ NoteAttachmentExt, NoteBase, NoteDetailExt, PackNoteBase, PackNoteMaybeFull, - PackNoteWithMaybeAttachments, + PackNoteWithMaybeAttachments, Reaction, }; use magnetar_sdk::types::{Id, MmXml}; use magnetar_sdk::{mmm, Packed, Required}; +use serde::Deserialize; use tokio::try_join; use super::drive::DriveModel; @@ -214,10 +218,80 @@ impl NoteModel { } let emoji_model = EmojiModel; + let shortcodes = emoji_model.deduplicate_emoji(ctx, emoji_extracted); - let emojis = emoji_model - .fetch_many_emojis(ctx, &shortcodes, note_data.user.host.as_deref()) - .await?; + // Parse the JSON into an ordered map and turn it into a Vec of pairs, parsing the reaction codes + // Failed reaction parses -> Left, Successful ones -> Right + let reactions_raw = + serde_json::Map::::deserialize(¬e_data.note.reactions)? + .into_iter() + .map(|(code, count)| { + ( + parse_reaction(&code).map_or_else(|| Either::Left(code), Either::Right), + count, + ) + }) + .map(|(code, count)| Ok((code, usize::deserialize(count)?))) + .collect::, serde_json::Error>>()?; + // Pick out all successfully-parsed shortcode emojis + let reactions_to_resolve = reactions_raw + .iter() + .map(|(code, _)| code) + .map(Either::as_ref) + .filter_map(Either::right) + .filter_map(|c| match c { + RawReaction::Shortcode { shortcode, host } => Some(EmojiTag { + name: shortcode, + host: host.as_deref(), + }), + _ => None, + }) + .collect::>(); + + let reaction_fetch = ctx + .service + .emoji_cache + .get_many_tagged(&reactions_to_resolve) + .map_err(PackError::from); + let emoji_fetch = + emoji_model.fetch_many_emojis(ctx, &shortcodes, note_data.user.host.as_deref()); + + let (reactions_fetched, emojis) = try_join!(reaction_fetch, emoji_fetch)?; + + // Left reactions and the Right ones that didn't resolve to any emoji are turned back into Unknown + let reactions = &reactions_raw + .into_iter() + .map(|(raw, count)| { + ( + raw.either( + |raw| Reaction::Unknown { raw }, + |raw| match raw { + RawReaction::Unicode(text) => Reaction::Unicode(text), + RawReaction::Shortcode { shortcode, host } => reactions_fetched + .iter() + .find(|e| e.host == host && e.name == shortcode) + .map_or_else( + || Reaction::Unknown { + raw: format!( + ":{shortcode}{}:", + host.as_deref() + .map(|h| format!("@{h}")) + .unwrap_or_default() + ), + }, + |e| Reaction::Shortcode { + name: shortcode.clone(), + host: host.clone(), + url: e.public_url.clone(), + }, + ), + }, + ), + count, + ) + }) + .collect::>(); + let emoji_context = &EmojiContext(emojis); let note_base = NoteBase::extract( @@ -236,7 +310,7 @@ impl NoteModel { .and_then(Result::ok) .map(MmXml) .as_ref(), - reactions: &vec![], + reactions, user, emoji_context, }, @@ -353,7 +427,7 @@ impl NoteModel { let extract_renote_attachments = self.extract_renote_target_attachemnts(ctx, &drive_model, ¬e); - // TODO: Polls, reactions, ... + // TODO: Polls, ... let ( PackNoteBase { id, note }, diff --git a/src/service/emoji_cache.rs b/src/service/emoji_cache.rs index 772ea54..dbba24b 100644 --- a/src/service/emoji_cache.rs +++ b/src/service/emoji_cache.rs @@ -1,5 +1,6 @@ use crate::web::ApiError; use lru::LruCache; +use magnetar_calckey_model::emoji::{EmojiResolver, EmojiTag}; use magnetar_calckey_model::{ck, CalckeyDbError, CalckeyModel}; use std::collections::HashSet; use std::sync::Arc; @@ -33,7 +34,7 @@ struct EmojiLocator { pub struct EmojiCacheService { cache: Mutex>>, - db: CalckeyModel, + db: EmojiResolver, } impl EmojiCacheService { @@ -41,7 +42,7 @@ impl EmojiCacheService { const CACHE_SIZE: usize = 4096; Self { cache: Mutex::new(LruCache::new(CACHE_SIZE.try_into().unwrap())), - db, + db: EmojiResolver::new(db), } } @@ -78,7 +79,7 @@ impl EmojiCacheService { host: Option<&str>, ) -> Result>, EmojiCacheError> { let locs = names - .into_iter() + .iter() .map(|n| EmojiLocator { name: n.clone(), host: host.map(str::to_string), @@ -119,4 +120,54 @@ impl EmojiCacheService { Ok(resolved) } + + pub async fn get_many_tagged( + &self, + tags: &[EmojiTag<'_>], + ) -> Result>, EmojiCacheError> { + let locs = tags + .iter() + .map(|tag| EmojiLocator { + name: tag.name.to_string(), + host: tag.host.map(str::to_string), + }) + .collect::>(); + let mut to_resolve = Vec::new(); + let mut resolved = Vec::new(); + + let mut read = self.cache.lock().await; + for loc in locs.iter() { + if let Some(emoji) = read.get(loc) { + resolved.push(emoji.clone()); + } else { + to_resolve.push(EmojiTag { + name: &loc.name, + host: loc.host.as_deref(), + }); + } + } + drop(read); + + let emoji = self + .db + .fetch_many_tagged_emojis(&to_resolve) + .await? + .into_iter() + .map(Arc::new) + .collect::>(); + resolved.extend(emoji.iter().cloned()); + + let mut write = self.cache.lock().await; + emoji.iter().for_each(|e| { + write.put( + EmojiLocator { + name: e.name.clone(), + host: e.host.clone(), + }, + e.clone(), + ); + }); + + Ok(resolved) + } } diff --git a/src/web/mod.rs b/src/web/mod.rs index a3df24d..cdac80f 100644 --- a/src/web/mod.rs +++ b/src/web/mod.rs @@ -3,6 +3,7 @@ use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use axum::Json; use magnetar_calckey_model::{CalckeyCacheError, CalckeyDbError}; +use magnetar_common::util::FediverseTagParseError; use serde::Serialize; use serde_json::json; @@ -55,6 +56,20 @@ impl IntoResponse for ApiError { } } +impl From for ApiError { + fn from(err: FediverseTagParseError) -> Self { + Self { + status: StatusCode::BAD_REQUEST, + code: err.error_code(), + message: if cfg!(debug_assertions) { + format!("Fediverse tag parse error: {}", err) + } else { + "Fediverse tag parse error".to_string() + }, + } + } +} + impl From for ApiError { fn from(err: CalckeyDbError) -> Self { Self {