Merge branch 'development'
ci/woodpecker/push/ociImagePush Pipeline was successful Details

This commit is contained in:
Natty 2023-10-31 01:24:52 +01:00
commit 970644ffc7
Signed by: natty
GPG Key ID: BF6CB659ADEE60EC
17 changed files with 421 additions and 118 deletions

2
Cargo.lock generated
View File

@ -1458,6 +1458,7 @@ dependencies = [
"futures-util", "futures-util",
"headers", "headers",
"hyper", "hyper",
"idna",
"itertools 0.11.0", "itertools 0.11.0",
"lru", "lru",
"magnetar_calckey_model", "magnetar_calckey_model",
@ -1535,6 +1536,7 @@ name = "magnetar_common"
version = "0.2.1-alpha" version = "0.2.1-alpha"
dependencies = [ dependencies = [
"magnetar_core", "magnetar_core",
"magnetar_sdk",
"percent-encoding", "percent-encoding",
"serde", "serde",
"thiserror", "thiserror",

View File

@ -37,6 +37,7 @@ futures-util = "0.3"
headers = "0.3" headers = "0.3"
http = "0.2" http = "0.2"
hyper = "0.14" hyper = "0.14"
idna = "0.4"
itertools = "0.11" itertools = "0.11"
lru = "0.12" lru = "0.12"
miette = "5.9" miette = "5.9"
@ -87,6 +88,8 @@ tokio = { workspace = true, features = ["full"] }
tower = { workspace = true } tower = { workspace = true }
tower-http = { workspace = true, features = ["cors", "trace", "fs"] } tower-http = { workspace = true, features = ["cors", "trace", "fs"] }
idna = { workspace = true }
tracing-subscriber = { workspace = true, features = ["env-filter"] } tracing-subscriber = { workspace = true, features = ["env-filter"] }
tracing = { workspace = true } tracing = { workspace = true }

View File

@ -0,0 +1,103 @@
use crate::{CalckeyDbError, CalckeyModel};
use ck::emoji;
use sea_orm::prelude::Expr;
use sea_orm::{ColumnTrait, EntityTrait, IntoSimpleExpr, QueryFilter, QueryOrder};
#[derive(Debug, Copy, Clone)]
pub struct EmojiTag<'a> {
pub name: &'a str,
pub host: Option<&'a str>,
}
pub struct EmojiResolver(CalckeyModel);
impl EmojiResolver {
pub fn new(db: CalckeyModel) -> Self {
EmojiResolver(db)
}
pub async fn get_local_emoji(&self) -> Result<Vec<emoji::Model>, CalckeyDbError> {
Ok(emoji::Entity::find()
.filter(emoji::Column::Host.is_null())
.order_by_asc(emoji::Column::Category)
.order_by_asc(emoji::Column::Name)
.all(self.0.inner())
.await?)
}
pub async fn fetch_emoji(
&self,
shortcode: &str,
host: Option<&str>,
) -> Result<Option<emoji::Model>, CalckeyDbError> {
let host_filter = if let Some(host) = host {
emoji::Column::Host.eq(host)
} else {
emoji::Column::Host.is_null()
};
let name_filter = emoji::Column::Name.eq(shortcode);
let filter = host_filter.and(name_filter);
Ok(emoji::Entity::find()
.filter(filter)
.one(self.0.inner())
.await?)
}
pub async fn fetch_many_emojis(
&self,
shortcodes: &[String],
host: Option<&str>,
) -> Result<Vec<emoji::Model>, CalckeyDbError> {
let host_filter = if let Some(host) = host {
emoji::Column::Host.eq(host)
} else {
emoji::Column::Host.is_null()
};
let name_filter = emoji::Column::Name.is_in(shortcodes);
let filter = host_filter.and(name_filter);
Ok(emoji::Entity::find()
.filter(filter)
.all(self.0.inner())
.await?)
}
pub async fn fetch_many_tagged_emojis(
&self,
shortcodes: &[EmojiTag<'_>],
) -> Result<Vec<emoji::Model>, CalckeyDbError> {
let remote_shortcode_host_pairs = shortcodes
.iter()
.filter_map(|s| {
s.host.map(|host| {
Expr::tuple([
Expr::value(s.name.to_string()),
Expr::value(host.to_string()),
])
})
})
.collect::<Vec<_>>();
let local_shortcodes = shortcodes
.iter()
.filter_map(|s| s.host.is_none().then_some(s.name.to_string()))
.collect::<Vec<_>>();
let remote_filter = Expr::tuple([
emoji::Column::Name.into_simple_expr(),
emoji::Column::Host.into_simple_expr(),
])
.is_in(remote_shortcode_host_pairs);
let local_filter = emoji::Column::Name
.is_in(local_shortcodes)
.and(emoji::Column::Host.is_null());
let filter = remote_filter.or(local_filter);
Ok(emoji::Entity::find()
.filter(filter)
.all(self.0.inner())
.await?)
}
}

View File

@ -1,3 +1,4 @@
pub mod emoji;
pub mod note_model; pub mod note_model;
pub use ck; pub use ck;
@ -192,48 +193,6 @@ impl CalckeyModel {
.await?) .await?)
} }
pub async fn get_local_emoji(&self) -> Result<Vec<emoji::Model>, CalckeyDbError> {
Ok(emoji::Entity::find()
.filter(emoji::Column::Host.is_null())
.order_by_asc(emoji::Column::Category)
.order_by_asc(emoji::Column::Name)
.all(&self.0)
.await?)
}
pub async fn fetch_emoji(
&self,
shortcode: &str,
host: Option<&str>,
) -> Result<Option<emoji::Model>, CalckeyDbError> {
let host_filter = if let Some(host) = host {
emoji::Column::Host.eq(host)
} else {
emoji::Column::Host.is_null()
};
let name_filter = emoji::Column::Name.eq(shortcode);
let filter = host_filter.and(name_filter);
Ok(emoji::Entity::find().filter(filter).one(&self.0).await?)
}
pub async fn fetch_many_emojis(
&self,
shortcodes: &[String],
host: Option<&str>,
) -> Result<Vec<emoji::Model>, CalckeyDbError> {
let host_filter = if let Some(host) = host {
emoji::Column::Host.eq(host)
} else {
emoji::Column::Host.is_null()
};
let name_filter = emoji::Column::Name.is_in(shortcodes);
let filter = host_filter.and(name_filter);
Ok(emoji::Entity::find().filter(filter).all(&self.0).await?)
}
pub async fn get_access_token( pub async fn get_access_token(
&self, &self,
token: &str, token: &str,

View File

@ -8,6 +8,7 @@ crate-type = ["rlib"]
[dependencies] [dependencies]
magnetar_core = { path = "../core" } magnetar_core = { path = "../core" }
magnetar_sdk = { path = "../magnetar_sdk" }
percent-encoding = { workspace = true } percent-encoding = { workspace = true }
serde = { workspace = true, features = ["derive"] } serde = { workspace = true, features = ["derive"] }

View File

@ -1,4 +1,6 @@
use magnetar_core::web_model::acct::Acct; use magnetar_core::web_model::acct::Acct;
use magnetar_sdk::mmm;
use magnetar_sdk::mmm::Token;
use percent_encoding::percent_decode_str; use percent_encoding::percent_decode_str;
use std::borrow::Cow; use std::borrow::Cow;
use thiserror::Error; use thiserror::Error;
@ -17,6 +19,12 @@ pub enum FediverseTagParseError {
InvalidUtf8(#[from] std::str::Utf8Error), InvalidUtf8(#[from] std::str::Utf8Error),
} }
impl From<&FediverseTagParseError> for &str {
fn from(_: &FediverseTagParseError) -> Self {
"FediverseTagParseError"
}
}
impl<S1: AsRef<str>, S2: AsRef<str>> From<(S1, Option<S2>)> for FediverseTag { impl<S1: AsRef<str>, S2: AsRef<str>> From<(S1, Option<S2>)> for FediverseTag {
fn from((name, host): (S1, Option<S2>)) -> Self { fn from((name, host): (S1, Option<S2>)) -> Self {
Self { Self {
@ -106,3 +114,24 @@ pub fn lenient_parse_tag_decode(
host: host_decoded, host: host_decoded,
}) })
} }
#[derive(Debug)]
pub enum RawReaction {
Unicode(String),
Shortcode {
shortcode: String,
host: Option<String>,
},
}
pub fn parse_reaction(tag: impl AsRef<str>) -> Option<RawReaction> {
let tok = mmm::Context::default().parse_ui(tag.as_ref());
match tok {
Token::UnicodeEmoji(text) => Some(RawReaction::Unicode(text)),
Token::ShortcodeEmoji { shortcode, host } => {
Some(RawReaction::Shortcode { shortcode, host })
}
_ => None,
}
}

View File

@ -86,7 +86,10 @@ pub enum Token {
mention_type: MentionType, mention_type: MentionType,
}, },
UnicodeEmoji(String), UnicodeEmoji(String),
ShortcodeEmoji(String), ShortcodeEmoji {
shortcode: String,
host: Option<String>,
},
Hashtag(String), Hashtag(String),
} }
@ -109,7 +112,6 @@ impl Token {
Token::Function { inner, .. } => inner.str_content_left(), Token::Function { inner, .. } => inner.str_content_left(),
Token::Mention { name, .. } => Some(name.as_ref()), Token::Mention { name, .. } => Some(name.as_ref()),
Token::UnicodeEmoji(code) => Some(code.as_ref()), Token::UnicodeEmoji(code) => Some(code.as_ref()),
Token::ShortcodeEmoji(_) => None,
Token::Hashtag(tag) => Some(tag.as_ref()), Token::Hashtag(tag) => Some(tag.as_ref()),
_ => None, _ => None,
} }
@ -160,7 +162,7 @@ impl Token {
Token::Function { inner, .. } => inner.inner(), Token::Function { inner, .. } => inner.inner(),
Token::Mention { name, .. } => Token::PlainText(name.clone().into()), Token::Mention { name, .. } => Token::PlainText(name.clone().into()),
Token::UnicodeEmoji(code) => Token::PlainText(code.clone().into()), Token::UnicodeEmoji(code) => Token::PlainText(code.clone().into()),
Token::ShortcodeEmoji(shortcode) => Token::PlainText(shortcode.clone().into()), Token::ShortcodeEmoji { shortcode, .. } => Token::PlainText(shortcode.clone().into()),
Token::Hashtag(tag) => Token::PlainText(tag.clone().into()), Token::Hashtag(tag) => Token::PlainText(tag.clone().into()),
} }
} }
@ -406,10 +408,14 @@ impl Token {
.create_element("ue") .create_element("ue")
.write_text_content(BytesText::new(text))?; .write_text_content(BytesText::new(text))?;
} }
Token::ShortcodeEmoji(shortcode) => { Token::ShortcodeEmoji { shortcode, host } => {
writer let mut ew = writer.create_element("ee");
.create_element("ee")
.write_text_content(BytesText::new(shortcode))?; if let Some(host) = host {
ew = ew.with_attribute(("host", host.as_str()));
}
ew.write_text_content(BytesText::new(shortcode))?;
} }
Token::Hashtag(tag) => { Token::Hashtag(tag) => {
writer writer
@ -1492,10 +1498,26 @@ impl Context {
)))), )))),
Span::into_fragment, Span::into_fragment,
)(input)?; )(input)?;
let (input, host) = opt(map(
tuple((
tag("@"),
map(
recognize(many1(alt((alphanumeric1, recognize(one_of("-.")))))),
Span::into_fragment,
),
)),
|(_at, host)| host,
))(input)?;
let (input, _) = tag(":")(input)?; let (input, _) = tag(":")(input)?;
let (input, _) = not(alphanumeric1_unicode)(input)?; let (input, _) = not(alphanumeric1_unicode)(input)?;
Ok((input, Token::ShortcodeEmoji(shortcode.into()))) Ok((
input,
Token::ShortcodeEmoji {
shortcode: shortcode.into(),
host: host.map(str::to_string),
},
))
} }
fn tag_mention<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> { fn tag_mention<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
@ -2242,17 +2264,34 @@ text</center>"#
fn parse_shortcodes() { fn parse_shortcodes() {
assert_eq!( assert_eq!(
parse_full(":bottom:"), parse_full(":bottom:"),
Token::ShortcodeEmoji("bottom".into()) Token::ShortcodeEmoji {
shortcode: "bottom".into(),
host: None
}
); );
assert_eq!( assert_eq!(
parse_full(":bottom::blobfox:"), parse_full(":bottom::blobfox:"),
Token::Sequence(vec![ Token::Sequence(vec![
Token::ShortcodeEmoji("bottom".into()), Token::ShortcodeEmoji {
Token::ShortcodeEmoji("blobfox".into()) shortcode: "bottom".into(),
host: None
},
Token::ShortcodeEmoji {
shortcode: "blobfox".into(),
host: None
}
]) ])
); );
assert_eq!(
parse_full(":bottom@magnetar.social:"),
Token::ShortcodeEmoji {
shortcode: "bottom".into(),
host: Some("magnetar.social".into())
}
);
assert_eq!( assert_eq!(
parse_full(":bottom:blobfox"), parse_full(":bottom:blobfox"),
Token::PlainText(":bottom:blobfox".into()) Token::PlainText(":bottom:blobfox".into())

View File

@ -53,3 +53,14 @@ pub struct UserByIdReq {
response = PackUserMaybeAll response = PackUserMaybeAll
)] )]
pub struct GetUserById; pub struct GetUserById;
// Get user by fedi tag
#[derive(Endpoint)]
#[endpoint(
endpoint = "/users/by-acct/:user_id",
method = Method::GET,
request = UserByIdReq,
response = PackUserMaybeAll
)]
pub struct GetUserByAcct;

View File

@ -63,8 +63,9 @@ pub struct NoteBase {
pub renoted_note_id: Option<String>, pub renoted_note_id: Option<String>,
pub reply_count: u64, pub reply_count: u64,
pub renote_count: u64, pub renote_count: u64,
pub mentions: Vec<String>,
pub hashtags: Vec<String>, pub hashtags: Vec<String>,
pub reactions: Vec<PackReactionBase>, pub reactions: Vec<ReactionPair>,
pub local_only: bool, pub local_only: bool,
pub has_poll: bool, pub has_poll: bool,
pub file_ids: Vec<String>, pub file_ids: Vec<String>,
@ -97,37 +98,17 @@ pack!(
); );
#[derive(Clone, Debug, Deserialize, Serialize, TS)] #[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[serde(untagged)]
pub enum Reaction { pub enum Reaction {
Unicode(String), Unicode(String),
Shortcode(String), Shortcode {
name: String,
host: Option<String>,
url: String,
},
Unknown {
raw: String,
},
} }
impl Reaction { pub type ReactionPair = (Reaction, usize);
pub fn guess_from(s: &str, default_emote: &str) -> Option<Self> {
let code = s.trim();
match code.chars().next() {
Some(':') => Some(Reaction::Shortcode(code.to_string())),
Some(_) => Some(Reaction::Unicode(code.to_string())),
None => Self::guess_from(default_emote, "👍"),
}
}
}
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
pub struct ReactionBase {
pub created_at: DateTime<Utc>,
pub user_id: String,
pub reaction: Reaction,
}
pack!(PackReactionBase, Required<Id> as id & Required<ReactionBase> as reaction);
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
pub struct ReactionUserExt {
pub user: Box<PackUserBase>,
}
pack!(
PackReactionWithUser,
Required<Id> as id & Required<ReactionBase> as reaction & Required<ReactionUserExt> as user
);

View File

@ -2,7 +2,7 @@ mod note;
mod user; mod user;
use crate::api_v1::note::handle_note; use crate::api_v1::note::handle_note;
use crate::api_v1::user::{handle_user_info, handle_user_info_self}; use crate::api_v1::user::{handle_user_info, handle_user_info_by_acct, handle_user_info_self};
use crate::service::MagnetarService; use crate::service::MagnetarService;
use crate::web::auth; use crate::web::auth;
use crate::web::auth::AuthState; use crate::web::auth::AuthState;
@ -14,6 +14,7 @@ use std::sync::Arc;
pub fn create_api_router(service: Arc<MagnetarService>) -> Router { pub fn create_api_router(service: Arc<MagnetarService>) -> Router {
Router::new() Router::new()
.route("/users/@self", get(handle_user_info_self)) .route("/users/@self", get(handle_user_info_self))
.route("/users/by-acct/:id", get(handle_user_info_by_acct))
.route("/users/:id", get(handle_user_info)) .route("/users/:id", get(handle_user_info))
.route("/notes/:id", get(handle_note)) .route("/notes/:id", get(handle_note))
.layer(from_fn_with_state( .layer(from_fn_with_state(

View File

@ -5,10 +5,12 @@ use crate::web::auth::{AuthenticatedUser, MaybeUser};
use crate::web::{ApiError, ObjectNotFound}; use crate::web::{ApiError, ObjectNotFound};
use axum::extract::{Path, Query, State}; use axum::extract::{Path, Query, State};
use axum::Json; use axum::Json;
use magnetar_sdk::endpoints::user::{GetUserById, GetUserSelf, UserByIdReq, UserSelfReq}; use magnetar_common::util::lenient_parse_tag;
use magnetar_sdk::endpoints::user::{
GetUserByAcct, GetUserById, GetUserSelf, UserByIdReq, UserSelfReq,
};
use magnetar_sdk::endpoints::{Req, Res}; use magnetar_sdk::endpoints::{Req, Res};
use std::sync::Arc; use std::sync::Arc;
pub async fn handle_user_info_self( pub async fn handle_user_info_self(
Query(UserSelfReq { Query(UserSelfReq {
detail: _, detail: _,
@ -45,7 +47,37 @@ pub async fn handle_user_info(
.db .db
.get_user_by_id(&id) .get_user_by_id(&id)
.await? .await?
.ok_or_else(|| ObjectNotFound(id))?; .ok_or(ObjectNotFound(id))?;
let user = UserModel.base_from_existing(&ctx, &user_model).await?;
Ok(Json(user.into()))
}
pub async fn handle_user_info_by_acct(
Path(tag_str): Path<String>,
Query(UserByIdReq {
detail: _,
pins: _,
profile: _,
relation: _,
auth: _,
}): Query<Req<GetUserByAcct>>,
State(service): State<Arc<MagnetarService>>,
MaybeUser(self_user): MaybeUser,
) -> Result<Json<Res<GetUserByAcct>>, ApiError> {
// TODO: Extended properties!
let mut tag = lenient_parse_tag(&tag_str)?;
if matches!(&tag.host, Some(host) if host == &service.config.networking.host) {
tag.host = None;
}
let ctx = PackingContext::new(service.clone(), self_user).await?;
let user_model = service
.db
.get_user_by_tag(&tag.name, tag.host.as_deref())
.await?
.ok_or(ObjectNotFound(tag_str))?;
let user = UserModel.base_from_existing(&ctx, &user_model).await?; let user = UserModel.base_from_existing(&ctx, &user_model).await?;
Ok(Json(user.into())) Ok(Json(user.into()))

View File

@ -1,12 +1,12 @@
use magnetar_calckey_model::ck; use magnetar_calckey_model::ck;
use magnetar_sdk::types::note::{ use magnetar_sdk::types::note::{
NoteDetailExt, PackNoteWithMaybeAttachments, PackPollBase, Reaction, NoteDetailExt, PackNoteWithMaybeAttachments, PackPollBase, ReactionPair,
}; };
use magnetar_sdk::types::user::PackUserBase; use magnetar_sdk::types::user::PackUserBase;
use magnetar_sdk::types::{ use magnetar_sdk::types::{
drive::PackDriveFileBase, drive::PackDriveFileBase,
emoji::EmojiContext, emoji::EmojiContext,
note::{NoteAttachmentExt, NoteBase, NoteVisibility, PackReactionBase, ReactionBase}, note::{NoteAttachmentExt, NoteBase, NoteVisibility},
user::UserBase, user::UserBase,
MmXml, MmXml,
}; };
@ -14,25 +14,11 @@ use magnetar_sdk::{Packed, Required};
use crate::model::{PackType, PackingContext}; use crate::model::{PackType, PackingContext};
impl PackType<&ck::note_reaction::Model> for ReactionBase {
fn extract(context: &PackingContext, reaction: &ck::note_reaction::Model) -> Self {
ReactionBase {
created_at: reaction.created_at.into(),
user_id: reaction.user_id.clone(),
reaction: Reaction::guess_from(
&reaction.reaction,
&context.instance_meta.default_reaction,
)
.unwrap_or_else(|| /* Shouldn't happen */ Reaction::Unicode("👍".to_string())),
}
}
}
pub struct NoteBaseSource<'a> { pub struct NoteBaseSource<'a> {
pub note: &'a ck::note::Model, pub note: &'a ck::note::Model,
pub cw_mm: Option<&'a MmXml>, pub cw_mm: Option<&'a MmXml>,
pub text_mm: Option<&'a MmXml>, pub text_mm: Option<&'a MmXml>,
pub reactions: &'a Vec<PackReactionBase>, pub reactions: &'a Vec<ReactionPair>,
pub user: &'a UserBase, pub user: &'a UserBase,
pub emoji_context: &'a EmojiContext, pub emoji_context: &'a EmojiContext,
} }
@ -73,6 +59,7 @@ impl PackType<NoteBaseSource<'_>> for NoteBase {
renoted_note_id: note.renote_id.clone(), renoted_note_id: note.renote_id.clone(),
reply_count: note.replies_count as u64, reply_count: note.replies_count as u64,
renote_count: note.renote_count as u64, renote_count: note.renote_count as u64,
mentions: note.mentions.clone(),
hashtags: note.tags.clone(), hashtags: note.tags.clone(),
reactions: reactions.clone(), reactions: reactions.clone(),
local_only: note.local_only, local_only: note.local_only,

View File

@ -2,6 +2,7 @@ use crate::model::processing::PackResult;
use crate::model::{PackType, PackingContext}; use crate::model::{PackType, PackingContext};
use itertools::Itertools; use itertools::Itertools;
use magnetar_calckey_model::ck; use magnetar_calckey_model::ck;
use magnetar_calckey_model::emoji::EmojiTag;
use magnetar_sdk::types::emoji::{EmojiBase, PackEmojiBase}; use magnetar_sdk::types::emoji::{EmojiBase, PackEmojiBase};
use magnetar_sdk::types::Id; use magnetar_sdk::types::Id;
use magnetar_sdk::{Packed, Required}; use magnetar_sdk::{Packed, Required};
@ -28,6 +29,17 @@ impl EmojiModel {
Ok(packed_emojis) Ok(packed_emojis)
} }
pub async fn fetch_many_tag_emojis(
&self,
ctx: &PackingContext,
tags: &[EmojiTag<'_>],
) -> PackResult<Vec<PackEmojiBase>> {
let emojis = ctx.service.emoji_cache.get_many_tagged(tags).await?;
let packed_emojis = emojis.iter().map(|e| self.pack_existing(ctx, &e)).collect();
Ok(packed_emojis)
}
pub fn deduplicate_emoji(&self, ctx: &PackingContext, emoji_list: Vec<String>) -> Vec<String> { pub fn deduplicate_emoji(&self, ctx: &PackingContext, emoji_list: Vec<String>) -> Vec<String> {
emoji_list emoji_list
.into_iter() .into_iter()

View File

@ -23,6 +23,8 @@ pub enum PackError {
InstanceMetaCacheError(#[from] InstanceMetaCacheError), InstanceMetaCacheError(#[from] InstanceMetaCacheError),
#[error("Generic cache error: {0}")] #[error("Generic cache error: {0}")]
GenericCacheError(#[from] GenericIdCacheError), GenericCacheError(#[from] GenericIdCacheError),
#[error("Deserializer error: {0}")]
DeserializerError(#[from] serde_json::Error),
} }
pub type PackResult<T> = Result<T, PackError>; pub type PackResult<T> = Result<T, PackError>;
@ -31,8 +33,9 @@ fn get_mm_token_emoji(token: &Token) -> Vec<String> {
let mut v = Vec::new(); let mut v = Vec::new();
token.walk_map_collect( token.walk_map_collect(
&|t| { &|t| {
if let Token::ShortcodeEmoji(e) = t { // TODO: Remote emoji
Some(e.to_owned()) if let Token::ShortcodeEmoji { shortcode, .. } = t {
Some(shortcode.to_owned())
} else { } else {
None None
} }

View File

@ -2,12 +2,14 @@ use crate::model::data::id::BaseId;
use crate::model::data::note::{NoteAttachmentSource, NoteBaseSource, NoteDetailSource}; use crate::model::data::note::{NoteAttachmentSource, NoteBaseSource, NoteDetailSource};
use crate::model::processing::emoji::EmojiModel; use crate::model::processing::emoji::EmojiModel;
use crate::model::processing::user::UserModel; use crate::model::processing::user::UserModel;
use crate::model::processing::{get_mm_token_emoji, PackResult}; use crate::model::processing::{get_mm_token_emoji, PackError, PackResult};
use crate::model::{PackType, PackingContext, UserRelationship}; use crate::model::{PackType, PackingContext, UserRelationship};
use compact_str::CompactString; use compact_str::CompactString;
use either::Either; use either::Either;
use futures_util::future::try_join_all; use futures_util::future::try_join_all;
use futures_util::TryFutureExt;
use magnetar_calckey_model::ck::sea_orm_active_enums::NoteVisibilityEnum; use magnetar_calckey_model::ck::sea_orm_active_enums::NoteVisibilityEnum;
use magnetar_calckey_model::emoji::EmojiTag;
use magnetar_calckey_model::note_model::{ use magnetar_calckey_model::note_model::{
NoteData, NoteResolveOptions, NoteVisibilityFilterFactory, NoteData, NoteResolveOptions, NoteVisibilityFilterFactory,
}; };
@ -15,15 +17,17 @@ use magnetar_calckey_model::sea_orm::prelude::Expr;
use magnetar_calckey_model::sea_orm::sea_query::{Alias, IntoIden, PgFunc, Query, SimpleExpr}; use magnetar_calckey_model::sea_orm::sea_query::{Alias, IntoIden, PgFunc, Query, SimpleExpr};
use magnetar_calckey_model::sea_orm::{ActiveEnum, ColumnTrait, IntoSimpleExpr}; use magnetar_calckey_model::sea_orm::{ActiveEnum, ColumnTrait, IntoSimpleExpr};
use magnetar_calckey_model::{ck, CalckeyDbError}; use magnetar_calckey_model::{ck, CalckeyDbError};
use magnetar_common::util::{parse_reaction, RawReaction};
use magnetar_sdk::mmm::Token; use magnetar_sdk::mmm::Token;
use magnetar_sdk::types::drive::PackDriveFileBase; use magnetar_sdk::types::drive::PackDriveFileBase;
use magnetar_sdk::types::emoji::EmojiContext; use magnetar_sdk::types::emoji::EmojiContext;
use magnetar_sdk::types::note::{ use magnetar_sdk::types::note::{
NoteAttachmentExt, NoteBase, NoteDetailExt, PackNoteBase, PackNoteMaybeFull, NoteAttachmentExt, NoteBase, NoteDetailExt, PackNoteBase, PackNoteMaybeFull,
PackNoteWithMaybeAttachments, PackNoteWithMaybeAttachments, Reaction,
}; };
use magnetar_sdk::types::{Id, MmXml}; use magnetar_sdk::types::{Id, MmXml};
use magnetar_sdk::{mmm, Packed, Required}; use magnetar_sdk::{mmm, Packed, Required};
use serde::Deserialize;
use tokio::try_join; use tokio::try_join;
use super::drive::DriveModel; use super::drive::DriveModel;
@ -214,10 +218,80 @@ impl NoteModel {
} }
let emoji_model = EmojiModel; let emoji_model = EmojiModel;
let shortcodes = emoji_model.deduplicate_emoji(ctx, emoji_extracted); let shortcodes = emoji_model.deduplicate_emoji(ctx, emoji_extracted);
let emojis = emoji_model // Parse the JSON into an ordered map and turn it into a Vec of pairs, parsing the reaction codes
.fetch_many_emojis(ctx, &shortcodes, note_data.user.host.as_deref()) // Failed reaction parses -> Left, Successful ones -> Right
.await?; let reactions_raw =
serde_json::Map::<String, serde_json::Value>::deserialize(&note_data.note.reactions)?
.into_iter()
.map(|(code, count)| {
(
parse_reaction(&code).map_or_else(|| Either::Left(code), Either::Right),
count,
)
})
.map(|(code, count)| Ok((code, usize::deserialize(count)?)))
.collect::<Result<Vec<_>, serde_json::Error>>()?;
// Pick out all successfully-parsed shortcode emojis
let reactions_to_resolve = reactions_raw
.iter()
.map(|(code, _)| code)
.map(Either::as_ref)
.filter_map(Either::right)
.filter_map(|c| match c {
RawReaction::Shortcode { shortcode, host } => Some(EmojiTag {
name: shortcode,
host: host.as_deref(),
}),
_ => None,
})
.collect::<Vec<_>>();
let reaction_fetch = ctx
.service
.emoji_cache
.get_many_tagged(&reactions_to_resolve)
.map_err(PackError::from);
let emoji_fetch =
emoji_model.fetch_many_emojis(ctx, &shortcodes, note_data.user.host.as_deref());
let (reactions_fetched, emojis) = try_join!(reaction_fetch, emoji_fetch)?;
// Left reactions and the Right ones that didn't resolve to any emoji are turned back into Unknown
let reactions = &reactions_raw
.into_iter()
.map(|(raw, count)| {
(
raw.either(
|raw| Reaction::Unknown { raw },
|raw| match raw {
RawReaction::Unicode(text) => Reaction::Unicode(text),
RawReaction::Shortcode { shortcode, host } => reactions_fetched
.iter()
.find(|e| e.host == host && e.name == shortcode)
.map_or_else(
|| Reaction::Unknown {
raw: format!(
":{shortcode}{}:",
host.as_deref()
.map(|h| format!("@{h}"))
.unwrap_or_default()
),
},
|e| Reaction::Shortcode {
name: shortcode.clone(),
host: host.clone(),
url: e.public_url.clone(),
},
),
},
),
count,
)
})
.collect::<Vec<_>>();
let emoji_context = &EmojiContext(emojis); let emoji_context = &EmojiContext(emojis);
let note_base = NoteBase::extract( let note_base = NoteBase::extract(
@ -236,7 +310,7 @@ impl NoteModel {
.and_then(Result::ok) .and_then(Result::ok)
.map(MmXml) .map(MmXml)
.as_ref(), .as_ref(),
reactions: &vec![], reactions,
user, user,
emoji_context, emoji_context,
}, },
@ -353,7 +427,7 @@ impl NoteModel {
let extract_renote_attachments = let extract_renote_attachments =
self.extract_renote_target_attachemnts(ctx, &drive_model, &note); self.extract_renote_target_attachemnts(ctx, &drive_model, &note);
// TODO: Polls, reactions, ... // TODO: Polls, ...
let ( let (
PackNoteBase { id, note }, PackNoteBase { id, note },

View File

@ -1,5 +1,6 @@
use crate::web::ApiError; use crate::web::ApiError;
use lru::LruCache; use lru::LruCache;
use magnetar_calckey_model::emoji::{EmojiResolver, EmojiTag};
use magnetar_calckey_model::{ck, CalckeyDbError, CalckeyModel}; use magnetar_calckey_model::{ck, CalckeyDbError, CalckeyModel};
use std::collections::HashSet; use std::collections::HashSet;
use std::sync::Arc; use std::sync::Arc;
@ -33,7 +34,7 @@ struct EmojiLocator {
pub struct EmojiCacheService { pub struct EmojiCacheService {
cache: Mutex<LruCache<EmojiLocator, Arc<ck::emoji::Model>>>, cache: Mutex<LruCache<EmojiLocator, Arc<ck::emoji::Model>>>,
db: CalckeyModel, db: EmojiResolver,
} }
impl EmojiCacheService { impl EmojiCacheService {
@ -41,7 +42,7 @@ impl EmojiCacheService {
const CACHE_SIZE: usize = 4096; const CACHE_SIZE: usize = 4096;
Self { Self {
cache: Mutex::new(LruCache::new(CACHE_SIZE.try_into().unwrap())), cache: Mutex::new(LruCache::new(CACHE_SIZE.try_into().unwrap())),
db, db: EmojiResolver::new(db),
} }
} }
@ -78,7 +79,7 @@ impl EmojiCacheService {
host: Option<&str>, host: Option<&str>,
) -> Result<Vec<Arc<ck::emoji::Model>>, EmojiCacheError> { ) -> Result<Vec<Arc<ck::emoji::Model>>, EmojiCacheError> {
let locs = names let locs = names
.into_iter() .iter()
.map(|n| EmojiLocator { .map(|n| EmojiLocator {
name: n.clone(), name: n.clone(),
host: host.map(str::to_string), host: host.map(str::to_string),
@ -119,4 +120,54 @@ impl EmojiCacheService {
Ok(resolved) Ok(resolved)
} }
pub async fn get_many_tagged(
&self,
tags: &[EmojiTag<'_>],
) -> Result<Vec<Arc<ck::emoji::Model>>, EmojiCacheError> {
let locs = tags
.iter()
.map(|tag| EmojiLocator {
name: tag.name.to_string(),
host: tag.host.map(str::to_string),
})
.collect::<HashSet<_>>();
let mut to_resolve = Vec::new();
let mut resolved = Vec::new();
let mut read = self.cache.lock().await;
for loc in locs.iter() {
if let Some(emoji) = read.get(loc) {
resolved.push(emoji.clone());
} else {
to_resolve.push(EmojiTag {
name: &loc.name,
host: loc.host.as_deref(),
});
}
}
drop(read);
let emoji = self
.db
.fetch_many_tagged_emojis(&to_resolve)
.await?
.into_iter()
.map(Arc::new)
.collect::<Vec<_>>();
resolved.extend(emoji.iter().cloned());
let mut write = self.cache.lock().await;
emoji.iter().for_each(|e| {
write.put(
EmojiLocator {
name: e.name.clone(),
host: e.host.clone(),
},
e.clone(),
);
});
Ok(resolved)
}
} }

View File

@ -3,6 +3,7 @@ use axum::http::StatusCode;
use axum::response::{IntoResponse, Response}; use axum::response::{IntoResponse, Response};
use axum::Json; use axum::Json;
use magnetar_calckey_model::{CalckeyCacheError, CalckeyDbError}; use magnetar_calckey_model::{CalckeyCacheError, CalckeyDbError};
use magnetar_common::util::FediverseTagParseError;
use serde::Serialize; use serde::Serialize;
use serde_json::json; use serde_json::json;
@ -55,6 +56,20 @@ impl IntoResponse for ApiError {
} }
} }
impl From<FediverseTagParseError> for ApiError {
fn from(err: FediverseTagParseError) -> Self {
Self {
status: StatusCode::BAD_REQUEST,
code: err.error_code(),
message: if cfg!(debug_assertions) {
format!("Fediverse tag parse error: {}", err)
} else {
"Fediverse tag parse error".to_string()
},
}
}
}
impl From<CalckeyDbError> for ApiError { impl From<CalckeyDbError> for ApiError {
fn from(err: CalckeyDbError) -> Self { fn from(err: CalckeyDbError) -> Self {
Self { Self {