From 6908a2f350c4475c0e25fa3fdf72de9c8e60d2b9 Mon Sep 17 00:00:00 2001 From: Natty Date: Sun, 5 Nov 2023 15:23:48 +0100 Subject: [PATCH] Proxied images and user instance meta resolving --- Cargo.lock | 7 +- Cargo.toml | 1 + config/default.toml | 13 +++ ext_calckey_model/src/lib.rs | 13 +++ ext_calckey_model/src/note_model.rs | 1 - magnetar_common/Cargo.toml | 2 + magnetar_common/src/config.rs | 33 +++++++- magnetar_sdk/src/lib.rs | 1 - magnetar_sdk/src/types/drive.rs | 2 + magnetar_sdk/src/types/instance.rs | 13 +++ magnetar_sdk/src/types/mod.rs | 1 + magnetar_sdk/src/types/note.rs | 2 + magnetar_sdk/src/types/user.rs | 8 +- src/model/data/drive.rs | 25 +++++- src/model/data/instance.rs | 16 ++++ src/model/data/mod.rs | 1 + src/model/data/note.rs | 3 + src/model/data/user.rs | 40 +++++++-- src/model/processing/drive.rs | 125 +++++++++++++++++++++++++++- src/model/processing/mod.rs | 5 ++ src/model/processing/user.rs | 54 +++++++++++- src/service/instance_cache.rs | 89 ++++++++++++++++++++ src/service/mod.rs | 10 +++ 23 files changed, 443 insertions(+), 22 deletions(-) create mode 100644 magnetar_sdk/src/types/instance.rs create mode 100644 src/model/data/instance.rs create mode 100644 src/service/instance_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 136912d..295c786 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1482,6 +1482,7 @@ dependencies = [ "tracing", "tracing-subscriber", "unicode-segmentation", + "url", ] [[package]] @@ -1537,12 +1538,14 @@ dependencies = [ name = "magnetar_common" version = "0.2.1-alpha" dependencies = [ + "idna", "magnetar_core", "magnetar_sdk", "percent-encoding", "serde", "thiserror", "toml 0.8.1", + "url", ] [[package]] @@ -3670,9 +3673,9 @@ checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", diff --git a/Cargo.toml b/Cargo.toml index e6a68d0..2af3aae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,7 @@ hyper = { workspace = true, features = ["full"] } tokio = { workspace = true, features = ["full"] } tower = { workspace = true } tower-http = { workspace = true, features = ["cors", "trace", "fs"] } +url = { workspace = true } idna = { workspace = true } diff --git a/config/default.toml b/config/default.toml index 07cd085..a855fba 100644 --- a/config/default.toml +++ b/config/default.toml @@ -47,6 +47,19 @@ # Environment variable: MAG_C_BIND_ADDR # networking.bind_addr = "::" +# [Optional] +# The URL of a media proxy +# Default: null +# Environment variable: MAG_C_MEDIA_PROXY +# networking.media_proxy = "" + +# [Optional] +# Whether to proxy remote files through this instance +# Default: false +# Environment variable: MAG_C_PROXY_REMOTE_FILES +# networking.proxy_remote_files = false + + # -----------------------------[ CALCKEY FRONTEND ]---------------------------- # [Optional] diff --git a/ext_calckey_model/src/lib.rs b/ext_calckey_model/src/lib.rs index 1afba36..573c749 100644 --- a/ext_calckey_model/src/lib.rs +++ b/ext_calckey_model/src/lib.rs @@ -61,6 +61,7 @@ impl CalckeyModel { .sqlx_logging_level(LevelFilter::Debug) .to_owned(); + info!("Attempting database connection..."); Ok(CalckeyModel(sea_orm::Database::connect(opt).await?)) } @@ -224,6 +225,18 @@ impl CalckeyModel { .await?) } + pub async fn get_instance( + &self, + host: &str, + ) -> Result, CalckeyDbError> { + let instance = instance::Entity::find() + .filter(instance::Column::Host.eq(host)) + .one(&self.0) + .await?; + + Ok(instance) + } + pub async fn get_instance_meta(&self) -> Result { let txn = self.0.begin().await?; diff --git a/ext_calckey_model/src/note_model.rs b/ext_calckey_model/src/note_model.rs index 006b7e8..8dfa9e4 100644 --- a/ext_calckey_model/src/note_model.rs +++ b/ext_calckey_model/src/note_model.rs @@ -5,7 +5,6 @@ use sea_orm::{ QueryFilter, QueryResult, QuerySelect, QueryTrait, RelationTrait, Select, }; use serde::{Deserialize, Serialize}; -use tracing::info; use ck::{drive_file, note, note_reaction, user}; use magnetar_sdk::types::RangeFilter; diff --git a/magnetar_common/Cargo.toml b/magnetar_common/Cargo.toml index 0aa0726..d6acd8d 100644 --- a/magnetar_common/Cargo.toml +++ b/magnetar_common/Cargo.toml @@ -10,7 +10,9 @@ crate-type = ["rlib"] magnetar_core = { path = "../core" } magnetar_sdk = { path = "../magnetar_sdk" } +idna = { workspace = true } percent-encoding = { workspace = true } serde = { workspace = true, features = ["derive"] } toml = { workspace = true } thiserror = { workspace = true } +url = { workspace = true } diff --git a/magnetar_common/src/config.rs b/magnetar_common/src/config.rs index a1d6946..77b5c09 100644 --- a/magnetar_common/src/config.rs +++ b/magnetar_common/src/config.rs @@ -10,6 +10,8 @@ pub struct MagnetarNetworking { pub port: u16, pub bind_addr: IpAddr, pub protocol: MagnetarNetworkingProtocol, + pub media_proxy: Option, + pub proxy_remote_files: bool, } #[derive(Deserialize, Debug)] @@ -66,6 +68,19 @@ fn env_protocol() -> MagnetarNetworkingProtocol { } } +fn env_media_proxy() -> Option { + std::env::var("MAG_C_MEDIA_PROXY") + .ok() + .filter(String::is_empty) +} + +fn env_proxy_remote_files() -> bool { + std::env::var("MAG_C_PROXY_REMOTE_FILES") + .unwrap_or_else(|_| "false".to_string()) + .parse() + .expect("MAG_C_PROXY_REMOTE_FILES must be a boolean") +} + impl Default for MagnetarNetworking { fn default() -> Self { MagnetarNetworking { @@ -73,6 +88,8 @@ impl Default for MagnetarNetworking { bind_addr: env_bind_addr(), port: env_port(), protocol: env_protocol(), + media_proxy: env_media_proxy(), + proxy_remote_files: env_proxy_remote_files(), } } } @@ -192,6 +209,8 @@ pub enum MagnetarConfigError { IoError(#[from] std::io::Error), #[error("Failed to parse configuration: {0}")] DeserializeError(#[from] toml::de::Error), + #[error("Configuration error: Not a valid hostname")] + ConfigHostnameError(#[from] idna::Errors), } pub fn load_config() -> Result { @@ -200,7 +219,19 @@ pub fn load_config() -> Result { let str_cfg = std::fs::read_to_string(path)?; - let config = toml::from_str(&str_cfg)?; + let mut config: MagnetarConfig = toml::from_str(&str_cfg)?; + + // Validate the host + idna::domain_to_unicode(&config.networking.host).1?; + + if config + .networking + .media_proxy + .as_deref() + .is_some_and(str::is_empty) + { + config.networking.media_proxy = None; + } Ok(config) } diff --git a/magnetar_sdk/src/lib.rs b/magnetar_sdk/src/lib.rs index f2a954d..c2ba37a 100644 --- a/magnetar_sdk/src/lib.rs +++ b/magnetar_sdk/src/lib.rs @@ -1,4 +1,3 @@ -use chrono::format; use serde::{Deserialize, Serialize}; use std::ops::{Deref, DerefMut}; use ts_rs::TS; diff --git a/magnetar_sdk/src/types/drive.rs b/magnetar_sdk/src/types/drive.rs index 49a8abd..691d3b6 100644 --- a/magnetar_sdk/src/types/drive.rs +++ b/magnetar_sdk/src/types/drive.rs @@ -53,7 +53,9 @@ pub struct DriveFileBase { pub mime_type: String, pub media_metadata: ImageMeta, pub url: Option, + pub source_url: String, pub thumbnail_url: Option, + pub blurhash: Option, pub sensitive: bool, pub comment: Option, pub folder_id: Option, diff --git a/magnetar_sdk/src/types/instance.rs b/magnetar_sdk/src/types/instance.rs new file mode 100644 index 0000000..1153e2c --- /dev/null +++ b/magnetar_sdk/src/types/instance.rs @@ -0,0 +1,13 @@ +use serde::{Deserialize, Serialize}; +use ts_rs::TS; + +#[derive(Clone, Debug, Deserialize, Serialize, TS)] +#[ts(export)] +pub struct InstanceTicker { + pub name: Option, + pub software_name: Option, + pub software_version: Option, + pub icon_url: Option, + pub favicon_url: Option, + pub theme_color: Option, +} diff --git a/magnetar_sdk/src/types/mod.rs b/magnetar_sdk/src/types/mod.rs index 215c045..126625a 100644 --- a/magnetar_sdk/src/types/mod.rs +++ b/magnetar_sdk/src/types/mod.rs @@ -1,5 +1,6 @@ pub mod drive; pub mod emoji; +pub mod instance; pub mod note; pub mod timeline; pub mod user; diff --git a/magnetar_sdk/src/types/note.rs b/magnetar_sdk/src/types/note.rs index 3c43b76..d8c89b7 100644 --- a/magnetar_sdk/src/types/note.rs +++ b/magnetar_sdk/src/types/note.rs @@ -51,6 +51,7 @@ pack!(PackPollBase, Required as id & Required as poll); #[ts(export)] pub struct NoteBase { pub created_at: DateTime, + pub updated_at: Option>, pub cw: Option, pub cw_mm: Option, pub uri: Option, @@ -64,6 +65,7 @@ pub struct NoteBase { pub reply_count: u64, pub renote_count: u64, pub mentions: Vec, + pub visible_user_ids: Option>, pub hashtags: Vec, pub reactions: Vec, pub local_only: bool, diff --git a/magnetar_sdk/src/types/user.rs b/magnetar_sdk/src/types/user.rs index d4ee9bd..b03657a 100644 --- a/magnetar_sdk/src/types/user.rs +++ b/magnetar_sdk/src/types/user.rs @@ -5,6 +5,7 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use ts_rs::TS; +use crate::types::instance::InstanceTicker; use crate::types::note::PackNoteMaybeFull; use magnetar_sdk_macros::pack; @@ -43,14 +44,14 @@ pub struct UserBase { pub host: Option, pub speech_transform: SpeechTransform, pub created_at: DateTime, - pub avatar_url: Option, + pub avatar_url: String, pub avatar_blurhash: Option, - pub avatar_color: Option, pub avatar_decoration: AvatarDecoration, pub is_admin: bool, pub is_moderator: bool, pub is_bot: bool, pub emojis: EmojiContext, + pub instance: Option, } pack!(PackUserBase, Required as id & Required as user); @@ -78,7 +79,6 @@ pub struct UserProfileExt { pub also_known_as: Option, pub banner_url: Option, - pub banner_color: Option, pub banner_blurhash: Option, pub has_public_reactions: bool, @@ -96,6 +96,8 @@ pub struct UserProfilePinsEx { pub struct UserRelationExt { pub follows_you: bool, pub you_follow: bool, + pub you_request_follow: bool, + pub they_request_follow: bool, pub blocks_you: bool, pub you_block: bool, pub mute: bool, diff --git a/src/model/data/drive.rs b/src/model/data/drive.rs index c994ae8..25f386c 100644 --- a/src/model/data/drive.rs +++ b/src/model/data/drive.rs @@ -1,11 +1,26 @@ use magnetar_calckey_model::ck; use magnetar_sdk::types::drive::{DriveFileBase, ImageMeta}; use serde::Deserialize; +use url::Url; use crate::model::{PackType, PackingContext}; -impl PackType<&ck::drive_file::Model> for DriveFileBase { - fn extract(_context: &PackingContext, file: &ck::drive_file::Model) -> Self { +#[derive(Debug)] +pub struct PackFileBaseInput<'a> { + pub file: &'a ck::drive_file::Model, + pub effective_url: Option<&'a Url>, + pub effective_thumbnail_url: Option<&'a Url>, +} + +impl PackType> for DriveFileBase { + fn extract( + _context: &PackingContext, + PackFileBaseInput { + file, + effective_url, + effective_thumbnail_url, + }: PackFileBaseInput<'_>, + ) -> Self { let media_metadata = ImageMeta::deserialize(file.properties.clone()).unwrap_or_default(); DriveFileBase { @@ -15,8 +30,10 @@ impl PackType<&ck::drive_file::Model> for DriveFileBase { hash: None, // TODO: blake3 mime_type: file.r#type.clone(), media_metadata, - url: Some(file.url.clone()), - thumbnail_url: file.thumbnail_url.clone(), + url: effective_url.map(Url::to_string), + source_url: file.url.clone(), + thumbnail_url: effective_thumbnail_url.map(Url::to_string), + blurhash: file.blurhash.clone(), sensitive: file.is_sensitive, comment: file.comment.clone(), folder_id: file.folder_id.clone(), diff --git a/src/model/data/instance.rs b/src/model/data/instance.rs new file mode 100644 index 0000000..d74a00f --- /dev/null +++ b/src/model/data/instance.rs @@ -0,0 +1,16 @@ +use crate::model::{PackType, PackingContext}; +use magnetar_calckey_model::ck; +use magnetar_sdk::types::instance::InstanceTicker; + +impl<'a> PackType<&'a ck::instance::Model> for InstanceTicker { + fn extract(_context: &PackingContext, data: &'a ck::instance::Model) -> Self { + InstanceTicker { + name: data.name.clone(), + software_name: data.software_name.clone(), + software_version: data.software_version.clone(), + icon_url: data.icon_url.clone(), + favicon_url: data.favicon_url.clone(), + theme_color: data.theme_color.clone(), + } + } +} diff --git a/src/model/data/mod.rs b/src/model/data/mod.rs index 0b1c42c..8dc31f4 100644 --- a/src/model/data/mod.rs +++ b/src/model/data/mod.rs @@ -1,6 +1,7 @@ pub mod drive; pub mod emoji; pub mod id; +pub mod instance; pub mod note; pub mod poll; pub mod user; diff --git a/src/model/data/note.rs b/src/model/data/note.rs index d4bd760..2a2e0f0 100644 --- a/src/model/data/note.rs +++ b/src/model/data/note.rs @@ -39,6 +39,7 @@ impl PackType> for NoteBase { use ck::sea_orm_active_enums::NoteVisibilityEnum as NVE; NoteBase { created_at: note.created_at.into(), + updated_at: note.updated_at.map(|d| d.into()), cw: note.cw.clone(), cw_mm: cw_mm.cloned(), uri: note.uri.clone(), @@ -62,6 +63,8 @@ impl PackType> for NoteBase { renote_count: note.renote_count as u64, mentions: note.mentions.clone(), hashtags: note.tags.clone(), + visible_user_ids: matches!(note.visibility, NVE::Specified) + .then(|| note.visible_user_ids.clone()), reactions: reactions.clone(), local_only: note.local_only, has_poll: note.has_poll, diff --git a/src/model/data/user.rs b/src/model/data/user.rs index a49ea80..373b39a 100644 --- a/src/model/data/user.rs +++ b/src/model/data/user.rs @@ -1,12 +1,15 @@ use magnetar_calckey_model::ck; use magnetar_calckey_model::ck::sea_orm_active_enums::UserProfileFfvisibilityEnum; +use magnetar_sdk::types::drive::PackDriveFileBase; use magnetar_sdk::types::emoji::{EmojiContext, PackEmojiBase}; +use magnetar_sdk::types::instance::InstanceTicker; use magnetar_sdk::types::note::PackNoteMaybeFull; use magnetar_sdk::types::user::{ AvatarDecoration, PackSecurityKeyBase, ProfileField, SecurityKeyBase, SpeechTransform, UserBase, UserDetailExt, UserProfileExt, UserProfilePinsEx, UserRelationExt, UserSecretsExt, }; use magnetar_sdk::types::MmXml; +use url::Url; use crate::model::{PackType, PackingContext}; @@ -19,8 +22,10 @@ impl PackType<&[PackEmojiBase]> for EmojiContext { pub struct UserBaseSource<'a> { pub user: &'a ck::user::Model, pub username_mm: Option<&'a MmXml>, - pub avatar: Option<&'a ck::drive_file::Model>, + pub avatar_url: &'a Url, + pub avatar: Option<&'a PackDriveFileBase>, pub emoji_context: &'a EmojiContext, + pub instance: Option<&'a InstanceTicker>, } impl PackType> for UserBase { @@ -29,8 +34,10 @@ impl PackType> for UserBase { UserBaseSource { user, username_mm, + avatar_url, avatar, emoji_context, + instance, }: UserBaseSource, ) -> Self { UserBase { @@ -49,9 +56,8 @@ impl PackType> for UserBase { SpeechTransform::None }, created_at: user.created_at.into(), - avatar_url: avatar.map(|v| v.url.clone()), - avatar_blurhash: avatar.and_then(|v| v.blurhash.clone()), - avatar_color: None, + avatar_url: avatar_url.to_string(), + avatar_blurhash: avatar.and_then(|v| v.file.0.blurhash.clone()), avatar_decoration: if user.is_cat { AvatarDecoration::CatEars } else { @@ -61,6 +67,7 @@ impl PackType> for UserBase { is_moderator: user.is_moderator, is_bot: user.is_bot, emojis: emoji_context.clone(), + instance: instance.map(|i| i.clone()), } } } @@ -69,6 +76,8 @@ pub struct UserProfileExtSource<'a> { pub user: &'a ck::user::Model, pub profile: &'a ck::user_profile::Model, pub profile_fields: &'a Vec, + pub banner_url: Option<&'a Url>, + pub banner: Option<&'a ck::drive_file::Model>, pub description_mm: Option<&'a MmXml>, pub relation: Option<&'a UserRelationExt>, } @@ -80,6 +89,8 @@ impl PackType> for UserProfileExt { user, profile, profile_fields, + banner_url, + banner, description_mm, relation, }: UserProfileExtSource, @@ -108,9 +119,8 @@ impl PackType> for UserProfileExt { url: profile.url.clone(), moved_to_uri: user.moved_to_uri.clone(), also_known_as: user.also_known_as.clone(), - banner_url: None, - banner_color: None, - banner_blurhash: None, + banner_url: banner_url.map(Url::to_string), + banner_blurhash: banner.and_then(|b| b.blurhash.clone()), has_public_reactions: profile.public_reactions, } } @@ -133,6 +143,8 @@ struct UserRelationExtSource<'a> { pub block_in: Option<&'a ck::blocking::Model>, pub mute: Option<&'a ck::muting::Model>, pub renote_mute: Option<&'a ck::renote_muting::Model>, + pub follow_request_out: Option<&'a ck::follow_request::Model>, + pub follow_request_in: Option<&'a ck::follow_request::Model>, } impl PackType> for UserRelationExt { @@ -145,6 +157,8 @@ impl PackType> for UserRelationExt { block_in, mute, renote_mute, + follow_request_in, + follow_request_out, }: UserRelationExtSource, ) -> Self { let self_user = context.self_user(); @@ -180,6 +194,18 @@ impl PackType> for UserRelationExt { self_user.id == renote_mute.muter_id && self_user.id != renote_mute.mutee_id }) }), + you_request_follow: self_user.is_some_and(|self_user| { + follow_request_in.is_some_and(|follow_req_in| { + self_user.id == follow_req_in.followee_id + && self_user.id != follow_req_in.follower_id + }) + }), + they_request_follow: self_user.is_some_and(|self_user| { + follow_request_out.is_some_and(|follow_req_out| { + self_user.id == follow_req_out.follower_id + && self_user.id != follow_req_out.followee_id + }) + }), } } } diff --git a/src/model/processing/drive.rs b/src/model/processing/drive.rs index 2e383d0..c400dd5 100644 --- a/src/model/processing/drive.rs +++ b/src/model/processing/drive.rs @@ -1,21 +1,144 @@ +use crate::model::data::drive::PackFileBaseInput; use crate::model::processing::PackResult; use crate::model::{PackType, PackingContext}; use magnetar_calckey_model::ck; use magnetar_sdk::types::drive::{DriveFileBase, PackDriveFileBase}; use magnetar_sdk::types::Id; use magnetar_sdk::{Packed, Required}; +use tracing::warn; +use url::Url; pub struct DriveModel; impl DriveModel { + pub fn media_proxy_url( + &self, + ctx: &PackingContext, + url: &str, + is_thumbnail: bool, + ) -> Option { + if let Some(proxy) = &ctx.service.config.networking.media_proxy { + let params = if is_thumbnail { + vec![("url", url), ("thumbnail", "1")] + } else { + vec![("url", url)] + }; + + let url = Url::parse_with_params(proxy, ¶ms); + + if let Err(e) = url { + warn!("Url parse error: {e}"); + } + + return url.ok(); + } + + None + } + + pub fn builtin_proxy_file( + &self, + ctx: &PackingContext, + file: &ck::drive_file::Model, + is_thumbnail: bool, + ) -> Option { + let key = if is_thumbnail { + file.thumbnail_access_key.as_deref() + } else { + file.webpublic_access_key.as_deref() + }; + + if let Some(k) = key { + if k != "/" { + let url_raw = format!( + "{}://{}/files/{}", + ctx.service.config.networking.protocol.as_ref(), + &ctx.service.config.networking.host, + k + ); + let url = Url::parse(&url_raw); + + if let Err(e) = url { + warn!("Url parse error: {e}"); + } + + return url.ok(); + } + } + + None + } + + pub fn get_public_url( + &self, + ctx: &PackingContext, + file: &ck::drive_file::Model, + is_thumbnail: bool, + ) -> Option { + if let Some(uri) = &file.uri { + if file.user_host.is_none() { + if let Some(media_proxy_url) = self.media_proxy_url(ctx, uri, is_thumbnail) { + return Some(media_proxy_url); + } + + if file.is_link && ctx.service.config.networking.proxy_remote_files { + if let Some(proxy_url) = self.builtin_proxy_file(ctx, file, is_thumbnail) { + return Some(proxy_url); + } + } + } + } + + let is_image = matches!( + file.r#type.as_str(), + "image/png" + | "image/apng" + | "image/gif" + | "image/jpeg" + | "image/webp" + | "image/svg+xml" + | "image/avif" + ); + + let url_raw = if is_thumbnail { + file.thumbnail_url + .as_deref() + .or(is_image.then_some(file.webpublic_url.as_deref().unwrap_or(file.url.as_str()))) + } else { + file.webpublic_url.as_deref().or(Some(file.url.as_str())) + }; + + if let Some(u) = url_raw { + let url = Url::parse(u); + + if let Err(e) = url { + warn!("Url parse error: {e}"); + } + + return url.ok(); + } + + None + } + pub fn pack_existing( &self, ctx: &PackingContext, file: &ck::drive_file::Model, ) -> PackDriveFileBase { + let url = self.get_public_url(ctx, file, false); + let thumbnail_url = self.get_public_url(ctx, file, false); + PackDriveFileBase::pack_from(( Required(Id::from(&file.id)), - Required(DriveFileBase::extract(ctx, &file)), + Required(DriveFileBase::extract( + ctx, + PackFileBaseInput { + file: &file, + effective_url: url.as_ref(), + effective_thumbnail_url: thumbnail_url.as_ref(), + }, + )), )) } diff --git a/src/model/processing/mod.rs b/src/model/processing/mod.rs index 33d2fc3..e678d80 100644 --- a/src/model/processing/mod.rs +++ b/src/model/processing/mod.rs @@ -1,5 +1,6 @@ use crate::service::emoji_cache::EmojiCacheError; use crate::service::generic_id_cache::GenericIdCacheError; +use crate::service::instance_cache::RemoteInstanceCacheError; use crate::service::instance_meta_cache::InstanceMetaCacheError; use magnetar_calckey_model::sea_orm::DbErr; use magnetar_calckey_model::CalckeyDbError; @@ -23,8 +24,12 @@ pub enum PackError { InstanceMetaCacheError(#[from] InstanceMetaCacheError), #[error("Generic cache error: {0}")] GenericCacheError(#[from] GenericIdCacheError), + #[error("Remote instance cache error: {0}")] + RemoteInstanceCacheError(#[from] RemoteInstanceCacheError), #[error("Deserializer error: {0}")] DeserializerError(#[from] serde_json::Error), + #[error("URL parse error: {0}")] + UrlParseError(#[from] url::ParseError), } pub type PackResult = Result; diff --git a/src/model/processing/user.rs b/src/model/processing/user.rs index 5243857..b244cae 100644 --- a/src/model/processing/user.rs +++ b/src/model/processing/user.rs @@ -1,13 +1,18 @@ use crate::model::data::user::UserBaseSource; +use crate::model::processing::drive::DriveModel; use crate::model::processing::emoji::EmojiModel; use crate::model::processing::{get_mm_token_emoji, PackResult}; use crate::model::{PackType, PackingContext}; use magnetar_calckey_model::ck; use magnetar_sdk::mmm::Token; +use magnetar_sdk::types::drive::PackDriveFileBase; use magnetar_sdk::types::emoji::EmojiContext; +use magnetar_sdk::types::instance::InstanceTicker; use magnetar_sdk::types::user::{PackUserBase, UserBase}; use magnetar_sdk::types::{Id, MmXml}; use magnetar_sdk::{mmm, Packed, Required}; +use tracing::warn; +use url::Url; pub struct UserModel; @@ -16,15 +21,48 @@ impl UserModel { mmm::Context::default().parse_ui(user.name.as_deref().unwrap_or(&user.username)) } + pub fn get_effective_avatar_url( + &self, + ctx: &PackingContext, + user: &ck::user::Model, + avatar: Option<&PackDriveFileBase>, + ) -> PackResult { + Ok(avatar + .and_then( + |PackDriveFileBase { + file: Required(base), + .. + }| base.thumbnail_url.as_deref(), + ) + .map(Url::parse) + .and_then(|r| { + if let Err(e) = r { + warn!("Failed to parse avatar URL: {e}"); + } + + r.ok().map(Ok) + }) + .unwrap_or_else(|| { + Url::parse(&format!( + "{}://{}/identicon/{}", + ctx.service.config.networking.protocol, + ctx.service.config.networking.host, + user.id + )) + })?) + } + pub async fn base_from_existing( &self, ctx: &PackingContext, user: &ck::user::Model, ) -> PackResult { + let drive_file_pack = DriveModel; let avatar = match &user.avatar_id { - Some(av_id) => ctx.service.drive_file_cache.get(av_id).await?, + Some(av_id) => drive_file_pack.get_cached_base(ctx, av_id).await?, None => None, }; + let avatar_url = &self.get_effective_avatar_url(ctx, user, avatar.as_ref())?; let username_mm = self.tokenize_username(&user); @@ -33,6 +71,16 @@ impl UserModel { let emojis = emoji_model .fetch_many_emojis(ctx, &shortcodes, user.host.as_deref()) .await?; + let instance = ctx + .service + .remote_instance_cache + .get( + user.host + .as_deref() + .unwrap_or(&ctx.service.config.networking.host), + ) + .await? + .map(|i| InstanceTicker::extract(ctx, i.as_ref())); let emoji_context = EmojiContext(emojis); let base = UserBase::extract( @@ -40,8 +88,10 @@ impl UserModel { UserBaseSource { user, username_mm: mmm::to_xml_string(&username_mm).map(MmXml).as_ref().ok(), - avatar: avatar.as_deref(), + avatar_url, + avatar: avatar.as_ref(), emoji_context: &emoji_context, + instance: instance.as_ref(), }, ); diff --git a/src/service/instance_cache.rs b/src/service/instance_cache.rs new file mode 100644 index 0000000..592ead3 --- /dev/null +++ b/src/service/instance_cache.rs @@ -0,0 +1,89 @@ +use crate::web::ApiError; +use lru::LruCache; +use magnetar_calckey_model::{ck, CalckeyDbError, CalckeyModel}; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use strum::EnumVariantNames; +use thiserror::Error; +use tokio::sync::Mutex; + +#[derive(Debug, Error, EnumVariantNames)] +pub enum RemoteInstanceCacheError { + #[error("Database error: {0}")] + DbError(#[from] CalckeyDbError), +} + +impl From for ApiError { + fn from(err: RemoteInstanceCacheError) -> Self { + let mut api_error: ApiError = match err { + RemoteInstanceCacheError::DbError(err) => err.into(), + }; + + api_error.message = format!("Remote instance cache error: {}", api_error.message); + + api_error + } +} + +#[derive(Debug)] +struct CacheEntry { + created: Instant, + data: Arc, +} + +impl CacheEntry { + fn new(data: Arc) -> Self { + Self { + created: Instant::now(), + data, + } + } +} + +pub struct RemoteInstanceCacheService { + cache: Mutex>, + lifetime_max: Duration, + db: CalckeyModel, +} + +impl RemoteInstanceCacheService { + pub(super) fn new(db: CalckeyModel, cache_size: usize, entry_lifetime: Duration) -> Self { + const CACHE_SIZE: usize = 256; + + Self { + cache: Mutex::new(LruCache::new( + cache_size + .try_into() + .unwrap_or(CACHE_SIZE.try_into().unwrap()), + )), + lifetime_max: entry_lifetime, + db, + } + } + + pub async fn get( + &self, + host: &str, + ) -> Result>, RemoteInstanceCacheError> { + let mut read = self.cache.lock().await; + if let Some(item) = read.peek(host) { + if item.created + self.lifetime_max >= Instant::now() { + let data = item.data.clone(); + read.promote(host); + return Ok(Some(data)); + } + } + drop(read); + + let val = self.db.get_instance(host).await?; + + if val.is_none() { + return Ok(None); + } + + let mut write = self.cache.lock().await; + let data = Arc::new(val.unwrap()); + write.put(host.to_string(), CacheEntry::new(data.clone())); + Ok(Some(data)) + } +} diff --git a/src/service/mod.rs b/src/service/mod.rs index 74a091e..91c0cc3 100644 --- a/src/service/mod.rs +++ b/src/service/mod.rs @@ -6,15 +6,19 @@ use thiserror::Error; pub mod emoji_cache; pub mod generic_id_cache; +pub mod instance_cache; pub mod instance_meta_cache; pub mod local_user_cache; +#[non_exhaustive] + pub struct MagnetarService { pub db: CalckeyModel, pub cache: CalckeyCache, pub config: &'static MagnetarConfig, pub local_user_cache: local_user_cache::LocalUserCacheService, pub instance_meta_cache: instance_meta_cache::InstanceMetaCacheService, + pub remote_instance_cache: instance_cache::RemoteInstanceCacheService, pub emoji_cache: emoji_cache::EmojiCacheService, pub drive_file_cache: generic_id_cache::GenericIdCacheService, } @@ -45,6 +49,11 @@ impl MagnetarService { local_user_cache::LocalUserCacheService::new(config, db.clone(), cache.clone()).await?; let instance_meta_cache = instance_meta_cache::InstanceMetaCacheService::new(db.clone()); let emoji_cache = emoji_cache::EmojiCacheService::new(db.clone()); + let remote_instance_cache = instance_cache::RemoteInstanceCacheService::new( + db.clone(), + 256, + Duration::from_secs(100), + ); let drive_file_cache = generic_id_cache::GenericIdCacheService::new(db.clone(), 128, Duration::from_secs(10)); @@ -54,6 +63,7 @@ impl MagnetarService { config, local_user_cache, instance_meta_cache, + remote_instance_cache, emoji_cache, drive_file_cache, })