Merge branch 'development'
ci/woodpecker/push/ociImagePush Pipeline was successful Details

This commit is contained in:
Natty 2023-10-29 17:06:11 +01:00
commit f4fa5925f7
Signed by: natty
GPG Key ID: BF6CB659ADEE60EC
7 changed files with 266 additions and 62 deletions

1
Cargo.lock generated
View File

@ -1455,6 +1455,7 @@ dependencies = [
"compact_str", "compact_str",
"dotenvy", "dotenvy",
"either", "either",
"futures-util",
"headers", "headers",
"hyper", "hyper",
"itertools 0.11.0", "itertools 0.11.0",

View File

@ -15,7 +15,7 @@ members = [
"magnetar_common", "magnetar_common",
"magnetar_sdk", "magnetar_sdk",
"magnetar_mmm_parser", "magnetar_mmm_parser",
"core" "core",
] ]
[workspace.package] [workspace.package]
@ -38,8 +38,6 @@ headers = "0.3"
http = "0.2" http = "0.2"
hyper = "0.14" hyper = "0.14"
itertools = "0.11" itertools = "0.11"
js-sys = "0.3"
log = "0.4"
lru = "0.12" lru = "0.12"
miette = "5.9" miette = "5.9"
nom = "7" nom = "7"
@ -67,9 +65,6 @@ ts-rs = "7"
unicode-segmentation = "1.10" unicode-segmentation = "1.10"
url = "2.3" url = "2.3"
walkdir = "2.3" walkdir = "2.3"
wasm-bindgen = "0.2"
wasm-bindgen-futures = "0.4"
web-sys = "0.3"
[dependencies] [dependencies]
magnetar_core = { path = "./core" } magnetar_core = { path = "./core" }
@ -101,6 +96,7 @@ itertools = { workspace = true }
compact_str = { workspace = true } compact_str = { workspace = true }
either = { workspace = true } either = { workspace = true }
futures-util = { workspace = true }
strum = { workspace = true, features = ["derive"] } strum = { workspace = true, features = ["derive"] }
thiserror = { workspace = true } thiserror = { workspace = true }
miette = { workspace = true, features = ["fancy"] } miette = { workspace = true, features = ["fancy"] }

View File

@ -222,7 +222,7 @@ impl NoteResolver {
ALIAS_RENOTE.clone(), ALIAS_RENOTE.clone(),
) )
.join_as( .join_as(
JoinType::InnerJoin, JoinType::LeftJoin,
note::Relation::User.with_alias(ALIAS_RENOTE.clone()), note::Relation::User.with_alias(ALIAS_RENOTE.clone()),
ALIAS_RENOTE_USER.clone(), ALIAS_RENOTE_USER.clone(),
); );

View File

@ -76,19 +76,19 @@ pack!(PackNoteBase, Required<Id> as id & Required<NoteBase> as note);
#[derive(Clone, Debug, Deserialize, Serialize, TS)] #[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[ts(export)] #[ts(export)]
pub struct NoteAttachmentExt { pub struct NoteAttachmentExt {
pub poll: Option<PackPollBase>,
pub attachments: Vec<PackDriveFileBase>, pub attachments: Vec<PackDriveFileBase>,
} }
pack!( pack!(
PackNoteWithAttachments, PackNoteWithMaybeAttachments,
Required<Id> as id & Required<NoteBase> as note & Required<NoteAttachmentExt> as attachment Required<Id> as id & Required<NoteBase> as note & Option<NoteAttachmentExt> as attachment
); );
#[derive(Clone, Debug, Deserialize, Serialize, TS)] #[derive(Clone, Debug, Deserialize, Serialize, TS)]
pub struct NoteDetailExt { pub struct NoteDetailExt {
pub poll: Option<PackPollBase>, pub parent_note: Option<Box<PackNoteWithMaybeAttachments>>,
pub parent_note: Option<Box<NoteBase>>, pub renoted_note: Option<Box<PackNoteWithMaybeAttachments>>,
pub renoted_note: Option<Box<NoteBase>>,
} }
pack!( pack!(

View File

@ -1,5 +1,5 @@
use axum::extract::{Path, Query, State}; use axum::extract::{Path, Query, State};
use axum::{debug_handler, Json}; use axum::Json;
use std::sync::Arc; use std::sync::Arc;
use crate::model::processing::note::NoteModel; use crate::model::processing::note::NoteModel;
@ -11,7 +11,6 @@ use crate::service::MagnetarService;
use crate::web::auth::MaybeUser; use crate::web::auth::MaybeUser;
use crate::web::{ApiError, ObjectNotFound}; use crate::web::{ApiError, ObjectNotFound};
#[debug_handler]
pub async fn handle_note( pub async fn handle_note(
Path(id): Path<String>, Path(id): Path<String>,
Query(NoteByIdReq { Query(NoteByIdReq {
@ -22,8 +21,11 @@ pub async fn handle_note(
MaybeUser(self_user): MaybeUser, MaybeUser(self_user): MaybeUser,
) -> Result<Json<Res<GetNoteById>>, ApiError> { ) -> Result<Json<Res<GetNoteById>>, ApiError> {
let ctx = PackingContext::new(service, self_user.clone()).await?; let ctx = PackingContext::new(service, self_user.clone()).await?;
let note = NoteModel let note = NoteModel {
.fetch_single(&ctx, self_user.as_deref(), &id, context, attachments) attachments,
with_context: context,
}
.fetch_single(&ctx, self_user.as_deref(), &id)
.await? .await?
.ok_or_else(|| ObjectNotFound(id))?; .ok_or_else(|| ObjectNotFound(id))?;

View File

@ -1,5 +1,7 @@
use magnetar_calckey_model::ck; use magnetar_calckey_model::ck;
use magnetar_sdk::types::note::Reaction; use magnetar_sdk::types::note::{
NoteDetailExt, PackNoteWithMaybeAttachments, PackPollBase, Reaction,
};
use magnetar_sdk::types::user::PackUserBase; use magnetar_sdk::types::user::PackUserBase;
use magnetar_sdk::types::{ use magnetar_sdk::types::{
drive::PackDriveFileBase, drive::PackDriveFileBase,
@ -81,10 +83,39 @@ impl PackType<NoteBaseSource<'_>> for NoteBase {
} }
} }
impl PackType<&Vec<PackDriveFileBase>> for NoteAttachmentExt { pub struct NoteDetailSource<'a> {
fn extract(_context: &PackingContext, data: &Vec<PackDriveFileBase>) -> Self { pub parent_note: Option<&'a PackNoteWithMaybeAttachments>,
pub renoted_note: Option<&'a PackNoteWithMaybeAttachments>,
}
impl<'a> PackType<NoteDetailSource<'a>> for NoteDetailExt {
fn extract(
_context: &PackingContext,
NoteDetailSource {
parent_note,
renoted_note,
}: NoteDetailSource<'a>,
) -> Self {
NoteDetailExt {
parent_note: parent_note.cloned().map(Box::new),
renoted_note: renoted_note.cloned().map(Box::new),
}
}
}
pub struct NoteAttachmentSource<'a> {
pub attachments: &'a [PackDriveFileBase],
pub poll: Option<&'a PackPollBase>,
}
impl<'a> PackType<NoteAttachmentSource<'a>> for NoteAttachmentExt {
fn extract(
_context: &PackingContext,
NoteAttachmentSource { poll, attachments }: NoteAttachmentSource<'a>,
) -> Self {
NoteAttachmentExt { NoteAttachmentExt {
attachments: data.clone(), attachments: attachments.to_vec(),
poll: poll.cloned(),
} }
} }
} }

View File

@ -1,22 +1,32 @@
use crate::model::data::id::BaseId; use crate::model::data::id::BaseId;
use crate::model::data::note::NoteBaseSource; use crate::model::data::note::{NoteAttachmentSource, NoteBaseSource, NoteDetailSource};
use crate::model::processing::emoji::EmojiModel; use crate::model::processing::emoji::EmojiModel;
use crate::model::processing::user::UserModel; use crate::model::processing::user::UserModel;
use crate::model::processing::{get_mm_token_emoji, PackResult}; use crate::model::processing::{get_mm_token_emoji, PackResult};
use crate::model::{PackType, PackingContext, UserRelationship}; use crate::model::{PackType, PackingContext, UserRelationship};
use compact_str::CompactString; use compact_str::CompactString;
use either::Either; use either::Either;
use futures_util::future::try_join_all;
use magnetar_calckey_model::ck::sea_orm_active_enums::NoteVisibilityEnum; use magnetar_calckey_model::ck::sea_orm_active_enums::NoteVisibilityEnum;
use magnetar_calckey_model::note_model::{NoteResolveOptions, NoteVisibilityFilterFactory}; use magnetar_calckey_model::note_model::{
NoteData, NoteResolveOptions, NoteVisibilityFilterFactory,
};
use magnetar_calckey_model::sea_orm::prelude::Expr; use magnetar_calckey_model::sea_orm::prelude::Expr;
use magnetar_calckey_model::sea_orm::sea_query::{Alias, IntoIden, PgFunc, Query, SimpleExpr}; use magnetar_calckey_model::sea_orm::sea_query::{Alias, IntoIden, PgFunc, Query, SimpleExpr};
use magnetar_calckey_model::sea_orm::{ActiveEnum, ColumnTrait, IntoSimpleExpr}; use magnetar_calckey_model::sea_orm::{ActiveEnum, ColumnTrait, IntoSimpleExpr};
use magnetar_calckey_model::{ck, CalckeyDbError}; use magnetar_calckey_model::{ck, CalckeyDbError};
use magnetar_sdk::mmm::Token; use magnetar_sdk::mmm::Token;
use magnetar_sdk::types::drive::PackDriveFileBase;
use magnetar_sdk::types::emoji::EmojiContext; use magnetar_sdk::types::emoji::EmojiContext;
use magnetar_sdk::types::note::{NoteBase, PackNoteMaybeFull}; use magnetar_sdk::types::note::{
NoteAttachmentExt, NoteBase, NoteDetailExt, PackNoteBase, PackNoteMaybeFull,
PackNoteWithMaybeAttachments,
};
use magnetar_sdk::types::{Id, MmXml}; use magnetar_sdk::types::{Id, MmXml};
use magnetar_sdk::{mmm, Packed, Required}; use magnetar_sdk::{mmm, Packed, Required};
use tokio::try_join;
use super::drive::DriveModel;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct NoteVisibilityFilterSimple(Option<String>); pub struct NoteVisibilityFilterSimple(Option<String>);
@ -157,7 +167,10 @@ impl SpeechTransformNyan {
} }
} }
pub struct NoteModel; pub struct NoteModel {
pub attachments: bool,
pub with_context: bool,
}
impl NoteModel { impl NoteModel {
pub fn tokenize_note_text(&self, note: &ck::note::Model) -> Option<Token> { pub fn tokenize_note_text(&self, note: &ck::note::Model) -> Option<Token> {
@ -172,36 +185,18 @@ impl NoteModel {
.map(|text| mmm::Context::default().parse_ui(text)) .map(|text| mmm::Context::default().parse_ui(text))
} }
pub async fn fetch_single( pub async fn extract_base(
&self, &self,
ctx: &PackingContext, ctx: &PackingContext,
as_user: Option<&ck::user::Model>, note_data: &NoteData,
id: &str, ) -> PackResult<PackNoteBase> {
show_context: bool, let Required(ref user) = UserModel
attachments: bool, .base_from_existing(ctx, &note_data.user)
) -> PackResult<Option<PackNoteMaybeFull>> {
let note_resolver = ctx.service.db.get_note_resolver();
let Some(note) = note_resolver
.get_one(&NoteResolveOptions {
ids: Some(vec![id.to_owned()]),
visibility_filter: Box::new(
NoteVisibilityFilterModel
.new_note_visibility_filter(as_user.map(ck::user::Model::get_id)),
),
time_range: None,
with_user: show_context,
with_reply_target: show_context,
with_renote_target: show_context,
})
.await? .await?
else { .user;
return Ok(None);
};
let Required(ref user) = UserModel.base_from_existing(ctx, &note.user).await?.user; let cw_tok = self.tokenize_note_cw(&note_data.note);
let mut text_tok = self.tokenize_note_text(&note_data.note);
let cw_tok = self.tokenize_note_cw(&note.note);
let mut text_tok = self.tokenize_note_text(&note.note);
let mut emoji_extracted = Vec::new(); let mut emoji_extracted = Vec::new();
@ -212,7 +207,7 @@ impl NoteModel {
if let Some(text_tok) = &mut text_tok { if let Some(text_tok) = &mut text_tok {
emoji_extracted.extend_from_slice(&get_mm_token_emoji(text_tok)); emoji_extracted.extend_from_slice(&get_mm_token_emoji(text_tok));
if note.user.is_cat && note.user.speak_as_cat { if note_data.user.is_cat && note_data.user.speak_as_cat {
let transformer = SpeechTransformNyan::new(); let transformer = SpeechTransformNyan::new();
text_tok.walk_speech_transform(&|text| transformer.transform(text)); text_tok.walk_speech_transform(&|text| transformer.transform(text));
} }
@ -221,16 +216,14 @@ impl NoteModel {
let emoji_model = EmojiModel; let emoji_model = EmojiModel;
let shortcodes = emoji_model.deduplicate_emoji(ctx, emoji_extracted); let shortcodes = emoji_model.deduplicate_emoji(ctx, emoji_extracted);
let emojis = emoji_model let emojis = emoji_model
.fetch_many_emojis(ctx, &shortcodes, note.user.host.as_deref()) .fetch_many_emojis(ctx, &shortcodes, note_data.user.host.as_deref())
.await?; .await?;
let emoji_context = &EmojiContext(emojis); let emoji_context = &EmojiContext(emojis);
// TODO: Polls, reactions, attachments, ...
let note_base = NoteBase::extract( let note_base = NoteBase::extract(
ctx, ctx,
NoteBaseSource { NoteBaseSource {
note: &note.note, note: &note_data.note,
cw_mm: cw_tok cw_mm: cw_tok
.as_ref() .as_ref()
.map(mmm::to_xml_string) .map(mmm::to_xml_string)
@ -249,11 +242,192 @@ impl NoteModel {
}, },
); );
Ok(Some(PackNoteMaybeFull::pack_from(( Ok(PackNoteBase::pack_from((
Required(Id::from(note.note.id.clone())), Required(Id::from(note_data.note.id.clone())),
Required(note_base), Required(note_base),
None, )))
None, }
async fn extract_reply_target_base(
&self,
ctx: &PackingContext,
note: &NoteData,
) -> PackResult<Option<PackNoteBase>> {
match note.reply.as_ref() {
Some(r) if self.with_context => self.extract_base(ctx, r).await.map(Some),
_ => Ok(None),
}
}
async fn extract_renote_target_base(
&self,
ctx: &PackingContext,
note: &NoteData,
) -> PackResult<Option<PackNoteBase>> {
match note.renote.as_ref() {
Some(r) if self.with_context => self.extract_base(ctx, r).await.map(Some),
_ => Ok(None),
}
}
async fn extract_reply_target_attachemnts(
&self,
ctx: &PackingContext,
drive_model: &DriveModel,
note: &NoteData,
) -> PackResult<Option<Vec<PackDriveFileBase>>> {
match note.reply.as_ref() {
Some(r) if self.with_context => {
self.extract_attachments(ctx, drive_model, &r.note).await
}
_ => Ok(None),
}
}
async fn extract_renote_target_attachemnts(
&self,
ctx: &PackingContext,
drive_model: &DriveModel,
note: &NoteData,
) -> PackResult<Option<Vec<PackDriveFileBase>>> {
match note.renote.as_ref() {
Some(r) if self.with_context => {
self.extract_attachments(ctx, drive_model, &r.note).await
}
_ => Ok(None),
}
}
async fn extract_attachments(
&self,
ctx: &PackingContext,
drive_model: &DriveModel,
note: &ck::note::Model,
) -> PackResult<Option<Vec<PackDriveFileBase>>> {
if self.attachments {
let futures = try_join_all(
note.file_ids
.iter()
.map(|id| drive_model.get_cached_base(ctx, id)),
);
let att = futures.await?.into_iter().flatten().collect::<Vec<_>>();
return Ok(Some(att));
}
Ok(None)
}
pub async fn fetch_single(
&self,
ctx: &PackingContext,
as_user: Option<&ck::user::Model>,
id: &str,
) -> PackResult<Option<PackNoteMaybeFull>> {
let note_resolver = ctx.service.db.get_note_resolver();
let Some(note) = note_resolver
.get_one(&NoteResolveOptions {
ids: Some(vec![id.to_owned()]),
visibility_filter: Box::new(
NoteVisibilityFilterModel
.new_note_visibility_filter(as_user.map(ck::user::Model::get_id)),
),
time_range: None,
with_user: self.with_context,
with_reply_target: self.with_context,
with_renote_target: self.with_context,
})
.await?
else {
return Ok(None);
};
let drive_model = DriveModel;
let extract_reply_target = self.extract_reply_target_base(ctx, &note);
let extract_renote_target = self.extract_renote_target_base(ctx, &note);
let extract_attachments = self.extract_attachments(ctx, &drive_model, &note.note);
let extract_reply_attachments =
self.extract_reply_target_attachemnts(ctx, &drive_model, &note);
let extract_renote_attachments =
self.extract_renote_target_attachemnts(ctx, &drive_model, &note);
// TODO: Polls, reactions, ...
let (
PackNoteBase { id, note },
reply_target_pack,
renote_target_pack,
attachments_pack,
reply_attachments_pack,
renote_attachments_pack,
) = try_join!(
self.extract_base(ctx, &note),
extract_reply_target,
extract_renote_target,
extract_attachments,
extract_reply_attachments,
extract_renote_attachments
)?;
let detail = self.with_context.then(|| {
NoteDetailExt::extract(
ctx,
NoteDetailSource {
parent_note: reply_target_pack
.map(|base| {
PackNoteWithMaybeAttachments::pack_from((
base.id,
base.note,
reply_attachments_pack.map(|attachments| {
NoteAttachmentExt::extract(
ctx,
NoteAttachmentSource {
attachments: &attachments,
poll: None, // TODO
},
)
}),
))
})
.as_ref(),
renoted_note: renote_target_pack
.map(|base| {
PackNoteWithMaybeAttachments::pack_from((
base.id,
base.note,
renote_attachments_pack.map(|attachments| {
NoteAttachmentExt::extract(
ctx,
NoteAttachmentSource {
attachments: &attachments,
poll: None, // TODO
},
)
}),
))
})
.as_ref(),
},
)
});
let attachments = attachments_pack.map(|attachments| {
NoteAttachmentExt::extract(
ctx,
NoteAttachmentSource {
attachments: &attachments,
poll: None, // TODO
},
)
});
Ok(Some(PackNoteMaybeFull::pack_from((
id,
note,
attachments,
detail,
)))) ))))
} }
} }