natty
/
magnetar
1
0
Fork 0
Code Issues 4 Pull Requests 1 Packages Projects Releases Wiki Activity

Note context fetching

This commit is contained in:
Natty 2023-10-29 17:05:42 +01:00
parent 7dc38ada9a
commit 734ace5d05
Signed by: natty
GPG Key ID: BF6CB659ADEE60EC
7 changed files with 266 additions and 62 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Cargo.lock generated
View File

@ -1455,6 +1455,7 @@ dependencies = [
"compact_str",
"dotenvy",
"either",
"futures-util",
"headers",
"hyper",
"itertools 0.11.0",

8
Cargo.toml
View File

@ -15,7 +15,7 @@ members = [
"magnetar_common",
"magnetar_sdk",
"magnetar_mmm_parser",
"core"
"core",
]
[workspace.package]
@ -38,8 +38,6 @@ headers = "0.3"
http = "0.2"
hyper = "0.14"
itertools = "0.11"
js-sys = "0.3"
log = "0.4"
lru = "0.12"
miette = "5.9"
nom = "7"
@ -67,9 +65,6 @@ ts-rs = "7"
unicode-segmentation = "1.10"
url = "2.3"
walkdir = "2.3"
wasm-bindgen = "0.2"
wasm-bindgen-futures = "0.4"
web-sys = "0.3"
[dependencies]
magnetar_core = { path = "./core" }
@ -101,6 +96,7 @@ itertools = { workspace = true }
compact_str = { workspace = true }
either = { workspace = true }
futures-util = { workspace = true }
strum = { workspace = true, features = ["derive"] }
thiserror = { workspace = true }
miette = { workspace = true, features = ["fancy"] }

2
ext_calckey_model/src/note_model.rs
View File

@ -222,7 +222,7 @@ impl NoteResolver {
ALIAS_RENOTE.clone(),
)
.join_as(
JoinType::InnerJoin,
JoinType::LeftJoin,
note::Relation::User.with_alias(ALIAS_RENOTE.clone()),
ALIAS_RENOTE_USER.clone(),
);

10
magnetar_sdk/src/types/note.rs
View File

@ -76,19 +76,19 @@ pack!(PackNoteBase, Required<Id> as id & Required<NoteBase> as note);
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[ts(export)]
pub struct NoteAttachmentExt {
pub poll: Option<PackPollBase>,
pub attachments: Vec<PackDriveFileBase>,
}
pack!(
PackNoteWithAttachments,
Required<Id> as id & Required<NoteBase> as note & Required<NoteAttachmentExt> as attachment
PackNoteWithMaybeAttachments,
Required<Id> as id & Required<NoteBase> as note & Option<NoteAttachmentExt> as attachment
);
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
pub struct NoteDetailExt {
pub poll: Option<PackPollBase>,
pub parent_note: Option<Box<NoteBase>>,
pub renoted_note: Option<Box<NoteBase>>,
pub parent_note: Option<Box<PackNoteWithMaybeAttachments>>,
pub renoted_note: Option<Box<PackNoteWithMaybeAttachments>>,
}
pack!(

10
src/api_v1/note.rs
View File

@ -1,5 +1,5 @@
use axum::extract::{Path, Query, State};
use axum::{debug_handler, Json};
use axum::Json;
use std::sync::Arc;
use crate::model::processing::note::NoteModel;
@ -11,7 +11,6 @@ use crate::service::MagnetarService;
use crate::web::auth::MaybeUser;
use crate::web::{ApiError, ObjectNotFound};
#[debug_handler]
pub async fn handle_note(
Path(id): Path<String>,
Query(NoteByIdReq {
@ -22,8 +21,11 @@ pub async fn handle_note(
MaybeUser(self_user): MaybeUser,
) -> Result<Json<Res<GetNoteById>>, ApiError> {
let ctx = PackingContext::new(service, self_user.clone()).await?;
let note = NoteModel
.fetch_single(&ctx, self_user.as_deref(), &id, context, attachments)
let note = NoteModel {
attachments,
with_context: context,
}
.fetch_single(&ctx, self_user.as_deref(), &id)
.await?
.ok_or_else(|| ObjectNotFound(id))?;

41
src/model/data/note.rs
View File

@ -1,5 +1,7 @@
use magnetar_calckey_model::ck;
use magnetar_sdk::types::note::Reaction;
use magnetar_sdk::types::note::{
NoteDetailExt, PackNoteWithMaybeAttachments, PackPollBase, Reaction,
};
use magnetar_sdk::types::user::PackUserBase;
use magnetar_sdk::types::{
drive::PackDriveFileBase,
@ -81,10 +83,39 @@ impl PackType<NoteBaseSource<'_>> for NoteBase {
}
}
impl PackType<&Vec<PackDriveFileBase>> for NoteAttachmentExt {
fn extract(_context: &PackingContext, data: &Vec<PackDriveFileBase>) -> Self {
NoteAttachmentExt {
attachments: data.clone(),
pub struct NoteDetailSource<'a> {
pub parent_note: Option<&'a PackNoteWithMaybeAttachments>,
pub renoted_note: Option<&'a PackNoteWithMaybeAttachments>,
}
impl<'a> PackType<NoteDetailSource<'a>> for NoteDetailExt {
fn extract(
_context: &PackingContext,
NoteDetailSource {
parent_note,
renoted_note,
}: NoteDetailSource<'a>,
) -> Self {
NoteDetailExt {
parent_note: parent_note.cloned().map(Box::new),
renoted_note: renoted_note.cloned().map(Box::new),
}
}
}
pub struct NoteAttachmentSource<'a> {
pub attachments: &'a [PackDriveFileBase],
pub poll: Option<&'a PackPollBase>,
}
impl<'a> PackType<NoteAttachmentSource<'a>> for NoteAttachmentExt {
fn extract(
_context: &PackingContext,
NoteAttachmentSource { poll, attachments }: NoteAttachmentSource<'a>,
) -> Self {
NoteAttachmentExt {
attachments: attachments.to_vec(),
poll: poll.cloned(),
}
}
}

252
src/model/processing/note.rs
View File

@ -1,22 +1,32 @@
use crate::model::data::id::BaseId;
use crate::model::data::note::NoteBaseSource;
use crate::model::data::note::{NoteAttachmentSource, NoteBaseSource, NoteDetailSource};
use crate::model::processing::emoji::EmojiModel;
use crate::model::processing::user::UserModel;
use crate::model::processing::{get_mm_token_emoji, PackResult};
use crate::model::{PackType, PackingContext, UserRelationship};
use compact_str::CompactString;
use either::Either;
use futures_util::future::try_join_all;
use magnetar_calckey_model::ck::sea_orm_active_enums::NoteVisibilityEnum;
use magnetar_calckey_model::note_model::{NoteResolveOptions, NoteVisibilityFilterFactory};
use magnetar_calckey_model::note_model::{
NoteData, NoteResolveOptions, NoteVisibilityFilterFactory,
};
use magnetar_calckey_model::sea_orm::prelude::Expr;
use magnetar_calckey_model::sea_orm::sea_query::{Alias, IntoIden, PgFunc, Query, SimpleExpr};
use magnetar_calckey_model::sea_orm::{ActiveEnum, ColumnTrait, IntoSimpleExpr};
use magnetar_calckey_model::{ck, CalckeyDbError};
use magnetar_sdk::mmm::Token;
use magnetar_sdk::types::drive::PackDriveFileBase;
use magnetar_sdk::types::emoji::EmojiContext;
use magnetar_sdk::types::note::{NoteBase, PackNoteMaybeFull};
use magnetar_sdk::types::note::{
NoteAttachmentExt, NoteBase, NoteDetailExt, PackNoteBase, PackNoteMaybeFull,
PackNoteWithMaybeAttachments,
};
use magnetar_sdk::types::{Id, MmXml};
use magnetar_sdk::{mmm, Packed, Required};
use tokio::try_join;
use super::drive::DriveModel;
#[derive(Debug, Clone)]
pub struct NoteVisibilityFilterSimple(Option<String>);
@ -157,7 +167,10 @@ impl SpeechTransformNyan {
}
}
pub struct NoteModel;
pub struct NoteModel {
pub attachments: bool,
pub with_context: bool,
}
impl NoteModel {
pub fn tokenize_note_text(&self, note: &ck::note::Model) -> Option<Token> {
@ -172,36 +185,18 @@ impl NoteModel {
.map(|text| mmm::Context::default().parse_ui(text))
}
pub async fn fetch_single(
pub async fn extract_base(
&self,
ctx: &PackingContext,
as_user: Option<&ck::user::Model>,
id: &str,
show_context: bool,
attachments: bool,
) -> PackResult<Option<PackNoteMaybeFull>> {
let note_resolver = ctx.service.db.get_note_resolver();
let Some(note) = note_resolver
.get_one(&NoteResolveOptions {
ids: Some(vec![id.to_owned()]),
visibility_filter: Box::new(
NoteVisibilityFilterModel
.new_note_visibility_filter(as_user.map(ck::user::Model::get_id)),
),
time_range: None,
with_user: show_context,
with_reply_target: show_context,
with_renote_target: show_context,
})
note_data: &NoteData,
) -> PackResult<PackNoteBase> {
let Required(ref user) = UserModel
.base_from_existing(ctx, &note_data.user)
.await?
else {
return Ok(None);
};
.user;
let Required(ref user) = UserModel.base_from_existing(ctx, &note.user).await?.user;
let cw_tok = self.tokenize_note_cw(&note.note);
let mut text_tok = self.tokenize_note_text(&note.note);
let cw_tok = self.tokenize_note_cw(&note_data.note);
let mut text_tok = self.tokenize_note_text(&note_data.note);
let mut emoji_extracted = Vec::new();
@ -212,7 +207,7 @@ impl NoteModel {
if let Some(text_tok) = &mut text_tok {
emoji_extracted.extend_from_slice(&get_mm_token_emoji(text_tok));
if note.user.is_cat && note.user.speak_as_cat {
if note_data.user.is_cat && note_data.user.speak_as_cat {
let transformer = SpeechTransformNyan::new();
text_tok.walk_speech_transform(&|text| transformer.transform(text));
}
@ -221,16 +216,14 @@ impl NoteModel {
let emoji_model = EmojiModel;
let shortcodes = emoji_model.deduplicate_emoji(ctx, emoji_extracted);
let emojis = emoji_model
.fetch_many_emojis(ctx, &shortcodes, note.user.host.as_deref())
.fetch_many_emojis(ctx, &shortcodes, note_data.user.host.as_deref())
.await?;
let emoji_context = &EmojiContext(emojis);
// TODO: Polls, reactions, attachments, ...
let note_base = NoteBase::extract(
ctx,
NoteBaseSource {
note: &note.note,
note: &note_data.note,
cw_mm: cw_tok
.as_ref()
.map(mmm::to_xml_string)
@ -249,11 +242,192 @@ impl NoteModel {
},
);
Ok(Some(PackNoteMaybeFull::pack_from((
Required(Id::from(note.note.id.clone())),
Ok(PackNoteBase::pack_from((
Required(Id::from(note_data.note.id.clone())),
Required(note_base),
None,
None,
)))
}
async fn extract_reply_target_base(
&self,
ctx: &PackingContext,
note: &NoteData,
) -> PackResult<Option<PackNoteBase>> {
match note.reply.as_ref() {
Some(r) if self.with_context => self.extract_base(ctx, r).await.map(Some),
_ => Ok(None),
}
}
async fn extract_renote_target_base(
&self,
ctx: &PackingContext,
note: &NoteData,
) -> PackResult<Option<PackNoteBase>> {
match note.renote.as_ref() {
Some(r) if self.with_context => self.extract_base(ctx, r).await.map(Some),
_ => Ok(None),
}
}
async fn extract_reply_target_attachemnts(
&self,
ctx: &PackingContext,
drive_model: &DriveModel,
note: &NoteData,
) -> PackResult<Option<Vec<PackDriveFileBase>>> {
match note.reply.as_ref() {
Some(r) if self.with_context => {
self.extract_attachments(ctx, drive_model, &r.note).await
}
_ => Ok(None),
}
}
async fn extract_renote_target_attachemnts(
&self,
ctx: &PackingContext,
drive_model: &DriveModel,
note: &NoteData,
) -> PackResult<Option<Vec<PackDriveFileBase>>> {
match note.renote.as_ref() {
Some(r) if self.with_context => {
self.extract_attachments(ctx, drive_model, &r.note).await
}
_ => Ok(None),
}
}
async fn extract_attachments(
&self,
ctx: &PackingContext,
drive_model: &DriveModel,
note: &ck::note::Model,
) -> PackResult<Option<Vec<PackDriveFileBase>>> {
if self.attachments {
let futures = try_join_all(
note.file_ids
.iter()
.map(|id| drive_model.get_cached_base(ctx, id)),
);
let att = futures.await?.into_iter().flatten().collect::<Vec<_>>();
return Ok(Some(att));
}
Ok(None)
}
pub async fn fetch_single(
&self,
ctx: &PackingContext,
as_user: Option<&ck::user::Model>,
id: &str,
) -> PackResult<Option<PackNoteMaybeFull>> {
let note_resolver = ctx.service.db.get_note_resolver();
let Some(note) = note_resolver
.get_one(&NoteResolveOptions {
ids: Some(vec![id.to_owned()]),
visibility_filter: Box::new(
NoteVisibilityFilterModel
.new_note_visibility_filter(as_user.map(ck::user::Model::get_id)),
),
time_range: None,
with_user: self.with_context,
with_reply_target: self.with_context,
with_renote_target: self.with_context,
})
.await?
else {
return Ok(None);
};
let drive_model = DriveModel;
let extract_reply_target = self.extract_reply_target_base(ctx, &note);
let extract_renote_target = self.extract_renote_target_base(ctx, &note);
let extract_attachments = self.extract_attachments(ctx, &drive_model, &note.note);
let extract_reply_attachments =
self.extract_reply_target_attachemnts(ctx, &drive_model, &note);
let extract_renote_attachments =
self.extract_renote_target_attachemnts(ctx, &drive_model, &note);
// TODO: Polls, reactions, ...
let (
PackNoteBase { id, note },
reply_target_pack,
renote_target_pack,
attachments_pack,
reply_attachments_pack,
renote_attachments_pack,
) = try_join!(
self.extract_base(ctx, &note),
extract_reply_target,
extract_renote_target,
extract_attachments,
extract_reply_attachments,
extract_renote_attachments
)?;
let detail = self.with_context.then(|| {
NoteDetailExt::extract(
ctx,
NoteDetailSource {
parent_note: reply_target_pack
.map(|base| {
PackNoteWithMaybeAttachments::pack_from((
base.id,
base.note,
reply_attachments_pack.map(|attachments| {
NoteAttachmentExt::extract(
ctx,
NoteAttachmentSource {
attachments: &attachments,
poll: None, // TODO
},
)
}),
))
})
.as_ref(),
renoted_note: renote_target_pack
.map(|base| {
PackNoteWithMaybeAttachments::pack_from((
base.id,
base.note,
renote_attachments_pack.map(|attachments| {
NoteAttachmentExt::extract(
ctx,
NoteAttachmentSource {
attachments: &attachments,
poll: None, // TODO
},
)
}),
))
})
.as_ref(),
},
)
});
let attachments = attachments_pack.map(|attachments| {
NoteAttachmentExt::extract(
ctx,
NoteAttachmentSource {
attachments: &attachments,
poll: None, // TODO
},
)
});
Ok(Some(PackNoteMaybeFull::pack_from((
id,
note,
attachments,
detail,
))))
}
}