natty
/
magnetar
1
0
Fork 0
Code Issues 4 Pull Requests 1 Packages Projects Releases Wiki Activity

Note by ID testing endpoint

This commit is contained in:
Natty 2023-10-29 02:10:48 +01:00
parent 4bbc368f92
commit 0755dac002
Signed by: natty
GPG Key ID: BF6CB659ADEE60EC
16 changed files with 274 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Cargo.lock generated
View File

@ -1452,6 +1452,7 @@ dependencies = [
"cached",
"cfg-if",
"chrono",
"compact_str",
"dotenvy",
"either",
"headers",
@ -1466,6 +1467,7 @@ dependencies = [
"magnetar_webfinger",
"miette",
"percent-encoding",
"regex",
"serde",
"serde_json",
"strum",

2
Cargo.toml
View File

@ -99,10 +99,12 @@ cfg-if = { workspace = true }
itertools = { workspace = true }
compact_str = { workspace = true }
either = { workspace = true }
strum = { workspace = true, features = ["derive"] }
thiserror = { workspace = true }
miette = { workspace = true, features = ["fancy"] }
regex = { workspace = true }
percent-encoding = { workspace = true }

5
ext_calckey_model/src/lib.rs
View File

@ -4,6 +4,7 @@ pub use ck;
use ck::*;
pub use sea_orm;
use crate::note_model::NoteResolver;
use chrono::Utc;
use ext_calckey_model_migration::{Migrator, MigratorTrait};
use futures_util::StreamExt;
@ -286,6 +287,10 @@ impl CalckeyModel {
Ok(meta)
}
pub fn get_note_resolver(&self) -> NoteResolver {
NoteResolver::new(self.clone())
}
}
#[derive(Debug)]

54
ext_calckey_model/src/note_model.rs
View File

@ -21,17 +21,17 @@ pub struct NoteData {
pub renote: Option<Box<NoteData>>,
}
const USER: &str = "user";
const USER_AVATAR: &str = "user.avatar";
const USER_BANNER: &str = "user.banner";
const REPLY: &str = "reply";
const REPLY_USER: &str = "reply.user";
const REPLY_USER_AVATAR: &str = "reply.user.avatar";
const REPLY_USER_BANNER: &str = "reply.user.banner";
const RENOTE: &str = "renote";
const RENOTE_USER: &str = "renote.user";
const RENOTE_USER_AVATAR: &str = "renote.user.avatar";
const RENOTE_USER_BANNER: &str = "renote.user.banner";
const USER: &str = "user.";
const USER_AVATAR: &str = "user.avatar.";
const USER_BANNER: &str = "user.banner.";
const REPLY: &str = "reply.";
const REPLY_USER: &str = "reply.user.";
const REPLY_USER_AVATAR: &str = "reply.user.avatar.";
const REPLY_USER_BANNER: &str = "reply.user.banner.";
const RENOTE: &str = "renote.";
const RENOTE_USER: &str = "renote.user.";
const RENOTE_USER_AVATAR: &str = "renote.user.avatar.";
const RENOTE_USER_BANNER: &str = "renote.user.banner.";
impl FromQueryResult for NoteData {
fn from_query_result(res: &QueryResult, _pre: &str) -> Result<Self, DbErr> {
@ -76,16 +76,17 @@ pub struct NoteResolver {
db: CalckeyModel,
}
pub trait NoteVisibilityFilterFactory {
fn with_note_and_user_tables(&self, note: Option<Alias>, user: Option<Alias>) -> SimpleExpr;
pub trait NoteVisibilityFilterFactory: Send + Sync {
fn with_note_and_user_tables(&self, note: Option<Alias>) -> SimpleExpr;
}
pub struct NoteResolveOptions {
visibility_filter: Box<dyn NoteVisibilityFilterFactory>,
time_range: Option<RangeFilter>,
with_user: bool,
with_reply_target: bool,
with_renote_target: bool,
pub ids: Option<Vec<String>>,
pub visibility_filter: Box<dyn NoteVisibilityFilterFactory>,
pub time_range: Option<RangeFilter>,
pub with_user: bool,
pub with_reply_target: bool,
pub with_renote_target: bool,
}
trait SelectColumnsExt {
@ -130,14 +131,16 @@ const ALIAS_RENOTE_USER_AVATAR: Lazy<Alias> = Lazy::new(|| Alias::new(RENOTE_USE
const ALIAS_RENOTE_USER_BANNER: Lazy<Alias> = Lazy::new(|| Alias::new(RENOTE_USER_BANNER));
impl NoteResolver {
pub fn new(db: CalckeyModel) -> Self {
NoteResolver { db }
}
pub async fn get_one(
&self,
options: &NoteResolveOptions,
) -> Result<Option<NoteData>, CalckeyDbError> {
let select = self.resolve(options);
let visibility_filter = options
.visibility_filter
.with_note_and_user_tables(None, Some(ALIAS_USER.clone()));
let visibility_filter = options.visibility_filter.with_note_and_user_tables(None);
let time_filter = options.time_range.as_ref().map(|f| match f {
RangeFilter::TimeStart(start) => note::Column::CreatedAt.gte(start.clone()),
RangeFilter::TimeRange(range) => {
@ -146,8 +149,17 @@ impl NoteResolver {
RangeFilter::TimeEnd(end) => note::Column::CreatedAt.lt(end.clone()),
});
let id_filter = options.ids.as_ref().map(|ids| {
if ids.len() == 1 {
note::Column::Id.eq(&ids[0])
} else {
note::Column::Id.is_in(ids)
}
});
let notes = select
.filter(visibility_filter)
.apply_if(id_filter, Select::<note::Entity>::filter)
.apply_if(time_filter, Select::<note::Entity>::filter)
.into_model::<NoteData>()
.one(self.db.inner())

19
magnetar_mmm_parser/src/lib.rs
View File

@ -255,6 +255,25 @@ impl Token {
}
}
pub fn walk_speech_transform(&mut self, func: &impl Fn(&mut CompactString)) {
match self {
Token::Sequence(items) => {
items
.iter_mut()
.for_each(|tok| tok.walk_speech_transform(func));
}
Token::Small(inner)
| Token::BoldItalic(inner)
| Token::Bold(inner)
| Token::Italic(inner)
| Token::Center(inner)
| Token::Function { inner, .. }
| Token::Strikethrough(inner) => inner.walk_speech_transform(func),
Token::PlainText(text) => func(text),
_ => {}
}
}
fn write<T: Write>(&self, writer: &mut quick_xml::Writer<T>) -> quick_xml::Result<()> {
match self {
Token::PlainText(plain) => {

1
magnetar_sdk/src/endpoints/mod.rs
View File

@ -1,3 +1,4 @@
pub mod note;
pub mod timeline;
pub mod user;

25
magnetar_sdk/src/endpoints/note.rs Normal file
View File

@ -0,0 +1,25 @@
use crate::endpoints::Endpoint;
use crate::types::note::PackNoteMaybeFull;
use http::Method;
use magnetar_sdk_macros::Endpoint;
use serde::{Deserialize, Serialize};
use ts_rs::TS;
// Get note by id
#[derive(Serialize, Deserialize, TS)]
#[ts(export)]
pub struct NoteByIdReq {
#[serde(default)]
pub context: bool,
#[serde(default)]
pub attachments: bool,
}
#[derive(Endpoint)]
#[endpoint(
endpoint = "/notes/:id",
method = Method::GET,
request = NoteByIdReq,
response = PackNoteMaybeFull
)]
pub struct GetNoteById;

6
magnetar_sdk/src/endpoints/timeline.rs
View File

@ -1,5 +1,5 @@
use crate::endpoints::Endpoint;
use crate::types::note::{NoteListFilter, PackNoteFull};
use crate::types::note::{NoteListFilter, PackNoteMaybeFull};
use crate::util_types::U64Range;
use http::Method;
use magnetar_sdk_macros::Endpoint;
@ -23,13 +23,13 @@ fn default_timeline_limit<const MIN: u64, const MAX: u64>() -> U64Range<MIN, MAX
#[derive(Serialize, Deserialize, TS)]
#[ts(export)]
#[repr(transparent)]
pub struct GetTimelineRes(pub Vec<PackNoteFull>);
pub struct GetTimelineRes(pub Vec<PackNoteMaybeFull>);
#[derive(Endpoint)]
#[endpoint(
endpoint = "/timeline",
method = Method::GET,
request = GetTimelineReq,
response = Vec::<PackNoteFull>,
response = Vec::<PackNoteMaybeFull>,
)]
pub struct GetTimeline;

4
magnetar_sdk/src/endpoints/user.rs
View File

@ -22,7 +22,7 @@ pub struct UserSelfReq {
#[derive(Endpoint)]
#[endpoint(
endpoint = "/users/@self/overview/info",
endpoint = "/users/@self",
method = Method::GET,
request = UserSelfReq,
response = PackUserSelfMaybeAll
@ -47,7 +47,7 @@ pub struct UserByIdReq {
#[derive(Endpoint)]
#[endpoint(
endpoint = "/users/:user_id/info",
endpoint = "/users/:user_id",
method = Method::GET,
request = UserByIdReq,
response = PackUserMaybeAll

6
magnetar_sdk/src/types/note.rs
View File

@ -56,7 +56,7 @@ pub struct NoteBase {
pub uri: Option<String>,
pub url: Option<String>,
pub text: String,
pub text_mm: MmXml,
pub text_mm: Option<MmXml>,
pub visibility: NoteVisibility,
pub user: Box<PackUserBase>,
pub parent_note_id: Option<String>,
@ -92,8 +92,8 @@ pub struct NoteDetailExt {
}
pack!(
PackNoteFull,
Required<Id> as id & Required<NoteBase> as note & Required<NoteAttachmentExt> as attachment & Required<NoteDetailExt> as detail
PackNoteMaybeFull,
Required<Id> as id & Required<NoteBase> as note & Option<NoteAttachmentExt> as attachment & Option<NoteDetailExt> as detail
);
#[derive(Clone, Debug, Deserialize, Serialize, TS)]

4
magnetar_sdk/src/types/user.rs
View File

@ -5,7 +5,7 @@ use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use ts_rs::TS;
use crate::types::note::PackNoteFull;
use crate::types::note::PackNoteMaybeFull;
use magnetar_sdk_macros::pack;
#[derive(Clone, Debug, Default, Deserialize, Serialize, TS)]
@ -88,7 +88,7 @@ pub struct UserProfileExt {
#[derive(Clone, Debug, Deserialize, Serialize, TS)]
#[ts(export)]
pub struct UserProfilePinsEx {
pub pinned_notes: Vec<PackNoteFull>,
pub pinned_notes: Vec<PackNoteMaybeFull>,
// pub pinned_page: Option<Page>,
}

3
src/api_v1/mod.rs
View File

@ -1,5 +1,7 @@
mod note;
mod user;
use crate::api_v1::note::handle_note;
use crate::api_v1::user::{handle_user_info, handle_user_info_self};
use crate::service::MagnetarService;
use crate::web::auth;
@ -13,6 +15,7 @@ pub fn create_api_router(service: Arc<MagnetarService>) -> Router {
Router::new()
.route("/users/@self", get(handle_user_info_self))
.route("/users/:id", get(handle_user_info))
.route("/notes/:id", get(handle_note))
.layer(from_fn_with_state(
AuthState::new(service.clone()),
auth::auth,

31
src/api_v1/note.rs Normal file
View File

@ -0,0 +1,31 @@
use axum::extract::{Path, Query, State};
use axum::{debug_handler, Json};
use std::sync::Arc;
use crate::model::processing::note::NoteModel;
use crate::model::PackingContext;
use magnetar_sdk::endpoints::note::{GetNoteById, NoteByIdReq};
use magnetar_sdk::endpoints::{Req, Res};
use crate::service::MagnetarService;
use crate::web::auth::MaybeUser;
use crate::web::{ApiError, ObjectNotFound};
#[debug_handler]
pub async fn handle_note(
Path(id): Path<String>,
Query(NoteByIdReq {
context,
attachments,
}): Query<Req<GetNoteById>>,
State(service): State<Arc<MagnetarService>>,
MaybeUser(self_user): MaybeUser,
) -> Result<Json<Res<GetNoteById>>, ApiError> {
let ctx = PackingContext::new(service, self_user.clone()).await?;
let note = NoteModel
.fetch_single(&ctx, self_user.as_deref(), &id, context, attachments)
.await?
.ok_or_else(|| ObjectNotFound(id))?;
Ok(Json(note.into()))
}

4
src/model/data/note.rs
View File

@ -29,7 +29,7 @@ impl PackType<&ck::note_reaction::Model> for ReactionBase {
pub struct NoteBaseSource<'a> {
pub note: &'a ck::note::Model,
pub cw_mm: Option<&'a MmXml>,
pub text_mm: &'a MmXml,
pub text_mm: Option<&'a MmXml>,
pub reactions: &'a Vec<PackReactionBase>,
pub user: &'a UserBase,
pub emoji_context: &'a EmojiContext,
@ -55,7 +55,7 @@ impl PackType<NoteBaseSource<'_>> for NoteBase {
uri: note.uri.clone(),
url: note.url.clone(),
text: note.text.clone().unwrap_or_default(),
text_mm: text_mm.clone(),
text_mm: text_mm.map(ToOwned::to_owned).clone(),
visibility: match note.visibility {
NVE::Followers => NoteVisibility::Followers,
NVE::Hidden => NoteVisibility::Direct,

6
src/model/data/user.rs
View File

@ -1,7 +1,7 @@
use magnetar_calckey_model::ck;
use magnetar_calckey_model::ck::sea_orm_active_enums::UserProfileFfvisibilityEnum;
use magnetar_sdk::types::emoji::{EmojiContext, PackEmojiBase};
use magnetar_sdk::types::note::PackNoteFull;
use magnetar_sdk::types::note::PackNoteMaybeFull;
use magnetar_sdk::types::user::{
AvatarDecoration, PackSecurityKeyBase, ProfileField, SecurityKeyBase, SpeechTransform,
UserBase, UserDetailExt, UserProfileExt, UserProfilePinsEx, UserRelationExt, UserSecretsExt,
@ -187,8 +187,8 @@ impl PackType<UserRelationExtSource<'_>> for UserRelationExt {
}
}
impl PackType<&[PackNoteFull]> for UserProfilePinsEx {
fn extract(_context: &PackingContext, pinned_notes: &[PackNoteFull]) -> Self {
impl PackType<&[PackNoteMaybeFull]> for UserProfilePinsEx {
fn extract(_context: &PackingContext, pinned_notes: &[PackNoteMaybeFull]) -> Self {
UserProfilePinsEx {
pinned_notes: pinned_notes.to_owned(),
}

154
src/model/processing/note.rs
View File

@ -1,37 +1,44 @@
use crate::model::{PackingContext, UserRelationship};
use crate::model::data::id::BaseId;
use crate::model::data::note::NoteBaseSource;
use crate::model::processing::emoji::EmojiModel;
use crate::model::processing::user::UserModel;
use crate::model::processing::{get_mm_token_emoji, PackResult};
use crate::model::{PackType, PackingContext, UserRelationship};
use compact_str::CompactString;
use either::Either;
use magnetar_calckey_model::ck::sea_orm_active_enums::NoteVisibilityEnum;
use magnetar_calckey_model::note_model::NoteVisibilityFilterFactory;
use magnetar_calckey_model::note_model::{NoteResolveOptions, NoteVisibilityFilterFactory};
use magnetar_calckey_model::sea_orm::prelude::Expr;
use magnetar_calckey_model::sea_orm::sea_query::{Alias, IntoIden, PgFunc, Query, SimpleExpr};
use magnetar_calckey_model::sea_orm::{ColumnTrait, IntoSimpleExpr};
use magnetar_calckey_model::sea_orm::{ActiveEnum, ColumnTrait, IntoSimpleExpr};
use magnetar_calckey_model::{ck, CalckeyDbError};
use magnetar_sdk::mmm::Token;
use magnetar_sdk::types::emoji::EmojiContext;
use magnetar_sdk::types::note::{NoteBase, PackNoteMaybeFull};
use magnetar_sdk::types::{Id, MmXml};
use magnetar_sdk::{mmm, Packed, Required};
#[derive(Debug, Clone)]
pub struct NoteVisibilityFilterSimple(Option<String>);
impl NoteVisibilityFilterFactory for NoteVisibilityFilterSimple {
fn with_note_and_user_tables(
&self,
note_tbl: Option<Alias>,
user_tbl: Option<Alias>,
) -> SimpleExpr {
fn with_note_and_user_tables(&self, note_tbl: Option<Alias>) -> SimpleExpr {
let note_tbl_name =
note_tbl.map_or_else(|| ck::note::Entity.into_iden(), |a| a.into_iden());
let user_tbl_name =
user_tbl.map_or_else(|| ck::user::Entity.into_iden(), |a| a.into_iden());
let note_visibility = Expr::col((note_tbl_name.clone(), ck::note::Column::Visibility));
let note_mentions = Expr::col((note_tbl_name.clone(), ck::note::Column::Mentions));
let note_reply_user_id = Expr::col((note_tbl_name.clone(), ck::note::Column::ReplyUserId));
let note_visible_user_ids =
Expr::col((note_tbl_name.clone(), ck::note::Column::VisibleUserIds));
let note_user_id = Expr::col((user_tbl_name, ck::note::Column::UserId));
let note_user_id = Expr::col((note_tbl_name, ck::note::Column::UserId));
let is_public = note_visibility
.clone()
.eq(NoteVisibilityEnum::Public)
.or(note_visibility.clone().eq(NoteVisibilityEnum::Home));
.eq(NoteVisibilityEnum::Public.as_enum())
.or(note_visibility
.clone()
.eq(NoteVisibilityEnum::Home.as_enum()));
let Some(user_id_str) = &self.0 else {
return is_public;
@ -44,8 +51,10 @@ impl NoteVisibilityFilterFactory for NoteVisibilityFilterSimple {
let is_visible_specified = {
let either_specified_or_followers = note_visibility
.clone()
.eq(NoteVisibilityEnum::Specified)
.or(note_visibility.clone().eq(NoteVisibilityEnum::Followers))
.eq(NoteVisibilityEnum::Specified.as_enum())
.or(note_visibility
.clone()
.eq(NoteVisibilityEnum::Followers.as_enum()))
.into_simple_expr();
let mentioned_or_specified = self_user_id
@ -58,7 +67,7 @@ impl NoteVisibilityFilterFactory for NoteVisibilityFilterSimple {
let is_visible_followers = {
note_visibility
.eq(NoteVisibilityEnum::Followers)
.eq(NoteVisibilityEnum::Followers.as_enum())
.and(
note_user_id.in_subquery(
Query::select()
@ -135,3 +144,116 @@ impl NoteVisibilityFilterModel {
NoteVisibilityFilterSimple(user.map(str::to_string))
}
}
struct SpeechTransformNyan;
impl SpeechTransformNyan {
fn new() -> Self {
SpeechTransformNyan
}
fn transform(&self, text: &mut CompactString) {
// TODO
}
}
pub struct NoteModel;
impl NoteModel {
pub fn tokenize_note_text(&self, note: &ck::note::Model) -> Option<Token> {
note.text
.as_deref()
.map(|text| mmm::Context::default().parse_full(text))
}
pub fn tokenize_note_cw(&self, note: &ck::note::Model) -> Option<Token> {
note.cw
.as_deref()
.map(|text| mmm::Context::default().parse_ui(text))
}
pub async fn fetch_single(
&self,
ctx: &PackingContext,
as_user: Option<&ck::user::Model>,
id: &str,
show_context: bool,
attachments: bool,
) -> PackResult<Option<PackNoteMaybeFull>> {
let note_resolver = ctx.service.db.get_note_resolver();
let Some(note) = note_resolver
.get_one(&NoteResolveOptions {
ids: Some(vec![id.to_owned()]),
visibility_filter: Box::new(
NoteVisibilityFilterModel
.new_note_visibility_filter(as_user.map(ck::user::Model::get_id)),
),
time_range: None,
with_user: show_context,
with_reply_target: show_context,
with_renote_target: show_context,
})
.await?
else {
return Ok(None);
};
let Required(ref user) = UserModel.base_from_existing(ctx, &note.user).await?.user;
let cw_tok = self.tokenize_note_cw(&note.note);
let mut text_tok = self.tokenize_note_text(&note.note);
let mut emoji_extracted = Vec::new();
if let Some(cw_tok) = &cw_tok {
emoji_extracted.extend_from_slice(&get_mm_token_emoji(cw_tok));
}
if let Some(text_tok) = &mut text_tok {
emoji_extracted.extend_from_slice(&get_mm_token_emoji(text_tok));
if note.user.is_cat && note.user.speak_as_cat {
let transformer = SpeechTransformNyan::new();
text_tok.walk_speech_transform(&|text| transformer.transform(text));
}
}
let emoji_model = EmojiModel;
let shortcodes = emoji_model.deduplicate_emoji(ctx, emoji_extracted);
let emojis = emoji_model
.fetch_many_emojis(ctx, &shortcodes, note.user.host.as_deref())
.await?;
let emoji_context = &EmojiContext(emojis);
// TODO: Polls, reactions, attachments, ...
let note_base = NoteBase::extract(
ctx,
NoteBaseSource {
note: &note.note,
cw_mm: cw_tok
.as_ref()
.map(mmm::to_xml_string)
.and_then(Result::ok)
.map(MmXml)
.as_ref(),
text_mm: text_tok
.as_ref()
.map(mmm::to_xml_string)
.and_then(Result::ok)
.map(MmXml)
.as_ref(),
reactions: &vec![],
user,
emoji_context,
},
);
Ok(Some(PackNoteMaybeFull::pack_from((
Required(Id::from(note.note.id.clone())),
Required(note_base),
None,
None,
))))
}
}