From 87e9fb36e168d31eec7b0cf1bc50a28c74cdce0a Mon Sep 17 00:00:00 2001 From: Natty Date: Thu, 5 Sep 2024 17:22:54 +0200 Subject: [PATCH] Fixed some past crimes Mainly fetching pins in a subquery instead of joining Parsing needs a bigger overhaul --- ext_model/src/note_model/mod.rs | 91 ++++++++++++++-------------- magnetar_mmm_parser/src/lib.rs | 87 ++++++-------------------- src/model/processing/note.rs | 8 +-- src/model/processing/notification.rs | 6 +- src/service/emoji_cache.rs | 4 +- src/service/generic_id_cache.rs | 4 +- src/service/instance_cache.rs | 4 +- src/service/instance_meta_cache.rs | 4 +- src/service/local_user_cache.rs | 6 +- 9 files changed, 81 insertions(+), 133 deletions(-) diff --git a/ext_model/src/note_model/mod.rs b/ext_model/src/note_model/mod.rs index 2fce228..eb56818 100644 --- a/ext_model/src/note_model/mod.rs +++ b/ext_model/src/note_model/mod.rs @@ -1,16 +1,12 @@ pub mod data; -use ext_model_migration::SelectStatement; -use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr}; -use sea_orm::{ - Condition, EntityTrait, Iden, JoinType, QueryFilter, QueryOrder, QuerySelect, QueryTrait, - Select, -}; -use std::sync::Arc; - use ck::{note, note_reaction, user_note_pining}; use data::{sub_interaction_reaction, sub_interaction_renote, NoteData}; +use ext_model_migration::SelectStatement; use magnetar_sdk::types::SpanFilter; +use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr}; +use sea_orm::{Condition, EntityTrait, Iden, JoinType, Order, QueryFilter, QuerySelect, QueryTrait, Select}; +use std::sync::Arc; use crate::model_ext::{ join_columns_default, AliasColumnExt, AliasSourceExt, AliasSuffixExt, CursorPaginationExt, @@ -19,10 +15,9 @@ use crate::model_ext::{ use crate::user_model::{UserResolveOptions, UserResolver}; use crate::{CalckeyDbError, CalckeyModel}; -const PIN: &str = "pin."; -const INTERACTION_REACTION: &str = "interaction.reaction."; -const INTERACTION_RENOTE: &str = "interaction.renote."; -const USER: &str = "user."; +const INTERACTION_REACTION: &str = "iact.rct."; +const INTERACTION_RENOTE: &str = "iact.rnt."; +const USER: &str = "u."; const REPLY: &str = "reply."; const RENOTE: &str = "renote."; @@ -36,16 +31,50 @@ pub trait NoteVisibilityFilterFactory: Send + Sync { fn with_note_and_user_tables(&self, note: &dyn Iden) -> SimpleExpr; } +#[derive(Clone)] +pub enum NoteResolveMode { + Single(String), + Multiple(Vec), + PinsFromUserId(String), +} + +impl NoteResolveMode { + fn as_expr(&self) -> SimpleExpr { + let id_col = note::Entity.base_prefix().col(note::Column::Id); + + match self { + NoteResolveMode::Single(id) => id_col.eq(id), + NoteResolveMode::Multiple(ids) => id_col.is_in(ids), + NoteResolveMode::PinsFromUserId(user_id) => { + let sub_query = Query::select() + .expr(SelectExpr { + expr: Expr::col(user_note_pining::Column::NoteId).into(), + alias: None, + window: None, + }) + .from(user_note_pining::Entity) + .and_where(Expr::col(note_reaction::Column::UserId).eq(user_id)) + .order_by_columns([ + (user_note_pining::Column::CreatedAt, Order::Desc), + (user_note_pining::Column::Id, Order::Desc) + ]) + .take(); + + id_col.in_subquery(sub_query) + } + } + } +} + #[derive(Clone)] pub struct NoteResolveOptions { - pub ids: Option>, + pub mode: Option, pub visibility_filter: Arc, pub time_range: Option, pub limit: Option, pub with_reply_target: bool, pub with_renote_target: bool, pub with_interactions_from: Option, // User ID - pub only_pins_from: Option, // User ID pub user_options: UserResolveOptions, } @@ -57,7 +86,7 @@ trait SelectNoteInteractionsExt { ) -> &mut Self; fn add_sub_select_interaction_renote(&mut self, note_tbl: &MagIden, user_id: &str) - -> &mut Self; + -> &mut Self; } impl SelectNoteInteractionsExt for SelectStatement { @@ -117,16 +146,6 @@ impl SelectNoteInteractionsExt for SelectStatement { } } -fn ids_into_expr(ids: &Vec) -> SimpleExpr { - let id_col = note::Entity.base_prefix().col(note::Column::Id); - - if ids.len() == 1 { - id_col.eq(&ids[0]) - } else { - id_col.is_in(ids) - } -} - impl NoteResolver { pub fn new(db: CalckeyModel, user_resolver: UserResolver) -> Self { NoteResolver { db, user_resolver } @@ -141,7 +160,7 @@ impl NoteResolver { .visibility_filter .with_note_and_user_tables(¬e::Entity.base_prefix()); - let id_filter = options.ids.as_ref().map(ids_into_expr); + let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr); let note = select .filter(visibility_filter) @@ -161,17 +180,11 @@ impl NoteResolver { let visibility_filter = options .visibility_filter .with_note_and_user_tables(¬e::Entity.base_prefix()); - let id_filter = options.ids.as_ref().map(ids_into_expr); + let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr); let notes_select = select .filter(visibility_filter) - .apply_if(id_filter, Select::::filter) - .apply_if(options.only_pins_from.as_deref(), |s, _| { - s.order_by_desc(Expr::col(( - note::Entity.base_prefix().into_iden().join_str(PIN), - user_note_pining::Column::CreatedAt, - ))) - }); + .apply_if(id_filter, Select::::filter); let notes = if let Some(pagination) = &options.time_range { notes_select @@ -304,16 +317,6 @@ impl NoteResolver { let mut select = Query::select(); select.from_as(note::Entity, note_tbl.clone().into_iden()); - if let Some(pins_user) = &options.only_pins_from { - select - .join_columns( - JoinType::InnerJoin, - note::Relation::UserNotePining.with_from_alias(¬e_tbl), - ¬e_tbl.join_str(PIN), - ) - .and_where(note_tbl.col(user_note_pining::Column::UserId).eq(pins_user)); - } - self.attach_note( &mut select, ¬e_tbl, diff --git a/magnetar_mmm_parser/src/lib.rs b/magnetar_mmm_parser/src/lib.rs index 8f2563f..18c6fa5 100644 --- a/magnetar_mmm_parser/src/lib.rs +++ b/magnetar_mmm_parser/src/lib.rs @@ -5,7 +5,6 @@ use std::marker::PhantomData; use compact_str::{CompactString, ToCompactString}; use either::Either; -use nom::{IResult, Offset, Parser, Slice}; use nom::branch::alt; use nom::bytes::complete::{tag, tag_no_case}; use nom::character::complete::{ @@ -16,6 +15,7 @@ use nom::combinator::{eof, fail, map, not, opt, peek, recognize}; use nom::error::ErrorKind; use nom::multi::{many0_count, many1, many1_count, many_till, separated_list1}; use nom::sequence::tuple; +use nom::{IResult, Offset, Parser, Slice}; use nom_locate::LocatedSpan; use quick_xml::events::{BytesText, Event}; use serde::{Deserialize, Serialize}; @@ -56,7 +56,6 @@ pub enum Token { Sequence(Vec), Quote(Box), Small(Box), - BoldItalic(Box), Bold(Box), Italic(Box), Center(Box), @@ -101,7 +100,6 @@ impl Token { Token::Sequence(tokens) => tokens.first().and_then(Token::str_content_left), Token::Quote(inner) => inner.str_content_left(), Token::Small(inner) => inner.str_content_left(), - Token::BoldItalic(inner) => inner.str_content_left(), Token::Bold(inner) => inner.str_content_left(), Token::Italic(inner) => inner.str_content_left(), Token::Center(inner) => inner.str_content_left(), @@ -124,7 +122,6 @@ impl Token { Token::Sequence(tokens) => tokens.last().and_then(Token::str_content_right), Token::Quote(inner) => inner.str_content_right(), Token::Small(inner) => inner.str_content_right(), - Token::BoldItalic(inner) => inner.str_content_right(), Token::Bold(inner) => inner.str_content_right(), Token::Italic(inner) => inner.str_content_right(), Token::Center(inner) => inner.str_content_right(), @@ -147,7 +144,6 @@ impl Token { sequence @ Token::Sequence(_) => sequence.clone(), Token::Quote(inner) => inner.inner(), Token::Small(inner) => inner.inner(), - Token::BoldItalic(inner) => inner.inner(), Token::Bold(inner) => inner.inner(), Token::Italic(inner) => inner.inner(), Token::Center(inner) => inner.inner(), @@ -213,7 +209,6 @@ impl Token { } Token::Quote(inner) => Token::Quote(Box::new(inner.merged())), Token::Small(inner) => Token::Small(Box::new(inner.merged())), - Token::BoldItalic(inner) => Token::BoldItalic(Box::new(inner.merged())), Token::Bold(inner) => Token::Bold(Box::new(inner.merged())), Token::Italic(inner) => Token::Italic(Box::new(inner.merged())), Token::Center(inner) => Token::Center(Box::new(inner.merged())), @@ -247,7 +242,6 @@ impl Token { } Token::Quote(inner) | Token::Small(inner) - | Token::BoldItalic(inner) | Token::Bold(inner) | Token::Italic(inner) | Token::Center(inner) @@ -266,7 +260,6 @@ impl Token { .for_each(|tok| tok.walk_speech_transform(func)); } Token::Small(inner) - | Token::BoldItalic(inner) | Token::Bold(inner) | Token::Italic(inner) | Token::Center(inner) @@ -295,16 +288,6 @@ impl Token { .create_element("small") .write_inner_content(|w| inner.write(w))?; } - Token::BoldItalic(inner) => { - writer - .create_element("b") - .write_inner_content::<_, quick_xml::Error>(|w| { - w.create_element("i") - .write_inner_content(|w| inner.write(w))?; - - Ok(()) - })?; - } Token::Bold(inner) => { writer .create_element("b") @@ -468,8 +451,8 @@ trait SliceOffset { fn up_to(&self, other: &Self) -> Self; fn fragment_between<'a>(&self, other: &Self) -> &'a str - where - Self: 'a; + where + Self: 'a; } impl SliceOffset for Span<'_> { @@ -478,8 +461,8 @@ impl SliceOffset for Span<'_> { } fn fragment_between<'a>(&self, other: &Self) -> &'a str - where - Self: 'a, + where + Self: 'a, { self.up_to(other).into_fragment() } @@ -792,8 +775,6 @@ impl Context { fn base_bold_italic<'a>(&self, input: Span<'a>) -> IResult, Token> { alt(( - self.partial(Self::tag_bold_italic_asterisk), - self.partial(Self::tag_bold_italic_underscore), self.partial(Self::tag_bold_asterisk), self.partial(Self::tag_italic_asterisk), self.partial(Self::tag_bold_underscore), @@ -1033,10 +1014,10 @@ impl Context { escape: bool, matcher: Matcher<'a, 'b, T>, fallback: Matcher<'a, 'b, S>, - ) -> impl Fn(Span<'b>) -> IResult, Token> + '_ - where - FOpen: Fn(Span<'b>) -> IResult, Span<'b>> + 'a, - FClose: Fn(Span<'b>) -> IResult, Span<'b>> + 'a, + ) -> impl Fn(Span<'b>) -> IResult, Token> + 'a + where + FOpen: Fn(Span<'b>) -> IResult, Span<'b>> + 'a, + FClose: Fn(Span<'b>) -> IResult, Span<'b>> + 'a, { let FlankingDelim(opening_tag, opening_rule, ..) = opening_tag.into(); let FlankingDelim(closing_tag, closing_rule, ..) = closing_tag.into(); @@ -1223,38 +1204,6 @@ impl Context { )(input) } - fn tag_bold_italic_asterisk<'a>(&self, input: Span<'a>) -> IResult, Token> { - self.tag_delimited( - (tag("***"), FlankingRule::Lenient), - (tag("***"), FlankingRule::Lenient), - true, - Matcher::new( - &self.partial(Self::inline_single), - &collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)), - ), - Matcher::new( - &self.partial(Self::inline_non_formatting_single), - &collect_sequence(Token::Sequence, identity), - ), - )(input) - } - - fn tag_bold_italic_underscore<'a>(&self, input: Span<'a>) -> IResult, Token> { - self.tag_delimited( - (tag("___"), FlankingRule::Strict), - (tag("___"), FlankingRule::Strict), - true, - Matcher::new( - &self.partial(Self::inline_single), - &collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)), - ), - Matcher::new( - &self.partial(Self::inline_non_formatting_single), - &collect_sequence(Token::Sequence, identity), - ), - )(input) - } - fn tag_bold<'a>(&self, input: Span<'a>) -> IResult, Token> { self.tag_delimited( tag_no_case(""), @@ -1498,7 +1447,7 @@ impl Context { return fail(input); }; - let grapheme = grapheme.trim_end_matches(|c| c == '\u{200c}' || c == '\u{200d}'); + let grapheme = grapheme.trim_end_matches(['\u{200c}', '\u{200d}']); let emoji = emojis::get(grapheme); @@ -1567,7 +1516,7 @@ impl Context { })(input)?; let (input, name) = map( - recognize(many1(alt((alphanumeric1, recognize(one_of("-_")))))), + recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))), Span::into_fragment, )(input)?; @@ -1591,7 +1540,7 @@ impl Context { mention_type }; let host = - host_opt.map(|(_, name)| name.trim_end_matches(|c| matches!(c, '.' | '-' | '_'))); + host_opt.map(|(_, name)| name.trim_end_matches(['.', '-', '_'])); let input = host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before); Ok(( @@ -1642,8 +1591,8 @@ impl Context { &'b self, mut func: F, ) -> impl FnMut(Span<'a>) -> IResult, O> + 'b - where - F: Parser, O, nom::error::Error>> + 'b, + where + F: Parser, O, nom::error::Error>> + 'b, { move |mut input| { if input.extra.depth >= self.depth_limit { @@ -1723,8 +1672,8 @@ impl Context { mut terminator: F, spaces: bool, ) -> impl FnMut(Span<'a>) -> IResult, Span<'a>> + 'b - where - F: Parser, Span<'a>, nom::error::Error>> + 'b, + where + F: Parser, Span<'a>, nom::error::Error>> + 'b, { move |input| { recognize(many1_count(tuple(( @@ -1746,7 +1695,7 @@ mod test { use nom::bytes::complete::tag; - use crate::{Context, DEFAULT_DEPTH_LIMIT, Span, SpanMeta, to_xml_string, Token}; + use crate::{to_xml_string, Context, Span, SpanMeta, Token, DEFAULT_DEPTH_LIMIT}; fn parse_full(string: &str) -> Token { Context::default() @@ -1904,7 +1853,7 @@ mod test { assert_eq!( parse_full(r#"***bold italic***"#), - Token::BoldItalic(Box::new(Token::PlainText("bold italic".into()))) + Token::Bold(Box::new(Token::Italic(Box::new(Token::PlainText("bold italic".into()))))) ); assert_eq!( diff --git a/src/model/processing/note.rs b/src/model/processing/note.rs index b2edfb4..cfed656 100644 --- a/src/model/processing/note.rs +++ b/src/model/processing/note.rs @@ -13,7 +13,7 @@ use magnetar_model::model_ext::AliasColumnExt; use magnetar_model::note_model::data::{ sub_interaction_reaction, sub_interaction_renote, NoteData, }; -use magnetar_model::note_model::{NoteResolveOptions, NoteVisibilityFilterFactory}; +use magnetar_model::note_model::{NoteResolveMode, NoteResolveOptions, NoteVisibilityFilterFactory}; use magnetar_model::poll::PollResolver; use magnetar_model::sea_orm::sea_query::{PgFunc, Query, SimpleExpr}; use magnetar_model::sea_orm::{ActiveEnum, ColumnTrait, Iden, IntoSimpleExpr}; @@ -518,7 +518,7 @@ impl NoteModel { let note_resolver = ctx.service.db.get_note_resolver(); let Some(note) = note_resolver .get_one(&NoteResolveOptions { - ids: Some(vec![id.to_owned()]), + mode: Some(NoteResolveMode::Single(id.to_owned())), visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter( ctx.self_user.as_deref().map(ck::user::Model::get_id), )), @@ -540,7 +540,6 @@ impl NoteModel { .map(str::to_string) }) .flatten(), - only_pins_from: None, }) .await? else { @@ -558,7 +557,7 @@ impl NoteModel { let note_resolver = ctx.service.db.get_note_resolver(); let notes = note_resolver .get_many(&NoteResolveOptions { - ids: None, + mode: Some(NoteResolveMode::PinsFromUserId(pin_user.id.clone())), visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter( ctx.self_user.as_deref().map(ck::user::Model::get_id), )), @@ -580,7 +579,6 @@ impl NoteModel { .map(str::to_string) }) .flatten(), - only_pins_from: Some(pin_user.id.clone()), }) .await?; diff --git a/src/model/processing/notification.rs b/src/model/processing/notification.rs index 56a0143..fd16b31 100644 --- a/src/model/processing/notification.rs +++ b/src/model/processing/notification.rs @@ -231,7 +231,7 @@ impl NotificationModel { .get_single( &NotificationResolveOptions { note_options: NoteResolveOptions { - ids: None, + mode: None, visibility_filter: Arc::new( NoteVisibilityFilterModel.new_note_visibility_filter(Some(user_id)), ), @@ -240,7 +240,6 @@ impl NotificationModel { with_reply_target: true, with_renote_target: true, with_interactions_from: self_id.map(str::to_string), - only_pins_from: None, user_options: user_resolve_options.clone(), }, user_options: user_resolve_options, @@ -294,7 +293,7 @@ impl NotificationModel { .get( &NotificationResolveOptions { note_options: NoteResolveOptions { - ids: None, + mode: None, visibility_filter: Arc::new( NoteVisibilityFilterModel.new_note_visibility_filter(Some(id)), ), @@ -303,7 +302,6 @@ impl NotificationModel { with_reply_target: true, with_renote_target: true, with_interactions_from: self_id.map(str::to_string), - only_pins_from: None, user_options: user_resolve_options.clone(), }, user_options: user_resolve_options, diff --git a/src/service/emoji_cache.rs b/src/service/emoji_cache.rs index 2a8ebcc..116b637 100644 --- a/src/service/emoji_cache.rs +++ b/src/service/emoji_cache.rs @@ -4,11 +4,11 @@ use magnetar_model::emoji::{EmojiResolver, EmojiTag}; use magnetar_model::{ck, CalckeyDbError, CalckeyModel}; use std::collections::HashSet; use std::sync::Arc; -use strum::EnumVariantNames; +use strum::VariantNames; use thiserror::Error; use tokio::sync::Mutex; -#[derive(Debug, Error, EnumVariantNames)] +#[derive(Debug, Error, VariantNames)] pub enum EmojiCacheError { #[error("Database error: {0}")] DbError(#[from] CalckeyDbError), diff --git a/src/service/generic_id_cache.rs b/src/service/generic_id_cache.rs index f77748f..01c3f00 100644 --- a/src/service/generic_id_cache.rs +++ b/src/service/generic_id_cache.rs @@ -5,11 +5,11 @@ use magnetar_model::{CalckeyDbError, CalckeyModel}; use std::marker::PhantomData; use std::sync::Arc; use std::time::{Duration, Instant}; -use strum::EnumVariantNames; +use strum::VariantNames; use thiserror::Error; use tokio::sync::Mutex; -#[derive(Debug, Error, EnumVariantNames)] +#[derive(Debug, Error, VariantNames)] pub enum GenericIdCacheError { #[error("Database error: {0}")] DbError(#[from] CalckeyDbError), diff --git a/src/service/instance_cache.rs b/src/service/instance_cache.rs index b667609..2d12662 100644 --- a/src/service/instance_cache.rs +++ b/src/service/instance_cache.rs @@ -4,11 +4,11 @@ use magnetar_common::config::MagnetarConfig; use magnetar_model::{ck, CalckeyDbError, CalckeyModel}; use std::sync::Arc; use std::time::{Duration, Instant}; -use strum::EnumVariantNames; +use strum::VariantNames; use thiserror::Error; use tokio::sync::Mutex; -#[derive(Debug, Error, EnumVariantNames)] +#[derive(Debug, Error, VariantNames)] pub enum RemoteInstanceCacheError { #[error("Database error: {0}")] DbError(#[from] CalckeyDbError), diff --git a/src/service/instance_meta_cache.rs b/src/service/instance_meta_cache.rs index 10815cb..63a0fdd 100644 --- a/src/service/instance_meta_cache.rs +++ b/src/service/instance_meta_cache.rs @@ -2,12 +2,12 @@ use crate::web::ApiError; use magnetar_model::{ck, CalckeyDbError, CalckeyModel}; use std::sync::Arc; use std::time::{Duration, Instant}; -use strum::EnumVariantNames; +use strum::VariantNames; use thiserror::Error; use tokio::sync::{mpsc, oneshot}; use tracing::error; -#[derive(Debug, Error, EnumVariantNames)] +#[derive(Debug, Error, VariantNames)] pub enum InstanceMetaCacheError { #[error("Database error: {0}")] DbError(#[from] CalckeyDbError), diff --git a/src/service/local_user_cache.rs b/src/service/local_user_cache.rs index fda63f6..d0cc1d1 100644 --- a/src/service/local_user_cache.rs +++ b/src/service/local_user_cache.rs @@ -2,20 +2,20 @@ use std::collections::HashMap; use std::sync::Arc; use cached::{Cached, TimedCache}; -use strum::EnumVariantNames; +use strum::{EnumVariantNames, VariantNames}; use thiserror::Error; use tokio::sync::Mutex; use tracing::error; use magnetar_common::config::MagnetarConfig; use magnetar_model::{ - CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub, ck, + ck, CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub, InternalStreamMessage, SubMessage, }; use crate::web::ApiError; -#[derive(Debug, Error, EnumVariantNames)] +#[derive(Debug, Error, VariantNames)] pub enum UserCacheError { #[error("Database error: {0}")] DbError(#[from] CalckeyDbError),