Fixed some past crimes

Mainly fetching pins in a subquery instead of joining

Parsing needs a bigger overhaul

Cargo.toml

@@ -30,31 +30,33 @@ async-stream = "0.3"
 axum = "0.7"
 axum-extra = "0.9"
 base64 = "0.22"
-cached = "0.47"
+cached = "0.53"
 cfg-if = "1"
 chrono = "0.4"
-compact_str = "0.7"
+compact_str = "0.8"
 dotenvy = "0.15"
 ed25519-dalek = "2.1"
 either = "1.9"
 emojis = "0.6"
 futures = "0.3"
+futures-channel = "0.3"
 futures-core = "0.3"
 futures-util = "0.3"
 headers = "0.4"
 http = "1.0"
 httpdate = "1"
 hyper = "1.1"
-idna = "0.5"
+idna = "1"
 indexmap = "2.2"
-itertools = "0.12"
+itertools = "0.13"
 lru = "0.12"
-miette = "5.9"
+miette = "7"
 nom = "7"
 nom_locate = "4"
 percent-encoding = "2.2"
-quick-xml = "0.31"
-redis = "0.24"
+priority-queue = "2.0"
+quick-xml = "0.36"
+redis = "0.26"
 regex = "1.9"
 rsa = "0.9"
 reqwest = "0.12"
@@ -64,14 +66,15 @@ serde = "1"
 serde_json = "1"
 serde_urlencoded = "0.7"
 sha2 = "0.10"
-strum = "0.25"
+smallvec = "1.13"
+strum = "0.26"
 tera = { version = "1", default-features = false }
 thiserror = "1"
 tokio = "1.24"
 tokio-util = "0.7"
 tokio-stream = "0.1"
 toml = "0.8"
-tower = "0.4"
+tower = "0.5"
 tower-http = "0.5"
 tracing = "0.1"
 tracing-subscriber = "0.3"

ext_model/src/note_model/mod.rs

@@ -1,16 +1,12 @@
 pub mod data;
 
-use ext_model_migration::SelectStatement;
-use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr};
-use sea_orm::{
-    Condition, EntityTrait, Iden, JoinType, QueryFilter, QueryOrder, QuerySelect, QueryTrait,
-    Select,
-};
-use std::sync::Arc;
-
 use ck::{note, note_reaction, user_note_pining};
 use data::{sub_interaction_reaction, sub_interaction_renote, NoteData};
+use ext_model_migration::SelectStatement;
 use magnetar_sdk::types::SpanFilter;
+use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr};
+use sea_orm::{Condition, EntityTrait, Iden, JoinType, Order, QueryFilter, QuerySelect, QueryTrait, Select};
+use std::sync::Arc;
 
 use crate::model_ext::{
     join_columns_default, AliasColumnExt, AliasSourceExt, AliasSuffixExt, CursorPaginationExt,
@@ -19,10 +15,9 @@ use crate::model_ext::{
 use crate::user_model::{UserResolveOptions, UserResolver};
 use crate::{CalckeyDbError, CalckeyModel};
 
-const PIN: &str = "pin.";
-const INTERACTION_REACTION: &str = "interaction.reaction.";
-const INTERACTION_RENOTE: &str = "interaction.renote.";
-const USER: &str = "user.";
+const INTERACTION_REACTION: &str = "iact.rct.";
+const INTERACTION_RENOTE: &str = "iact.rnt.";
+const USER: &str = "u.";
 const REPLY: &str = "reply.";
 const RENOTE: &str = "renote.";
 
@@ -36,16 +31,50 @@ pub trait NoteVisibilityFilterFactory: Send + Sync {
     fn with_note_and_user_tables(&self, note: &dyn Iden) -> SimpleExpr;
 }
 
+#[derive(Clone)]
+pub enum NoteResolveMode {
+    Single(String),
+    Multiple(Vec<String>),
+    PinsFromUserId(String),
+}
+
+impl NoteResolveMode {
+    fn as_expr(&self) -> SimpleExpr {
+        let id_col = note::Entity.base_prefix().col(note::Column::Id);
+
+        match self {
+            NoteResolveMode::Single(id) => id_col.eq(id),
+            NoteResolveMode::Multiple(ids) => id_col.is_in(ids),
+            NoteResolveMode::PinsFromUserId(user_id) => {
+                let sub_query = Query::select()
+                    .expr(SelectExpr {
+                        expr: Expr::col(user_note_pining::Column::NoteId).into(),
+                        alias: None,
+                        window: None,
+                    })
+                    .from(user_note_pining::Entity)
+                    .and_where(Expr::col(note_reaction::Column::UserId).eq(user_id))
+                    .order_by_columns([
+                        (user_note_pining::Column::CreatedAt, Order::Desc),
+                        (user_note_pining::Column::Id, Order::Desc)
+                    ])
+                    .take();
+
+                id_col.in_subquery(sub_query)
+            }
+        }
+    }
+}
+
 #[derive(Clone)]
 pub struct NoteResolveOptions {
-    pub ids: Option<Vec<String>>,
+    pub mode: Option<NoteResolveMode>,
     pub visibility_filter: Arc<dyn NoteVisibilityFilterFactory>,
     pub time_range: Option<SpanFilter>,
     pub limit: Option<u64>,
     pub with_reply_target: bool,
     pub with_renote_target: bool,
     pub with_interactions_from: Option<String>, // User ID
-    pub only_pins_from: Option<String>,         // User ID
     pub user_options: UserResolveOptions,
 }
 
@@ -117,16 +146,6 @@ impl SelectNoteInteractionsExt for SelectStatement {
     }
 }
 
-fn ids_into_expr(ids: &Vec<String>) -> SimpleExpr {
-    let id_col = note::Entity.base_prefix().col(note::Column::Id);
-
-    if ids.len() == 1 {
-        id_col.eq(&ids[0])
-    } else {
-        id_col.is_in(ids)
-    }
-}
-
 impl NoteResolver {
     pub fn new(db: CalckeyModel, user_resolver: UserResolver) -> Self {
         NoteResolver { db, user_resolver }
@@ -141,7 +160,7 @@ impl NoteResolver {
             .visibility_filter
             .with_note_and_user_tables(&note::Entity.base_prefix());
 
-        let id_filter = options.ids.as_ref().map(ids_into_expr);
+        let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr);
 
         let note = select
             .filter(visibility_filter)
@@ -161,17 +180,11 @@ impl NoteResolver {
         let visibility_filter = options
             .visibility_filter
             .with_note_and_user_tables(&note::Entity.base_prefix());
-        let id_filter = options.ids.as_ref().map(ids_into_expr);
+        let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr);
 
         let notes_select = select
             .filter(visibility_filter)
-            .apply_if(id_filter, Select::<note::Entity>::filter)
-            .apply_if(options.only_pins_from.as_deref(), |s, _| {
-                s.order_by_desc(Expr::col((
-                    note::Entity.base_prefix().into_iden().join_str(PIN),
-                    user_note_pining::Column::CreatedAt,
-                )))
-            });
+            .apply_if(id_filter, Select::<note::Entity>::filter);
 
         let notes = if let Some(pagination) = &options.time_range {
             notes_select
@@ -304,16 +317,6 @@ impl NoteResolver {
         let mut select = Query::select();
         select.from_as(note::Entity, note_tbl.clone().into_iden());
 
-        if let Some(pins_user) = &options.only_pins_from {
-            select
-                .join_columns(
-                    JoinType::InnerJoin,
-                    note::Relation::UserNotePining.with_from_alias(&note_tbl),
-                    &note_tbl.join_str(PIN),
-                )
-                .and_where(note_tbl.col(user_note_pining::Column::UserId).eq(pins_user));
-        }
-
         self.attach_note(
             &mut select,
             &note_tbl,

magnetar_mmm_parser/Cargo.toml

@@ -18,4 +18,4 @@ serde = { workspace = true, features = ["derive"] }
 strum = { workspace = true, features = ["derive"] }
 tracing = { workspace = true }
 unicode-segmentation = { workspace = true }
-quick-xml = { workspace = "true", optional = true, features = ["serialize"] }
+quick-xml = { workspace = true, optional = true, features = ["serialize"] }

magnetar_mmm_parser/src/lib.rs

@@ -5,7 +5,6 @@ use std::marker::PhantomData;
 
 use compact_str::{CompactString, ToCompactString};
 use either::Either;
-use nom::{IResult, Offset, Parser, Slice};
 use nom::branch::alt;
 use nom::bytes::complete::{tag, tag_no_case};
 use nom::character::complete::{
@@ -16,6 +15,7 @@ use nom::combinator::{eof, fail, map, not, opt, peek, recognize};
 use nom::error::ErrorKind;
 use nom::multi::{many0_count, many1, many1_count, many_till, separated_list1};
 use nom::sequence::tuple;
+use nom::{IResult, Offset, Parser, Slice};
 use nom_locate::LocatedSpan;
 use quick_xml::events::{BytesText, Event};
 use serde::{Deserialize, Serialize};
@@ -56,7 +56,6 @@ pub enum Token {
     Sequence(Vec<Token>),
     Quote(Box<Token>),
     Small(Box<Token>),
-    BoldItalic(Box<Token>),
     Bold(Box<Token>),
     Italic(Box<Token>),
     Center(Box<Token>),
@@ -101,7 +100,6 @@ impl Token {
             Token::Sequence(tokens) => tokens.first().and_then(Token::str_content_left),
             Token::Quote(inner) => inner.str_content_left(),
             Token::Small(inner) => inner.str_content_left(),
-            Token::BoldItalic(inner) => inner.str_content_left(),
             Token::Bold(inner) => inner.str_content_left(),
             Token::Italic(inner) => inner.str_content_left(),
             Token::Center(inner) => inner.str_content_left(),
@@ -124,7 +122,6 @@ impl Token {
             Token::Sequence(tokens) => tokens.last().and_then(Token::str_content_right),
             Token::Quote(inner) => inner.str_content_right(),
             Token::Small(inner) => inner.str_content_right(),
-            Token::BoldItalic(inner) => inner.str_content_right(),
             Token::Bold(inner) => inner.str_content_right(),
             Token::Italic(inner) => inner.str_content_right(),
             Token::Center(inner) => inner.str_content_right(),
@@ -147,7 +144,6 @@ impl Token {
             sequence @ Token::Sequence(_) => sequence.clone(),
             Token::Quote(inner) => inner.inner(),
             Token::Small(inner) => inner.inner(),
-            Token::BoldItalic(inner) => inner.inner(),
             Token::Bold(inner) => inner.inner(),
             Token::Italic(inner) => inner.inner(),
             Token::Center(inner) => inner.inner(),
@@ -213,7 +209,6 @@ impl Token {
             }
             Token::Quote(inner) => Token::Quote(Box::new(inner.merged())),
             Token::Small(inner) => Token::Small(Box::new(inner.merged())),
-            Token::BoldItalic(inner) => Token::BoldItalic(Box::new(inner.merged())),
             Token::Bold(inner) => Token::Bold(Box::new(inner.merged())),
             Token::Italic(inner) => Token::Italic(Box::new(inner.merged())),
             Token::Center(inner) => Token::Center(Box::new(inner.merged())),
@@ -247,7 +242,6 @@ impl Token {
             }
             Token::Quote(inner)
             | Token::Small(inner)
-            | Token::BoldItalic(inner)
             | Token::Bold(inner)
             | Token::Italic(inner)
             | Token::Center(inner)
@@ -266,7 +260,6 @@ impl Token {
                     .for_each(|tok| tok.walk_speech_transform(func));
             }
             Token::Small(inner)
-            | Token::BoldItalic(inner)
             | Token::Bold(inner)
             | Token::Italic(inner)
             | Token::Center(inner)
@@ -295,16 +288,6 @@ impl Token {
                     .create_element("small")
                     .write_inner_content(|w| inner.write(w))?;
             }
-            Token::BoldItalic(inner) => {
-                writer
-                    .create_element("b")
-                    .write_inner_content::<_, quick_xml::Error>(|w| {
-                        w.create_element("i")
-                            .write_inner_content(|w| inner.write(w))?;
-
-                        Ok(())
-                    })?;
-            }
             Token::Bold(inner) => {
                 writer
                     .create_element("b")
@@ -792,8 +775,6 @@ impl Context {
 
     fn base_bold_italic<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
         alt((
-            self.partial(Self::tag_bold_italic_asterisk),
-            self.partial(Self::tag_bold_italic_underscore),
             self.partial(Self::tag_bold_asterisk),
             self.partial(Self::tag_italic_asterisk),
             self.partial(Self::tag_bold_underscore),
@@ -1033,7 +1014,7 @@ impl Context {
         escape: bool,
         matcher: Matcher<'a, 'b, T>,
         fallback: Matcher<'a, 'b, S>,
-    ) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token> + '_
+    ) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token> + 'a
     where
         FOpen: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
         FClose: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
@@ -1223,38 +1204,6 @@ impl Context {
         )(input)
     }
 
-    fn tag_bold_italic_asterisk<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
-        self.tag_delimited(
-            (tag("***"), FlankingRule::Lenient),
-            (tag("***"), FlankingRule::Lenient),
-            true,
-            Matcher::new(
-                &self.partial(Self::inline_single),
-                &collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
-            ),
-            Matcher::new(
-                &self.partial(Self::inline_non_formatting_single),
-                &collect_sequence(Token::Sequence, identity),
-            ),
-        )(input)
-    }
-
-    fn tag_bold_italic_underscore<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
-        self.tag_delimited(
-            (tag("___"), FlankingRule::Strict),
-            (tag("___"), FlankingRule::Strict),
-            true,
-            Matcher::new(
-                &self.partial(Self::inline_single),
-                &collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
-            ),
-            Matcher::new(
-                &self.partial(Self::inline_non_formatting_single),
-                &collect_sequence(Token::Sequence, identity),
-            ),
-        )(input)
-    }
-
     fn tag_bold<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
         self.tag_delimited(
             tag_no_case("<b>"),
@@ -1498,7 +1447,7 @@ impl Context {
             return fail(input);
         };
 
-        let grapheme = grapheme.trim_end_matches(|c| c == '\u{200c}' || c == '\u{200d}');
+        let grapheme = grapheme.trim_end_matches(['\u{200c}', '\u{200d}']);
 
         let emoji = emojis::get(grapheme);
 
@@ -1567,7 +1516,7 @@ impl Context {
         })(input)?;
 
         let (input, name) = map(
-            recognize(many1(alt((alphanumeric1, recognize(one_of("-_")))))),
+            recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))),
             Span::into_fragment,
         )(input)?;
 
@@ -1591,7 +1540,7 @@ impl Context {
             mention_type
         };
         let host =
-            host_opt.map(|(_, name)| name.trim_end_matches(|c| matches!(c, '.' | '-' | '_')));
+            host_opt.map(|(_, name)| name.trim_end_matches(['.', '-', '_']));
         let input = host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before);
 
         Ok((
@@ -1746,7 +1695,7 @@ mod test {
 
     use nom::bytes::complete::tag;
 
-    use crate::{Context, DEFAULT_DEPTH_LIMIT, Span, SpanMeta, to_xml_string, Token};
+    use crate::{to_xml_string, Context, Span, SpanMeta, Token, DEFAULT_DEPTH_LIMIT};
 
     fn parse_full(string: &str) -> Token {
         Context::default()
@@ -1904,7 +1853,7 @@ mod test {
 
         assert_eq!(
             parse_full(r#"***bold italic***"#),
-            Token::BoldItalic(Box::new(Token::PlainText("bold italic".into())))
+            Token::Bold(Box::new(Token::Italic(Box::new(Token::PlainText("bold italic".into())))))
         );
 
         assert_eq!(

src/model/processing/note.rs

@@ -13,7 +13,7 @@ use magnetar_model::model_ext::AliasColumnExt;
 use magnetar_model::note_model::data::{
     sub_interaction_reaction, sub_interaction_renote, NoteData,
 };
-use magnetar_model::note_model::{NoteResolveOptions, NoteVisibilityFilterFactory};
+use magnetar_model::note_model::{NoteResolveMode, NoteResolveOptions, NoteVisibilityFilterFactory};
 use magnetar_model::poll::PollResolver;
 use magnetar_model::sea_orm::sea_query::{PgFunc, Query, SimpleExpr};
 use magnetar_model::sea_orm::{ActiveEnum, ColumnTrait, Iden, IntoSimpleExpr};
@@ -518,7 +518,7 @@ impl NoteModel {
         let note_resolver = ctx.service.db.get_note_resolver();
         let Some(note) = note_resolver
             .get_one(&NoteResolveOptions {
-                ids: Some(vec![id.to_owned()]),
+                mode: Some(NoteResolveMode::Single(id.to_owned())),
                 visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter(
                     ctx.self_user.as_deref().map(ck::user::Model::get_id),
                 )),
@@ -540,7 +540,6 @@ impl NoteModel {
                             .map(str::to_string)
                     })
                     .flatten(),
-                only_pins_from: None,
             })
             .await?
         else {
@@ -558,7 +557,7 @@ impl NoteModel {
         let note_resolver = ctx.service.db.get_note_resolver();
         let notes = note_resolver
             .get_many(&NoteResolveOptions {
-                ids: None,
+                mode: Some(NoteResolveMode::PinsFromUserId(pin_user.id.clone())),
                 visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter(
                     ctx.self_user.as_deref().map(ck::user::Model::get_id),
                 )),
@@ -580,7 +579,6 @@ impl NoteModel {
                             .map(str::to_string)
                     })
                     .flatten(),
-                only_pins_from: Some(pin_user.id.clone()),
             })
             .await?;
 

src/model/processing/notification.rs

@@ -231,7 +231,7 @@ impl NotificationModel {
             .get_single(
                 &NotificationResolveOptions {
                     note_options: NoteResolveOptions {
-                        ids: None,
+                        mode: None,
                         visibility_filter: Arc::new(
                             NoteVisibilityFilterModel.new_note_visibility_filter(Some(user_id)),
                         ),
@@ -240,7 +240,6 @@ impl NotificationModel {
                         with_reply_target: true,
                         with_renote_target: true,
                         with_interactions_from: self_id.map(str::to_string),
-                        only_pins_from: None,
                         user_options: user_resolve_options.clone(),
                     },
                     user_options: user_resolve_options,
@@ -294,7 +293,7 @@ impl NotificationModel {
             .get(
                 &NotificationResolveOptions {
                     note_options: NoteResolveOptions {
-                        ids: None,
+                        mode: None,
                         visibility_filter: Arc::new(
                             NoteVisibilityFilterModel.new_note_visibility_filter(Some(id)),
                         ),
@@ -303,7 +302,6 @@ impl NotificationModel {
                         with_reply_target: true,
                         with_renote_target: true,
                         with_interactions_from: self_id.map(str::to_string),
-                        only_pins_from: None,
                         user_options: user_resolve_options.clone(),
                     },
                     user_options: user_resolve_options,

src/service/emoji_cache.rs

@@ -4,11 +4,11 @@ use magnetar_model::emoji::{EmojiResolver, EmojiTag};
 use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
 use std::collections::HashSet;
 use std::sync::Arc;
-use strum::EnumVariantNames;
+use strum::VariantNames;
 use thiserror::Error;
 use tokio::sync::Mutex;
 
-#[derive(Debug, Error, EnumVariantNames)]
+#[derive(Debug, Error, VariantNames)]
 pub enum EmojiCacheError {
     #[error("Database error: {0}")]
     DbError(#[from] CalckeyDbError),

src/service/generic_id_cache.rs

@@ -5,11 +5,11 @@ use magnetar_model::{CalckeyDbError, CalckeyModel};
 use std::marker::PhantomData;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use strum::EnumVariantNames;
+use strum::VariantNames;
 use thiserror::Error;
 use tokio::sync::Mutex;
 
-#[derive(Debug, Error, EnumVariantNames)]
+#[derive(Debug, Error, VariantNames)]
 pub enum GenericIdCacheError {
     #[error("Database error: {0}")]
     DbError(#[from] CalckeyDbError),

src/service/instance_cache.rs

@@ -4,11 +4,11 @@ use magnetar_common::config::MagnetarConfig;
 use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use strum::EnumVariantNames;
+use strum::VariantNames;
 use thiserror::Error;
 use tokio::sync::Mutex;
 
-#[derive(Debug, Error, EnumVariantNames)]
+#[derive(Debug, Error, VariantNames)]
 pub enum RemoteInstanceCacheError {
     #[error("Database error: {0}")]
     DbError(#[from] CalckeyDbError),

src/service/instance_meta_cache.rs

@@ -2,12 +2,12 @@ use crate::web::ApiError;
 use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use strum::EnumVariantNames;
+use strum::VariantNames;
 use thiserror::Error;
 use tokio::sync::{mpsc, oneshot};
 use tracing::error;
 
-#[derive(Debug, Error, EnumVariantNames)]
+#[derive(Debug, Error, VariantNames)]
 pub enum InstanceMetaCacheError {
     #[error("Database error: {0}")]
     DbError(#[from] CalckeyDbError),

src/service/local_user_cache.rs

@@ -2,20 +2,20 @@ use std::collections::HashMap;
 use std::sync::Arc;
 
 use cached::{Cached, TimedCache};
-use strum::EnumVariantNames;
+use strum::{EnumVariantNames, VariantNames};
 use thiserror::Error;
 use tokio::sync::Mutex;
 use tracing::error;
 
 use magnetar_common::config::MagnetarConfig;
 use magnetar_model::{
-    CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub, ck,
+    ck, CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub,
     InternalStreamMessage, SubMessage,
 };
 
 use crate::web::ApiError;
 
-#[derive(Debug, Error, EnumVariantNames)]
+#[derive(Debug, Error, VariantNames)]
 pub enum UserCacheError {
     #[error("Database error: {0}")]
     DbError(#[from] CalckeyDbError),