Compare commits

..

2 Commits

Author SHA1 Message Date
Natty 87e9fb36e1
Fixed some past crimes
ci/woodpecker/push/ociImagePush Pipeline failed Details
Mainly fetching pins in a subquery instead of joining

Parsing needs a bigger overhaul
2024-09-05 17:22:54 +02:00
Natty 1173360265
Updated and fixed dependencies 2024-09-05 15:58:42 +02:00
12 changed files with 1104 additions and 831 deletions

1676
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -7,17 +7,17 @@ license = "AGPL-3.0-only"
[workspace]
members = [
".",
"ext_activity_streams",
"ext_federation",
"ext_nodeinfo",
"ext_webfinger",
"ext_model",
"fe_calckey",
"magnetar_common",
"magnetar_sdk",
"magnetar_mmm_parser",
"core",
".",
"ext_activity_streams",
"ext_federation",
"ext_nodeinfo",
"ext_webfinger",
"ext_model",
"fe_calckey",
"magnetar_common",
"magnetar_sdk",
"magnetar_mmm_parser",
"core",
]
[workspace.package]
@ -30,31 +30,33 @@ async-stream = "0.3"
axum = "0.7"
axum-extra = "0.9"
base64 = "0.22"
cached = "0.47"
cached = "0.53"
cfg-if = "1"
chrono = "0.4"
compact_str = "0.7"
compact_str = "0.8"
dotenvy = "0.15"
ed25519-dalek = "2.1"
either = "1.9"
emojis = "0.6"
futures = "0.3"
futures-channel = "0.3"
futures-core = "0.3"
futures-util = "0.3"
headers = "0.4"
http = "1.0"
httpdate = "1"
hyper = "1.1"
idna = "0.5"
idna = "1"
indexmap = "2.2"
itertools = "0.12"
itertools = "0.13"
lru = "0.12"
miette = "5.9"
miette = "7"
nom = "7"
nom_locate = "4"
percent-encoding = "2.2"
quick-xml = "0.31"
redis = "0.24"
priority-queue = "2.0"
quick-xml = "0.36"
redis = "0.26"
regex = "1.9"
rsa = "0.9"
reqwest = "0.12"
@ -64,14 +66,15 @@ serde = "1"
serde_json = "1"
serde_urlencoded = "0.7"
sha2 = "0.10"
strum = "0.25"
smallvec = "1.13"
strum = "0.26"
tera = { version = "1", default-features = false }
thiserror = "1"
tokio = "1.24"
tokio-util = "0.7"
tokio-stream = "0.1"
toml = "0.8"
tower = "0.4"
tower = "0.5"
tower-http = "0.5"
tracing = "0.1"
tracing-subscriber = "0.3"

View File

@ -1,16 +1,12 @@
pub mod data;
use ext_model_migration::SelectStatement;
use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr};
use sea_orm::{
Condition, EntityTrait, Iden, JoinType, QueryFilter, QueryOrder, QuerySelect, QueryTrait,
Select,
};
use std::sync::Arc;
use ck::{note, note_reaction, user_note_pining};
use data::{sub_interaction_reaction, sub_interaction_renote, NoteData};
use ext_model_migration::SelectStatement;
use magnetar_sdk::types::SpanFilter;
use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr};
use sea_orm::{Condition, EntityTrait, Iden, JoinType, Order, QueryFilter, QuerySelect, QueryTrait, Select};
use std::sync::Arc;
use crate::model_ext::{
join_columns_default, AliasColumnExt, AliasSourceExt, AliasSuffixExt, CursorPaginationExt,
@ -19,10 +15,9 @@ use crate::model_ext::{
use crate::user_model::{UserResolveOptions, UserResolver};
use crate::{CalckeyDbError, CalckeyModel};
const PIN: &str = "pin.";
const INTERACTION_REACTION: &str = "interaction.reaction.";
const INTERACTION_RENOTE: &str = "interaction.renote.";
const USER: &str = "user.";
const INTERACTION_REACTION: &str = "iact.rct.";
const INTERACTION_RENOTE: &str = "iact.rnt.";
const USER: &str = "u.";
const REPLY: &str = "reply.";
const RENOTE: &str = "renote.";
@ -36,16 +31,50 @@ pub trait NoteVisibilityFilterFactory: Send + Sync {
fn with_note_and_user_tables(&self, note: &dyn Iden) -> SimpleExpr;
}
#[derive(Clone)]
pub enum NoteResolveMode {
Single(String),
Multiple(Vec<String>),
PinsFromUserId(String),
}
impl NoteResolveMode {
fn as_expr(&self) -> SimpleExpr {
let id_col = note::Entity.base_prefix().col(note::Column::Id);
match self {
NoteResolveMode::Single(id) => id_col.eq(id),
NoteResolveMode::Multiple(ids) => id_col.is_in(ids),
NoteResolveMode::PinsFromUserId(user_id) => {
let sub_query = Query::select()
.expr(SelectExpr {
expr: Expr::col(user_note_pining::Column::NoteId).into(),
alias: None,
window: None,
})
.from(user_note_pining::Entity)
.and_where(Expr::col(note_reaction::Column::UserId).eq(user_id))
.order_by_columns([
(user_note_pining::Column::CreatedAt, Order::Desc),
(user_note_pining::Column::Id, Order::Desc)
])
.take();
id_col.in_subquery(sub_query)
}
}
}
}
#[derive(Clone)]
pub struct NoteResolveOptions {
pub ids: Option<Vec<String>>,
pub mode: Option<NoteResolveMode>,
pub visibility_filter: Arc<dyn NoteVisibilityFilterFactory>,
pub time_range: Option<SpanFilter>,
pub limit: Option<u64>,
pub with_reply_target: bool,
pub with_renote_target: bool,
pub with_interactions_from: Option<String>, // User ID
pub only_pins_from: Option<String>, // User ID
pub user_options: UserResolveOptions,
}
@ -57,7 +86,7 @@ trait SelectNoteInteractionsExt {
) -> &mut Self;
fn add_sub_select_interaction_renote(&mut self, note_tbl: &MagIden, user_id: &str)
-> &mut Self;
-> &mut Self;
}
impl SelectNoteInteractionsExt for SelectStatement {
@ -117,16 +146,6 @@ impl SelectNoteInteractionsExt for SelectStatement {
}
}
fn ids_into_expr(ids: &Vec<String>) -> SimpleExpr {
let id_col = note::Entity.base_prefix().col(note::Column::Id);
if ids.len() == 1 {
id_col.eq(&ids[0])
} else {
id_col.is_in(ids)
}
}
impl NoteResolver {
pub fn new(db: CalckeyModel, user_resolver: UserResolver) -> Self {
NoteResolver { db, user_resolver }
@ -141,7 +160,7 @@ impl NoteResolver {
.visibility_filter
.with_note_and_user_tables(&note::Entity.base_prefix());
let id_filter = options.ids.as_ref().map(ids_into_expr);
let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr);
let note = select
.filter(visibility_filter)
@ -161,17 +180,11 @@ impl NoteResolver {
let visibility_filter = options
.visibility_filter
.with_note_and_user_tables(&note::Entity.base_prefix());
let id_filter = options.ids.as_ref().map(ids_into_expr);
let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr);
let notes_select = select
.filter(visibility_filter)
.apply_if(id_filter, Select::<note::Entity>::filter)
.apply_if(options.only_pins_from.as_deref(), |s, _| {
s.order_by_desc(Expr::col((
note::Entity.base_prefix().into_iden().join_str(PIN),
user_note_pining::Column::CreatedAt,
)))
});
.apply_if(id_filter, Select::<note::Entity>::filter);
let notes = if let Some(pagination) = &options.time_range {
notes_select
@ -304,16 +317,6 @@ impl NoteResolver {
let mut select = Query::select();
select.from_as(note::Entity, note_tbl.clone().into_iden());
if let Some(pins_user) = &options.only_pins_from {
select
.join_columns(
JoinType::InnerJoin,
note::Relation::UserNotePining.with_from_alias(&note_tbl),
&note_tbl.join_str(PIN),
)
.and_where(note_tbl.col(user_note_pining::Column::UserId).eq(pins_user));
}
self.attach_note(
&mut select,
&note_tbl,

View File

@ -18,4 +18,4 @@ serde = { workspace = true, features = ["derive"] }
strum = { workspace = true, features = ["derive"] }
tracing = { workspace = true }
unicode-segmentation = { workspace = true }
quick-xml = { workspace = "true", optional = true, features = ["serialize"] }
quick-xml = { workspace = true, optional = true, features = ["serialize"] }

View File

@ -5,7 +5,6 @@ use std::marker::PhantomData;
use compact_str::{CompactString, ToCompactString};
use either::Either;
use nom::{IResult, Offset, Parser, Slice};
use nom::branch::alt;
use nom::bytes::complete::{tag, tag_no_case};
use nom::character::complete::{
@ -16,6 +15,7 @@ use nom::combinator::{eof, fail, map, not, opt, peek, recognize};
use nom::error::ErrorKind;
use nom::multi::{many0_count, many1, many1_count, many_till, separated_list1};
use nom::sequence::tuple;
use nom::{IResult, Offset, Parser, Slice};
use nom_locate::LocatedSpan;
use quick_xml::events::{BytesText, Event};
use serde::{Deserialize, Serialize};
@ -56,7 +56,6 @@ pub enum Token {
Sequence(Vec<Token>),
Quote(Box<Token>),
Small(Box<Token>),
BoldItalic(Box<Token>),
Bold(Box<Token>),
Italic(Box<Token>),
Center(Box<Token>),
@ -101,7 +100,6 @@ impl Token {
Token::Sequence(tokens) => tokens.first().and_then(Token::str_content_left),
Token::Quote(inner) => inner.str_content_left(),
Token::Small(inner) => inner.str_content_left(),
Token::BoldItalic(inner) => inner.str_content_left(),
Token::Bold(inner) => inner.str_content_left(),
Token::Italic(inner) => inner.str_content_left(),
Token::Center(inner) => inner.str_content_left(),
@ -124,7 +122,6 @@ impl Token {
Token::Sequence(tokens) => tokens.last().and_then(Token::str_content_right),
Token::Quote(inner) => inner.str_content_right(),
Token::Small(inner) => inner.str_content_right(),
Token::BoldItalic(inner) => inner.str_content_right(),
Token::Bold(inner) => inner.str_content_right(),
Token::Italic(inner) => inner.str_content_right(),
Token::Center(inner) => inner.str_content_right(),
@ -147,7 +144,6 @@ impl Token {
sequence @ Token::Sequence(_) => sequence.clone(),
Token::Quote(inner) => inner.inner(),
Token::Small(inner) => inner.inner(),
Token::BoldItalic(inner) => inner.inner(),
Token::Bold(inner) => inner.inner(),
Token::Italic(inner) => inner.inner(),
Token::Center(inner) => inner.inner(),
@ -213,7 +209,6 @@ impl Token {
}
Token::Quote(inner) => Token::Quote(Box::new(inner.merged())),
Token::Small(inner) => Token::Small(Box::new(inner.merged())),
Token::BoldItalic(inner) => Token::BoldItalic(Box::new(inner.merged())),
Token::Bold(inner) => Token::Bold(Box::new(inner.merged())),
Token::Italic(inner) => Token::Italic(Box::new(inner.merged())),
Token::Center(inner) => Token::Center(Box::new(inner.merged())),
@ -247,7 +242,6 @@ impl Token {
}
Token::Quote(inner)
| Token::Small(inner)
| Token::BoldItalic(inner)
| Token::Bold(inner)
| Token::Italic(inner)
| Token::Center(inner)
@ -266,7 +260,6 @@ impl Token {
.for_each(|tok| tok.walk_speech_transform(func));
}
Token::Small(inner)
| Token::BoldItalic(inner)
| Token::Bold(inner)
| Token::Italic(inner)
| Token::Center(inner)
@ -295,16 +288,6 @@ impl Token {
.create_element("small")
.write_inner_content(|w| inner.write(w))?;
}
Token::BoldItalic(inner) => {
writer
.create_element("b")
.write_inner_content::<_, quick_xml::Error>(|w| {
w.create_element("i")
.write_inner_content(|w| inner.write(w))?;
Ok(())
})?;
}
Token::Bold(inner) => {
writer
.create_element("b")
@ -468,8 +451,8 @@ trait SliceOffset {
fn up_to(&self, other: &Self) -> Self;
fn fragment_between<'a>(&self, other: &Self) -> &'a str
where
Self: 'a;
where
Self: 'a;
}
impl SliceOffset for Span<'_> {
@ -478,8 +461,8 @@ impl SliceOffset for Span<'_> {
}
fn fragment_between<'a>(&self, other: &Self) -> &'a str
where
Self: 'a,
where
Self: 'a,
{
self.up_to(other).into_fragment()
}
@ -792,8 +775,6 @@ impl Context {
fn base_bold_italic<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
alt((
self.partial(Self::tag_bold_italic_asterisk),
self.partial(Self::tag_bold_italic_underscore),
self.partial(Self::tag_bold_asterisk),
self.partial(Self::tag_italic_asterisk),
self.partial(Self::tag_bold_underscore),
@ -1033,10 +1014,10 @@ impl Context {
escape: bool,
matcher: Matcher<'a, 'b, T>,
fallback: Matcher<'a, 'b, S>,
) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token> + '_
where
FOpen: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
FClose: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token> + 'a
where
FOpen: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
FClose: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
{
let FlankingDelim(opening_tag, opening_rule, ..) = opening_tag.into();
let FlankingDelim(closing_tag, closing_rule, ..) = closing_tag.into();
@ -1223,38 +1204,6 @@ impl Context {
)(input)
}
fn tag_bold_italic_asterisk<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
(tag("***"), FlankingRule::Lenient),
(tag("***"), FlankingRule::Lenient),
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
)(input)
}
fn tag_bold_italic_underscore<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
(tag("___"), FlankingRule::Strict),
(tag("___"), FlankingRule::Strict),
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
)(input)
}
fn tag_bold<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
tag_no_case("<b>"),
@ -1498,7 +1447,7 @@ impl Context {
return fail(input);
};
let grapheme = grapheme.trim_end_matches(|c| c == '\u{200c}' || c == '\u{200d}');
let grapheme = grapheme.trim_end_matches(['\u{200c}', '\u{200d}']);
let emoji = emojis::get(grapheme);
@ -1567,7 +1516,7 @@ impl Context {
})(input)?;
let (input, name) = map(
recognize(many1(alt((alphanumeric1, recognize(one_of("-_")))))),
recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))),
Span::into_fragment,
)(input)?;
@ -1591,7 +1540,7 @@ impl Context {
mention_type
};
let host =
host_opt.map(|(_, name)| name.trim_end_matches(|c| matches!(c, '.' | '-' | '_')));
host_opt.map(|(_, name)| name.trim_end_matches(['.', '-', '_']));
let input = host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before);
Ok((
@ -1642,8 +1591,8 @@ impl Context {
&'b self,
mut func: F,
) -> impl FnMut(Span<'a>) -> IResult<Span<'a>, O> + 'b
where
F: Parser<Span<'a>, O, nom::error::Error<Span<'a>>> + 'b,
where
F: Parser<Span<'a>, O, nom::error::Error<Span<'a>>> + 'b,
{
move |mut input| {
if input.extra.depth >= self.depth_limit {
@ -1723,8 +1672,8 @@ impl Context {
mut terminator: F,
spaces: bool,
) -> impl FnMut(Span<'a>) -> IResult<Span<'a>, Span<'a>> + 'b
where
F: Parser<Span<'a>, Span<'a>, nom::error::Error<Span<'a>>> + 'b,
where
F: Parser<Span<'a>, Span<'a>, nom::error::Error<Span<'a>>> + 'b,
{
move |input| {
recognize(many1_count(tuple((
@ -1746,7 +1695,7 @@ mod test {
use nom::bytes::complete::tag;
use crate::{Context, DEFAULT_DEPTH_LIMIT, Span, SpanMeta, to_xml_string, Token};
use crate::{to_xml_string, Context, Span, SpanMeta, Token, DEFAULT_DEPTH_LIMIT};
fn parse_full(string: &str) -> Token {
Context::default()
@ -1904,7 +1853,7 @@ mod test {
assert_eq!(
parse_full(r#"***bold italic***"#),
Token::BoldItalic(Box::new(Token::PlainText("bold italic".into())))
Token::Bold(Box::new(Token::Italic(Box::new(Token::PlainText("bold italic".into())))))
);
assert_eq!(

View File

@ -13,7 +13,7 @@ use magnetar_model::model_ext::AliasColumnExt;
use magnetar_model::note_model::data::{
sub_interaction_reaction, sub_interaction_renote, NoteData,
};
use magnetar_model::note_model::{NoteResolveOptions, NoteVisibilityFilterFactory};
use magnetar_model::note_model::{NoteResolveMode, NoteResolveOptions, NoteVisibilityFilterFactory};
use magnetar_model::poll::PollResolver;
use magnetar_model::sea_orm::sea_query::{PgFunc, Query, SimpleExpr};
use magnetar_model::sea_orm::{ActiveEnum, ColumnTrait, Iden, IntoSimpleExpr};
@ -518,7 +518,7 @@ impl NoteModel {
let note_resolver = ctx.service.db.get_note_resolver();
let Some(note) = note_resolver
.get_one(&NoteResolveOptions {
ids: Some(vec![id.to_owned()]),
mode: Some(NoteResolveMode::Single(id.to_owned())),
visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter(
ctx.self_user.as_deref().map(ck::user::Model::get_id),
)),
@ -540,7 +540,6 @@ impl NoteModel {
.map(str::to_string)
})
.flatten(),
only_pins_from: None,
})
.await?
else {
@ -558,7 +557,7 @@ impl NoteModel {
let note_resolver = ctx.service.db.get_note_resolver();
let notes = note_resolver
.get_many(&NoteResolveOptions {
ids: None,
mode: Some(NoteResolveMode::PinsFromUserId(pin_user.id.clone())),
visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter(
ctx.self_user.as_deref().map(ck::user::Model::get_id),
)),
@ -580,7 +579,6 @@ impl NoteModel {
.map(str::to_string)
})
.flatten(),
only_pins_from: Some(pin_user.id.clone()),
})
.await?;

View File

@ -231,7 +231,7 @@ impl NotificationModel {
.get_single(
&NotificationResolveOptions {
note_options: NoteResolveOptions {
ids: None,
mode: None,
visibility_filter: Arc::new(
NoteVisibilityFilterModel.new_note_visibility_filter(Some(user_id)),
),
@ -240,7 +240,6 @@ impl NotificationModel {
with_reply_target: true,
with_renote_target: true,
with_interactions_from: self_id.map(str::to_string),
only_pins_from: None,
user_options: user_resolve_options.clone(),
},
user_options: user_resolve_options,
@ -294,7 +293,7 @@ impl NotificationModel {
.get(
&NotificationResolveOptions {
note_options: NoteResolveOptions {
ids: None,
mode: None,
visibility_filter: Arc::new(
NoteVisibilityFilterModel.new_note_visibility_filter(Some(id)),
),
@ -303,7 +302,6 @@ impl NotificationModel {
with_reply_target: true,
with_renote_target: true,
with_interactions_from: self_id.map(str::to_string),
only_pins_from: None,
user_options: user_resolve_options.clone(),
},
user_options: user_resolve_options,

View File

@ -4,11 +4,11 @@ use magnetar_model::emoji::{EmojiResolver, EmojiTag};
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::collections::HashSet;
use std::sync::Arc;
use strum::EnumVariantNames;
use strum::VariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, EnumVariantNames)]
#[derive(Debug, Error, VariantNames)]
pub enum EmojiCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

View File

@ -5,11 +5,11 @@ use magnetar_model::{CalckeyDbError, CalckeyModel};
use std::marker::PhantomData;
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::EnumVariantNames;
use strum::VariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, EnumVariantNames)]
#[derive(Debug, Error, VariantNames)]
pub enum GenericIdCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

View File

@ -4,11 +4,11 @@ use magnetar_common::config::MagnetarConfig;
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::EnumVariantNames;
use strum::VariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, EnumVariantNames)]
#[derive(Debug, Error, VariantNames)]
pub enum RemoteInstanceCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

View File

@ -2,12 +2,12 @@ use crate::web::ApiError;
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::EnumVariantNames;
use strum::VariantNames;
use thiserror::Error;
use tokio::sync::{mpsc, oneshot};
use tracing::error;
#[derive(Debug, Error, EnumVariantNames)]
#[derive(Debug, Error, VariantNames)]
pub enum InstanceMetaCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

View File

@ -2,20 +2,20 @@ use std::collections::HashMap;
use std::sync::Arc;
use cached::{Cached, TimedCache};
use strum::EnumVariantNames;
use strum::{EnumVariantNames, VariantNames};
use thiserror::Error;
use tokio::sync::Mutex;
use tracing::error;
use magnetar_common::config::MagnetarConfig;
use magnetar_model::{
CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub, ck,
ck, CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub,
InternalStreamMessage, SubMessage,
};
use crate::web::ApiError;
#[derive(Debug, Error, EnumVariantNames)]
#[derive(Debug, Error, VariantNames)]
pub enum UserCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),