Compare commits

..

No commits in common. "87e9fb36e168d31eec7b0cf1bc50a28c74cdce0a" and "78c93f3c2004713465b19fcfd361daebced12980" have entirely different histories.

12 changed files with 820 additions and 1093 deletions

1654
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -7,17 +7,17 @@ license = "AGPL-3.0-only"
[workspace]
members = [
".",
"ext_activity_streams",
"ext_federation",
"ext_nodeinfo",
"ext_webfinger",
"ext_model",
"fe_calckey",
"magnetar_common",
"magnetar_sdk",
"magnetar_mmm_parser",
"core",
".",
"ext_activity_streams",
"ext_federation",
"ext_nodeinfo",
"ext_webfinger",
"ext_model",
"fe_calckey",
"magnetar_common",
"magnetar_sdk",
"magnetar_mmm_parser",
"core",
]
[workspace.package]
@ -30,33 +30,31 @@ async-stream = "0.3"
axum = "0.7"
axum-extra = "0.9"
base64 = "0.22"
cached = "0.53"
cached = "0.47"
cfg-if = "1"
chrono = "0.4"
compact_str = "0.8"
compact_str = "0.7"
dotenvy = "0.15"
ed25519-dalek = "2.1"
either = "1.9"
emojis = "0.6"
futures = "0.3"
futures-channel = "0.3"
futures-core = "0.3"
futures-util = "0.3"
headers = "0.4"
http = "1.0"
httpdate = "1"
hyper = "1.1"
idna = "1"
idna = "0.5"
indexmap = "2.2"
itertools = "0.13"
itertools = "0.12"
lru = "0.12"
miette = "7"
miette = "5.9"
nom = "7"
nom_locate = "4"
percent-encoding = "2.2"
priority-queue = "2.0"
quick-xml = "0.36"
redis = "0.26"
quick-xml = "0.31"
redis = "0.24"
regex = "1.9"
rsa = "0.9"
reqwest = "0.12"
@ -66,15 +64,14 @@ serde = "1"
serde_json = "1"
serde_urlencoded = "0.7"
sha2 = "0.10"
smallvec = "1.13"
strum = "0.26"
strum = "0.25"
tera = { version = "1", default-features = false }
thiserror = "1"
tokio = "1.24"
tokio-util = "0.7"
tokio-stream = "0.1"
toml = "0.8"
tower = "0.5"
tower = "0.4"
tower-http = "0.5"
tracing = "0.1"
tracing-subscriber = "0.3"

View File

@ -1,12 +1,16 @@
pub mod data;
use ext_model_migration::SelectStatement;
use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr};
use sea_orm::{
Condition, EntityTrait, Iden, JoinType, QueryFilter, QueryOrder, QuerySelect, QueryTrait,
Select,
};
use std::sync::Arc;
use ck::{note, note_reaction, user_note_pining};
use data::{sub_interaction_reaction, sub_interaction_renote, NoteData};
use ext_model_migration::SelectStatement;
use magnetar_sdk::types::SpanFilter;
use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr};
use sea_orm::{Condition, EntityTrait, Iden, JoinType, Order, QueryFilter, QuerySelect, QueryTrait, Select};
use std::sync::Arc;
use crate::model_ext::{
join_columns_default, AliasColumnExt, AliasSourceExt, AliasSuffixExt, CursorPaginationExt,
@ -15,9 +19,10 @@ use crate::model_ext::{
use crate::user_model::{UserResolveOptions, UserResolver};
use crate::{CalckeyDbError, CalckeyModel};
const INTERACTION_REACTION: &str = "iact.rct.";
const INTERACTION_RENOTE: &str = "iact.rnt.";
const USER: &str = "u.";
const PIN: &str = "pin.";
const INTERACTION_REACTION: &str = "interaction.reaction.";
const INTERACTION_RENOTE: &str = "interaction.renote.";
const USER: &str = "user.";
const REPLY: &str = "reply.";
const RENOTE: &str = "renote.";
@ -31,50 +36,16 @@ pub trait NoteVisibilityFilterFactory: Send + Sync {
fn with_note_and_user_tables(&self, note: &dyn Iden) -> SimpleExpr;
}
#[derive(Clone)]
pub enum NoteResolveMode {
Single(String),
Multiple(Vec<String>),
PinsFromUserId(String),
}
impl NoteResolveMode {
fn as_expr(&self) -> SimpleExpr {
let id_col = note::Entity.base_prefix().col(note::Column::Id);
match self {
NoteResolveMode::Single(id) => id_col.eq(id),
NoteResolveMode::Multiple(ids) => id_col.is_in(ids),
NoteResolveMode::PinsFromUserId(user_id) => {
let sub_query = Query::select()
.expr(SelectExpr {
expr: Expr::col(user_note_pining::Column::NoteId).into(),
alias: None,
window: None,
})
.from(user_note_pining::Entity)
.and_where(Expr::col(note_reaction::Column::UserId).eq(user_id))
.order_by_columns([
(user_note_pining::Column::CreatedAt, Order::Desc),
(user_note_pining::Column::Id, Order::Desc)
])
.take();
id_col.in_subquery(sub_query)
}
}
}
}
#[derive(Clone)]
pub struct NoteResolveOptions {
pub mode: Option<NoteResolveMode>,
pub ids: Option<Vec<String>>,
pub visibility_filter: Arc<dyn NoteVisibilityFilterFactory>,
pub time_range: Option<SpanFilter>,
pub limit: Option<u64>,
pub with_reply_target: bool,
pub with_renote_target: bool,
pub with_interactions_from: Option<String>, // User ID
pub only_pins_from: Option<String>, // User ID
pub user_options: UserResolveOptions,
}
@ -86,7 +57,7 @@ trait SelectNoteInteractionsExt {
) -> &mut Self;
fn add_sub_select_interaction_renote(&mut self, note_tbl: &MagIden, user_id: &str)
-> &mut Self;
-> &mut Self;
}
impl SelectNoteInteractionsExt for SelectStatement {
@ -146,6 +117,16 @@ impl SelectNoteInteractionsExt for SelectStatement {
}
}
fn ids_into_expr(ids: &Vec<String>) -> SimpleExpr {
let id_col = note::Entity.base_prefix().col(note::Column::Id);
if ids.len() == 1 {
id_col.eq(&ids[0])
} else {
id_col.is_in(ids)
}
}
impl NoteResolver {
pub fn new(db: CalckeyModel, user_resolver: UserResolver) -> Self {
NoteResolver { db, user_resolver }
@ -160,7 +141,7 @@ impl NoteResolver {
.visibility_filter
.with_note_and_user_tables(&note::Entity.base_prefix());
let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr);
let id_filter = options.ids.as_ref().map(ids_into_expr);
let note = select
.filter(visibility_filter)
@ -180,11 +161,17 @@ impl NoteResolver {
let visibility_filter = options
.visibility_filter
.with_note_and_user_tables(&note::Entity.base_prefix());
let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr);
let id_filter = options.ids.as_ref().map(ids_into_expr);
let notes_select = select
.filter(visibility_filter)
.apply_if(id_filter, Select::<note::Entity>::filter);
.apply_if(id_filter, Select::<note::Entity>::filter)
.apply_if(options.only_pins_from.as_deref(), |s, _| {
s.order_by_desc(Expr::col((
note::Entity.base_prefix().into_iden().join_str(PIN),
user_note_pining::Column::CreatedAt,
)))
});
let notes = if let Some(pagination) = &options.time_range {
notes_select
@ -317,6 +304,16 @@ impl NoteResolver {
let mut select = Query::select();
select.from_as(note::Entity, note_tbl.clone().into_iden());
if let Some(pins_user) = &options.only_pins_from {
select
.join_columns(
JoinType::InnerJoin,
note::Relation::UserNotePining.with_from_alias(&note_tbl),
&note_tbl.join_str(PIN),
)
.and_where(note_tbl.col(user_note_pining::Column::UserId).eq(pins_user));
}
self.attach_note(
&mut select,
&note_tbl,

View File

@ -18,4 +18,4 @@ serde = { workspace = true, features = ["derive"] }
strum = { workspace = true, features = ["derive"] }
tracing = { workspace = true }
unicode-segmentation = { workspace = true }
quick-xml = { workspace = true, optional = true, features = ["serialize"] }
quick-xml = { workspace = "true", optional = true, features = ["serialize"] }

View File

@ -5,6 +5,7 @@ use std::marker::PhantomData;
use compact_str::{CompactString, ToCompactString};
use either::Either;
use nom::{IResult, Offset, Parser, Slice};
use nom::branch::alt;
use nom::bytes::complete::{tag, tag_no_case};
use nom::character::complete::{
@ -15,7 +16,6 @@ use nom::combinator::{eof, fail, map, not, opt, peek, recognize};
use nom::error::ErrorKind;
use nom::multi::{many0_count, many1, many1_count, many_till, separated_list1};
use nom::sequence::tuple;
use nom::{IResult, Offset, Parser, Slice};
use nom_locate::LocatedSpan;
use quick_xml::events::{BytesText, Event};
use serde::{Deserialize, Serialize};
@ -56,6 +56,7 @@ pub enum Token {
Sequence(Vec<Token>),
Quote(Box<Token>),
Small(Box<Token>),
BoldItalic(Box<Token>),
Bold(Box<Token>),
Italic(Box<Token>),
Center(Box<Token>),
@ -100,6 +101,7 @@ impl Token {
Token::Sequence(tokens) => tokens.first().and_then(Token::str_content_left),
Token::Quote(inner) => inner.str_content_left(),
Token::Small(inner) => inner.str_content_left(),
Token::BoldItalic(inner) => inner.str_content_left(),
Token::Bold(inner) => inner.str_content_left(),
Token::Italic(inner) => inner.str_content_left(),
Token::Center(inner) => inner.str_content_left(),
@ -122,6 +124,7 @@ impl Token {
Token::Sequence(tokens) => tokens.last().and_then(Token::str_content_right),
Token::Quote(inner) => inner.str_content_right(),
Token::Small(inner) => inner.str_content_right(),
Token::BoldItalic(inner) => inner.str_content_right(),
Token::Bold(inner) => inner.str_content_right(),
Token::Italic(inner) => inner.str_content_right(),
Token::Center(inner) => inner.str_content_right(),
@ -144,6 +147,7 @@ impl Token {
sequence @ Token::Sequence(_) => sequence.clone(),
Token::Quote(inner) => inner.inner(),
Token::Small(inner) => inner.inner(),
Token::BoldItalic(inner) => inner.inner(),
Token::Bold(inner) => inner.inner(),
Token::Italic(inner) => inner.inner(),
Token::Center(inner) => inner.inner(),
@ -209,6 +213,7 @@ impl Token {
}
Token::Quote(inner) => Token::Quote(Box::new(inner.merged())),
Token::Small(inner) => Token::Small(Box::new(inner.merged())),
Token::BoldItalic(inner) => Token::BoldItalic(Box::new(inner.merged())),
Token::Bold(inner) => Token::Bold(Box::new(inner.merged())),
Token::Italic(inner) => Token::Italic(Box::new(inner.merged())),
Token::Center(inner) => Token::Center(Box::new(inner.merged())),
@ -242,6 +247,7 @@ impl Token {
}
Token::Quote(inner)
| Token::Small(inner)
| Token::BoldItalic(inner)
| Token::Bold(inner)
| Token::Italic(inner)
| Token::Center(inner)
@ -260,6 +266,7 @@ impl Token {
.for_each(|tok| tok.walk_speech_transform(func));
}
Token::Small(inner)
| Token::BoldItalic(inner)
| Token::Bold(inner)
| Token::Italic(inner)
| Token::Center(inner)
@ -288,6 +295,16 @@ impl Token {
.create_element("small")
.write_inner_content(|w| inner.write(w))?;
}
Token::BoldItalic(inner) => {
writer
.create_element("b")
.write_inner_content::<_, quick_xml::Error>(|w| {
w.create_element("i")
.write_inner_content(|w| inner.write(w))?;
Ok(())
})?;
}
Token::Bold(inner) => {
writer
.create_element("b")
@ -451,8 +468,8 @@ trait SliceOffset {
fn up_to(&self, other: &Self) -> Self;
fn fragment_between<'a>(&self, other: &Self) -> &'a str
where
Self: 'a;
where
Self: 'a;
}
impl SliceOffset for Span<'_> {
@ -461,8 +478,8 @@ impl SliceOffset for Span<'_> {
}
fn fragment_between<'a>(&self, other: &Self) -> &'a str
where
Self: 'a,
where
Self: 'a,
{
self.up_to(other).into_fragment()
}
@ -775,6 +792,8 @@ impl Context {
fn base_bold_italic<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
alt((
self.partial(Self::tag_bold_italic_asterisk),
self.partial(Self::tag_bold_italic_underscore),
self.partial(Self::tag_bold_asterisk),
self.partial(Self::tag_italic_asterisk),
self.partial(Self::tag_bold_underscore),
@ -1014,10 +1033,10 @@ impl Context {
escape: bool,
matcher: Matcher<'a, 'b, T>,
fallback: Matcher<'a, 'b, S>,
) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token> + 'a
where
FOpen: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
FClose: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token> + '_
where
FOpen: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
FClose: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
{
let FlankingDelim(opening_tag, opening_rule, ..) = opening_tag.into();
let FlankingDelim(closing_tag, closing_rule, ..) = closing_tag.into();
@ -1204,6 +1223,38 @@ impl Context {
)(input)
}
fn tag_bold_italic_asterisk<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
(tag("***"), FlankingRule::Lenient),
(tag("***"), FlankingRule::Lenient),
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
)(input)
}
fn tag_bold_italic_underscore<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
(tag("___"), FlankingRule::Strict),
(tag("___"), FlankingRule::Strict),
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
)(input)
}
fn tag_bold<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
tag_no_case("<b>"),
@ -1447,7 +1498,7 @@ impl Context {
return fail(input);
};
let grapheme = grapheme.trim_end_matches(['\u{200c}', '\u{200d}']);
let grapheme = grapheme.trim_end_matches(|c| c == '\u{200c}' || c == '\u{200d}');
let emoji = emojis::get(grapheme);
@ -1516,7 +1567,7 @@ impl Context {
})(input)?;
let (input, name) = map(
recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))),
recognize(many1(alt((alphanumeric1, recognize(one_of("-_")))))),
Span::into_fragment,
)(input)?;
@ -1540,7 +1591,7 @@ impl Context {
mention_type
};
let host =
host_opt.map(|(_, name)| name.trim_end_matches(['.', '-', '_']));
host_opt.map(|(_, name)| name.trim_end_matches(|c| matches!(c, '.' | '-' | '_')));
let input = host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before);
Ok((
@ -1591,8 +1642,8 @@ impl Context {
&'b self,
mut func: F,
) -> impl FnMut(Span<'a>) -> IResult<Span<'a>, O> + 'b
where
F: Parser<Span<'a>, O, nom::error::Error<Span<'a>>> + 'b,
where
F: Parser<Span<'a>, O, nom::error::Error<Span<'a>>> + 'b,
{
move |mut input| {
if input.extra.depth >= self.depth_limit {
@ -1672,8 +1723,8 @@ impl Context {
mut terminator: F,
spaces: bool,
) -> impl FnMut(Span<'a>) -> IResult<Span<'a>, Span<'a>> + 'b
where
F: Parser<Span<'a>, Span<'a>, nom::error::Error<Span<'a>>> + 'b,
where
F: Parser<Span<'a>, Span<'a>, nom::error::Error<Span<'a>>> + 'b,
{
move |input| {
recognize(many1_count(tuple((
@ -1695,7 +1746,7 @@ mod test {
use nom::bytes::complete::tag;
use crate::{to_xml_string, Context, Span, SpanMeta, Token, DEFAULT_DEPTH_LIMIT};
use crate::{Context, DEFAULT_DEPTH_LIMIT, Span, SpanMeta, to_xml_string, Token};
fn parse_full(string: &str) -> Token {
Context::default()
@ -1853,7 +1904,7 @@ mod test {
assert_eq!(
parse_full(r#"***bold italic***"#),
Token::Bold(Box::new(Token::Italic(Box::new(Token::PlainText("bold italic".into())))))
Token::BoldItalic(Box::new(Token::PlainText("bold italic".into())))
);
assert_eq!(

View File

@ -13,7 +13,7 @@ use magnetar_model::model_ext::AliasColumnExt;
use magnetar_model::note_model::data::{
sub_interaction_reaction, sub_interaction_renote, NoteData,
};
use magnetar_model::note_model::{NoteResolveMode, NoteResolveOptions, NoteVisibilityFilterFactory};
use magnetar_model::note_model::{NoteResolveOptions, NoteVisibilityFilterFactory};
use magnetar_model::poll::PollResolver;
use magnetar_model::sea_orm::sea_query::{PgFunc, Query, SimpleExpr};
use magnetar_model::sea_orm::{ActiveEnum, ColumnTrait, Iden, IntoSimpleExpr};
@ -518,7 +518,7 @@ impl NoteModel {
let note_resolver = ctx.service.db.get_note_resolver();
let Some(note) = note_resolver
.get_one(&NoteResolveOptions {
mode: Some(NoteResolveMode::Single(id.to_owned())),
ids: Some(vec![id.to_owned()]),
visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter(
ctx.self_user.as_deref().map(ck::user::Model::get_id),
)),
@ -540,6 +540,7 @@ impl NoteModel {
.map(str::to_string)
})
.flatten(),
only_pins_from: None,
})
.await?
else {
@ -557,7 +558,7 @@ impl NoteModel {
let note_resolver = ctx.service.db.get_note_resolver();
let notes = note_resolver
.get_many(&NoteResolveOptions {
mode: Some(NoteResolveMode::PinsFromUserId(pin_user.id.clone())),
ids: None,
visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter(
ctx.self_user.as_deref().map(ck::user::Model::get_id),
)),
@ -579,6 +580,7 @@ impl NoteModel {
.map(str::to_string)
})
.flatten(),
only_pins_from: Some(pin_user.id.clone()),
})
.await?;

View File

@ -231,7 +231,7 @@ impl NotificationModel {
.get_single(
&NotificationResolveOptions {
note_options: NoteResolveOptions {
mode: None,
ids: None,
visibility_filter: Arc::new(
NoteVisibilityFilterModel.new_note_visibility_filter(Some(user_id)),
),
@ -240,6 +240,7 @@ impl NotificationModel {
with_reply_target: true,
with_renote_target: true,
with_interactions_from: self_id.map(str::to_string),
only_pins_from: None,
user_options: user_resolve_options.clone(),
},
user_options: user_resolve_options,
@ -293,7 +294,7 @@ impl NotificationModel {
.get(
&NotificationResolveOptions {
note_options: NoteResolveOptions {
mode: None,
ids: None,
visibility_filter: Arc::new(
NoteVisibilityFilterModel.new_note_visibility_filter(Some(id)),
),
@ -302,6 +303,7 @@ impl NotificationModel {
with_reply_target: true,
with_renote_target: true,
with_interactions_from: self_id.map(str::to_string),
only_pins_from: None,
user_options: user_resolve_options.clone(),
},
user_options: user_resolve_options,

View File

@ -4,11 +4,11 @@ use magnetar_model::emoji::{EmojiResolver, EmojiTag};
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::collections::HashSet;
use std::sync::Arc;
use strum::VariantNames;
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum EmojiCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

View File

@ -5,11 +5,11 @@ use magnetar_model::{CalckeyDbError, CalckeyModel};
use std::marker::PhantomData;
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::VariantNames;
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum GenericIdCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

View File

@ -4,11 +4,11 @@ use magnetar_common::config::MagnetarConfig;
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::VariantNames;
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum RemoteInstanceCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

View File

@ -2,12 +2,12 @@ use crate::web::ApiError;
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::VariantNames;
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::{mpsc, oneshot};
use tracing::error;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum InstanceMetaCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

View File

@ -2,20 +2,20 @@ use std::collections::HashMap;
use std::sync::Arc;
use cached::{Cached, TimedCache};
use strum::{EnumVariantNames, VariantNames};
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
use tracing::error;
use magnetar_common::config::MagnetarConfig;
use magnetar_model::{
ck, CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub,
CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub, ck,
InternalStreamMessage, SubMessage,
};
use crate::web::ApiError;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum UserCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),