natty
/
magnetar
1
0
Fork 0
Code Issues 4 Pull Requests 1 Packages Projects Releases Wiki Activity

Compare commits

base: natty:87e9fb36e168d31eec7b0cf1bc50a28c74cdce0a
Branches Tags
natty:main
natty:development
natty:mmm
natty:activity-pub
..
compare: natty:78c93f3c2004713465b19fcfd361daebced12980
Branches Tags
natty:development
natty:main
natty:mmm
natty:activity-pub

No commits in common. "87e9fb36e168d31eec7b0cf1bc50a28c74cdce0a" and "78c93f3c2004713465b19fcfd361daebced12980" have entirely different histories.
87e9fb36e1 ... 78c93f3c20

12 changed files with 820 additions and 1093 deletions
Show Stats Expand all files Collapse all files

1654
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

21
Cargo.toml
View File

@ -30,33 +30,31 @@ async-stream = "0.3"
axum = "0.7"
axum-extra = "0.9"
base64 = "0.22"
cached = "0.53"
cached = "0.47"
cfg-if = "1"
chrono = "0.4"
compact_str = "0.8"
compact_str = "0.7"
dotenvy = "0.15"
ed25519-dalek = "2.1"
either = "1.9"
emojis = "0.6"
futures = "0.3"
futures-channel = "0.3"
futures-core = "0.3"
futures-util = "0.3"
headers = "0.4"
http = "1.0"
httpdate = "1"
hyper = "1.1"
idna = "1"
idna = "0.5"
indexmap = "2.2"
itertools = "0.13"
itertools = "0.12"
lru = "0.12"
miette = "7"
miette = "5.9"
nom = "7"
nom_locate = "4"
percent-encoding = "2.2"
priority-queue = "2.0"
quick-xml = "0.36"
redis = "0.26"
quick-xml = "0.31"
redis = "0.24"
regex = "1.9"
rsa = "0.9"
reqwest = "0.12"
@ -66,15 +64,14 @@ serde = "1"
serde_json = "1"
serde_urlencoded = "0.7"
sha2 = "0.10"
smallvec = "1.13"
strum = "0.26"
strum = "0.25"
tera = { version = "1", default-features = false }
thiserror = "1"
tokio = "1.24"
tokio-util = "0.7"
tokio-stream = "0.1"
toml = "0.8"
tower = "0.5"
tower = "0.4"
tower-http = "0.5"
tracing = "0.1"
tracing-subscriber = "0.3"

89
ext_model/src/note_model/mod.rs
View File

@ -1,12 +1,16 @@
pub mod data;
use ext_model_migration::SelectStatement;
use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr};
use sea_orm::{
Condition, EntityTrait, Iden, JoinType, QueryFilter, QueryOrder, QuerySelect, QueryTrait,
Select,
};
use std::sync::Arc;
use ck::{note, note_reaction, user_note_pining};
use data::{sub_interaction_reaction, sub_interaction_renote, NoteData};
use ext_model_migration::SelectStatement;
use magnetar_sdk::types::SpanFilter;
use sea_orm::sea_query::{Asterisk, Expr, IntoIden, Query, SelectExpr, SimpleExpr};
use sea_orm::{Condition, EntityTrait, Iden, JoinType, Order, QueryFilter, QuerySelect, QueryTrait, Select};
use std::sync::Arc;
use crate::model_ext::{
join_columns_default, AliasColumnExt, AliasSourceExt, AliasSuffixExt, CursorPaginationExt,
@ -15,9 +19,10 @@ use crate::model_ext::{
use crate::user_model::{UserResolveOptions, UserResolver};
use crate::{CalckeyDbError, CalckeyModel};
const INTERACTION_REACTION: &str = "iact.rct.";
const INTERACTION_RENOTE: &str = "iact.rnt.";
const USER: &str = "u.";
const PIN: &str = "pin.";
const INTERACTION_REACTION: &str = "interaction.reaction.";
const INTERACTION_RENOTE: &str = "interaction.renote.";
const USER: &str = "user.";
const REPLY: &str = "reply.";
const RENOTE: &str = "renote.";
@ -31,50 +36,16 @@ pub trait NoteVisibilityFilterFactory: Send + Sync {
fn with_note_and_user_tables(&self, note: &dyn Iden) -> SimpleExpr;
}
#[derive(Clone)]
pub enum NoteResolveMode {
Single(String),
Multiple(Vec<String>),
PinsFromUserId(String),
}
impl NoteResolveMode {
fn as_expr(&self) -> SimpleExpr {
let id_col = note::Entity.base_prefix().col(note::Column::Id);
match self {
NoteResolveMode::Single(id) => id_col.eq(id),
NoteResolveMode::Multiple(ids) => id_col.is_in(ids),
NoteResolveMode::PinsFromUserId(user_id) => {
let sub_query = Query::select()
.expr(SelectExpr {
expr: Expr::col(user_note_pining::Column::NoteId).into(),
alias: None,
window: None,
})
.from(user_note_pining::Entity)
.and_where(Expr::col(note_reaction::Column::UserId).eq(user_id))
.order_by_columns([
(user_note_pining::Column::CreatedAt, Order::Desc),
(user_note_pining::Column::Id, Order::Desc)
])
.take();
id_col.in_subquery(sub_query)
}
}
}
}
#[derive(Clone)]
pub struct NoteResolveOptions {
pub mode: Option<NoteResolveMode>,
pub ids: Option<Vec<String>>,
pub visibility_filter: Arc<dyn NoteVisibilityFilterFactory>,
pub time_range: Option<SpanFilter>,
pub limit: Option<u64>,
pub with_reply_target: bool,
pub with_renote_target: bool,
pub with_interactions_from: Option<String>, // User ID
pub only_pins_from: Option<String>, // User ID
pub user_options: UserResolveOptions,
}
@ -146,6 +117,16 @@ impl SelectNoteInteractionsExt for SelectStatement {
}
}
fn ids_into_expr(ids: &Vec<String>) -> SimpleExpr {
let id_col = note::Entity.base_prefix().col(note::Column::Id);
if ids.len() == 1 {
id_col.eq(&ids[0])
} else {
id_col.is_in(ids)
}
}
impl NoteResolver {
pub fn new(db: CalckeyModel, user_resolver: UserResolver) -> Self {
NoteResolver { db, user_resolver }
@ -160,7 +141,7 @@ impl NoteResolver {
.visibility_filter
.with_note_and_user_tables(&note::Entity.base_prefix());
let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr);
let id_filter = options.ids.as_ref().map(ids_into_expr);
let note = select
.filter(visibility_filter)
@ -180,11 +161,17 @@ impl NoteResolver {
let visibility_filter = options
.visibility_filter
.with_note_and_user_tables(&note::Entity.base_prefix());
let id_filter = options.mode.as_ref().map(NoteResolveMode::as_expr);
let id_filter = options.ids.as_ref().map(ids_into_expr);
let notes_select = select
.filter(visibility_filter)
.apply_if(id_filter, Select::<note::Entity>::filter);
.apply_if(id_filter, Select::<note::Entity>::filter)
.apply_if(options.only_pins_from.as_deref(), |s, _| {
s.order_by_desc(Expr::col((
note::Entity.base_prefix().into_iden().join_str(PIN),
user_note_pining::Column::CreatedAt,
)))
});
let notes = if let Some(pagination) = &options.time_range {
notes_select
@ -317,6 +304,16 @@ impl NoteResolver {
let mut select = Query::select();
select.from_as(note::Entity, note_tbl.clone().into_iden());
if let Some(pins_user) = &options.only_pins_from {
select
.join_columns(
JoinType::InnerJoin,
note::Relation::UserNotePining.with_from_alias(&note_tbl),
&note_tbl.join_str(PIN),
)
.and_where(note_tbl.col(user_note_pining::Column::UserId).eq(pins_user));
}
self.attach_note(
&mut select,
&note_tbl,

2
magnetar_mmm_parser/Cargo.toml
View File

@ -18,4 +18,4 @@ serde = { workspace = true, features = ["derive"] }
strum = { workspace = true, features = ["derive"] }
tracing = { workspace = true }
unicode-segmentation = { workspace = true }
quick-xml = { workspace = true, optional = true, features = ["serialize"] }
quick-xml = { workspace = "true", optional = true, features = ["serialize"] }

65
magnetar_mmm_parser/src/lib.rs
View File

@ -5,6 +5,7 @@ use std::marker::PhantomData;
use compact_str::{CompactString, ToCompactString};
use either::Either;
use nom::{IResult, Offset, Parser, Slice};
use nom::branch::alt;
use nom::bytes::complete::{tag, tag_no_case};
use nom::character::complete::{
@ -15,7 +16,6 @@ use nom::combinator::{eof, fail, map, not, opt, peek, recognize};
use nom::error::ErrorKind;
use nom::multi::{many0_count, many1, many1_count, many_till, separated_list1};
use nom::sequence::tuple;
use nom::{IResult, Offset, Parser, Slice};
use nom_locate::LocatedSpan;
use quick_xml::events::{BytesText, Event};
use serde::{Deserialize, Serialize};
@ -56,6 +56,7 @@ pub enum Token {
Sequence(Vec<Token>),
Quote(Box<Token>),
Small(Box<Token>),
BoldItalic(Box<Token>),
Bold(Box<Token>),
Italic(Box<Token>),
Center(Box<Token>),
@ -100,6 +101,7 @@ impl Token {
Token::Sequence(tokens) => tokens.first().and_then(Token::str_content_left),
Token::Quote(inner) => inner.str_content_left(),
Token::Small(inner) => inner.str_content_left(),
Token::BoldItalic(inner) => inner.str_content_left(),
Token::Bold(inner) => inner.str_content_left(),
Token::Italic(inner) => inner.str_content_left(),
Token::Center(inner) => inner.str_content_left(),
@ -122,6 +124,7 @@ impl Token {
Token::Sequence(tokens) => tokens.last().and_then(Token::str_content_right),
Token::Quote(inner) => inner.str_content_right(),
Token::Small(inner) => inner.str_content_right(),
Token::BoldItalic(inner) => inner.str_content_right(),
Token::Bold(inner) => inner.str_content_right(),
Token::Italic(inner) => inner.str_content_right(),
Token::Center(inner) => inner.str_content_right(),
@ -144,6 +147,7 @@ impl Token {
sequence @ Token::Sequence(_) => sequence.clone(),
Token::Quote(inner) => inner.inner(),
Token::Small(inner) => inner.inner(),
Token::BoldItalic(inner) => inner.inner(),
Token::Bold(inner) => inner.inner(),
Token::Italic(inner) => inner.inner(),
Token::Center(inner) => inner.inner(),
@ -209,6 +213,7 @@ impl Token {
}
Token::Quote(inner) => Token::Quote(Box::new(inner.merged())),
Token::Small(inner) => Token::Small(Box::new(inner.merged())),
Token::BoldItalic(inner) => Token::BoldItalic(Box::new(inner.merged())),
Token::Bold(inner) => Token::Bold(Box::new(inner.merged())),
Token::Italic(inner) => Token::Italic(Box::new(inner.merged())),
Token::Center(inner) => Token::Center(Box::new(inner.merged())),
@ -242,6 +247,7 @@ impl Token {
}
Token::Quote(inner)
| Token::Small(inner)
| Token::BoldItalic(inner)
| Token::Bold(inner)
| Token::Italic(inner)
| Token::Center(inner)
@ -260,6 +266,7 @@ impl Token {
.for_each(|tok| tok.walk_speech_transform(func));
}
Token::Small(inner)
| Token::BoldItalic(inner)
| Token::Bold(inner)
| Token::Italic(inner)
| Token::Center(inner)
@ -288,6 +295,16 @@ impl Token {
.create_element("small")
.write_inner_content(|w| inner.write(w))?;
}
Token::BoldItalic(inner) => {
writer
.create_element("b")
.write_inner_content::<_, quick_xml::Error>(|w| {
w.create_element("i")
.write_inner_content(|w| inner.write(w))?;
Ok(())
})?;
}
Token::Bold(inner) => {
writer
.create_element("b")
@ -775,6 +792,8 @@ impl Context {
fn base_bold_italic<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
alt((
self.partial(Self::tag_bold_italic_asterisk),
self.partial(Self::tag_bold_italic_underscore),
self.partial(Self::tag_bold_asterisk),
self.partial(Self::tag_italic_asterisk),
self.partial(Self::tag_bold_underscore),
@ -1014,7 +1033,7 @@ impl Context {
escape: bool,
matcher: Matcher<'a, 'b, T>,
fallback: Matcher<'a, 'b, S>,
) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token> + 'a
) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token> + '_
where
FOpen: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
FClose: Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
@ -1204,6 +1223,38 @@ impl Context {
)(input)
}
fn tag_bold_italic_asterisk<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
(tag("***"), FlankingRule::Lenient),
(tag("***"), FlankingRule::Lenient),
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
)(input)
}
fn tag_bold_italic_underscore<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
(tag("___"), FlankingRule::Strict),
(tag("___"), FlankingRule::Strict),
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
)(input)
}
fn tag_bold<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
self.tag_delimited(
tag_no_case("<b>"),
@ -1447,7 +1498,7 @@ impl Context {
return fail(input);
};
let grapheme = grapheme.trim_end_matches(['\u{200c}', '\u{200d}']);
let grapheme = grapheme.trim_end_matches(|c| c == '\u{200c}' || c == '\u{200d}');
let emoji = emojis::get(grapheme);
@ -1516,7 +1567,7 @@ impl Context {
})(input)?;
let (input, name) = map(
recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))),
recognize(many1(alt((alphanumeric1, recognize(one_of("-_")))))),
Span::into_fragment,
)(input)?;
@ -1540,7 +1591,7 @@ impl Context {
mention_type
};
let host =
host_opt.map(|(_, name)| name.trim_end_matches(['.', '-', '_']));
host_opt.map(|(_, name)| name.trim_end_matches(|c| matches!(c, '.' | '-' | '_')));
let input = host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before);
Ok((
@ -1695,7 +1746,7 @@ mod test {
use nom::bytes::complete::tag;
use crate::{to_xml_string, Context, Span, SpanMeta, Token, DEFAULT_DEPTH_LIMIT};
use crate::{Context, DEFAULT_DEPTH_LIMIT, Span, SpanMeta, to_xml_string, Token};
fn parse_full(string: &str) -> Token {
Context::default()
@ -1853,7 +1904,7 @@ mod test {
assert_eq!(
parse_full(r#"***bold italic***"#),
Token::Bold(Box::new(Token::Italic(Box::new(Token::PlainText("bold italic".into())))))
Token::BoldItalic(Box::new(Token::PlainText("bold italic".into())))
);
assert_eq!(

8
src/model/processing/note.rs
View File

@ -13,7 +13,7 @@ use magnetar_model::model_ext::AliasColumnExt;
use magnetar_model::note_model::data::{
sub_interaction_reaction, sub_interaction_renote, NoteData,
};
use magnetar_model::note_model::{NoteResolveMode, NoteResolveOptions, NoteVisibilityFilterFactory};
use magnetar_model::note_model::{NoteResolveOptions, NoteVisibilityFilterFactory};
use magnetar_model::poll::PollResolver;
use magnetar_model::sea_orm::sea_query::{PgFunc, Query, SimpleExpr};
use magnetar_model::sea_orm::{ActiveEnum, ColumnTrait, Iden, IntoSimpleExpr};
@ -518,7 +518,7 @@ impl NoteModel {
let note_resolver = ctx.service.db.get_note_resolver();
let Some(note) = note_resolver
.get_one(&NoteResolveOptions {
mode: Some(NoteResolveMode::Single(id.to_owned())),
ids: Some(vec![id.to_owned()]),
visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter(
ctx.self_user.as_deref().map(ck::user::Model::get_id),
)),
@ -540,6 +540,7 @@ impl NoteModel {
.map(str::to_string)
})
.flatten(),
only_pins_from: None,
})
.await?
else {
@ -557,7 +558,7 @@ impl NoteModel {
let note_resolver = ctx.service.db.get_note_resolver();
let notes = note_resolver
.get_many(&NoteResolveOptions {
mode: Some(NoteResolveMode::PinsFromUserId(pin_user.id.clone())),
ids: None,
visibility_filter: Arc::new(NoteVisibilityFilterModel.new_note_visibility_filter(
ctx.self_user.as_deref().map(ck::user::Model::get_id),
)),
@ -579,6 +580,7 @@ impl NoteModel {
.map(str::to_string)
})
.flatten(),
only_pins_from: Some(pin_user.id.clone()),
})
.await?;

6
src/model/processing/notification.rs
View File

@ -231,7 +231,7 @@ impl NotificationModel {
.get_single(
&NotificationResolveOptions {
note_options: NoteResolveOptions {
mode: None,
ids: None,
visibility_filter: Arc::new(
NoteVisibilityFilterModel.new_note_visibility_filter(Some(user_id)),
),
@ -240,6 +240,7 @@ impl NotificationModel {
with_reply_target: true,
with_renote_target: true,
with_interactions_from: self_id.map(str::to_string),
only_pins_from: None,
user_options: user_resolve_options.clone(),
},
user_options: user_resolve_options,
@ -293,7 +294,7 @@ impl NotificationModel {
.get(
&NotificationResolveOptions {
note_options: NoteResolveOptions {
mode: None,
ids: None,
visibility_filter: Arc::new(
NoteVisibilityFilterModel.new_note_visibility_filter(Some(id)),
),
@ -302,6 +303,7 @@ impl NotificationModel {
with_reply_target: true,
with_renote_target: true,
with_interactions_from: self_id.map(str::to_string),
only_pins_from: None,
user_options: user_resolve_options.clone(),
},
user_options: user_resolve_options,

4
src/service/emoji_cache.rs
View File

@ -4,11 +4,11 @@ use magnetar_model::emoji::{EmojiResolver, EmojiTag};
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::collections::HashSet;
use std::sync::Arc;
use strum::VariantNames;
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum EmojiCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

4
src/service/generic_id_cache.rs
View File

@ -5,11 +5,11 @@ use magnetar_model::{CalckeyDbError, CalckeyModel};
use std::marker::PhantomData;
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::VariantNames;
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum GenericIdCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

4
src/service/instance_cache.rs
View File

@ -4,11 +4,11 @@ use magnetar_common::config::MagnetarConfig;
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::VariantNames;
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum RemoteInstanceCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

4
src/service/instance_meta_cache.rs
View File

@ -2,12 +2,12 @@ use crate::web::ApiError;
use magnetar_model::{ck, CalckeyDbError, CalckeyModel};
use std::sync::Arc;
use std::time::{Duration, Instant};
use strum::VariantNames;
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::{mpsc, oneshot};
use tracing::error;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum InstanceMetaCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),

6
src/service/local_user_cache.rs
View File

@ -2,20 +2,20 @@ use std::collections::HashMap;
use std::sync::Arc;
use cached::{Cached, TimedCache};
use strum::{EnumVariantNames, VariantNames};
use strum::EnumVariantNames;
use thiserror::Error;
use tokio::sync::Mutex;
use tracing::error;
use magnetar_common::config::MagnetarConfig;
use magnetar_model::{
ck, CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub,
CalckeyCache, CalckeyCacheError, CalckeyDbError, CalckeyModel, CalckeySub, ck,
InternalStreamMessage, SubMessage,
};
use crate::web::ApiError;
#[derive(Debug, Error, VariantNames)]
#[derive(Debug, Error, EnumVariantNames)]
pub enum UserCacheError {
#[error("Database error: {0}")]
DbError(#[from] CalckeyDbError),