magnetar/magnetar_mmm_parser/src/lib.rs

1581 lines
52 KiB
Rust
Raw Normal View History

use either::Either;
2023-10-01 21:04:32 +00:00
use nom::branch::alt;
use nom::bytes::complete::tag;
use nom::character::complete::{
2023-10-05 17:09:26 +00:00
alpha1, alphanumeric1, anychar, char as one_char, line_ending, not_line_ending, one_of, space1,
tab,
};
2023-10-05 18:05:03 +00:00
use nom::combinator::{eof, fail, map, not, opt, recognize};
2023-10-01 21:04:32 +00:00
use nom::error::ErrorKind;
2023-10-05 22:17:52 +00:00
use nom::multi::{many0, many0_count, many1, many1_count, many_till, separated_list1};
2023-10-01 21:04:32 +00:00
use nom::sequence::tuple;
use nom::{IResult, Offset, Slice};
use nom_locate::LocatedSpan;
use std::borrow::Cow;
use std::collections::HashMap;
use std::convert::{identity, Infallible};
2023-10-05 19:21:23 +00:00
use unicode_segmentation::UnicodeSegmentation;
2023-10-01 21:04:32 +00:00
2023-10-05 22:17:52 +00:00
#[derive(Copy, Clone, Debug, Eq, PartialEq)]
2023-10-05 18:05:03 +00:00
pub enum MentionType {
Community,
User,
}
impl MentionType {
pub fn to_char(&self) -> char {
match self {
MentionType::Community => '!',
MentionType::User => '@',
}
}
}
2023-10-05 22:17:52 +00:00
#[derive(Clone, Debug, Eq, PartialEq)]
pub enum Token<'a> {
2023-10-01 21:04:32 +00:00
PlainText(Cow<'a, str>),
Sequence(Vec<Token<'a>>),
Quote(Box<Token<'a>>),
Small(Box<Token<'a>>),
BoldItalic(Box<Token<'a>>),
Bold(Box<Token<'a>>),
Italic(Box<Token<'a>>),
Center(Box<Token<'a>>),
Strikethrough(Box<Token<'a>>),
PlainTag(Cow<'a, str>),
InlineCode(Cow<'a, str>),
InlineMath(Cow<'a, str>),
2023-10-05 17:09:26 +00:00
UrlRaw(Cow<'a, str>),
UrlNoEmbed(Cow<'a, str>),
Link {
2023-10-07 18:40:01 +00:00
label: Box<Token<'a>>,
2023-10-05 17:09:26 +00:00
href: Cow<'a, str>,
embed: bool,
},
BlockCode {
lang: Option<Cow<'a, str>>,
inner: Cow<'a, str>,
},
BlockMath(Cow<'a, str>),
Function {
name: Cow<'a, str>,
params: HashMap<Cow<'a, str>, Option<Cow<'a, str>>>,
inner: Box<Token<'a>>,
},
2023-10-05 18:05:03 +00:00
Mention {
name: Cow<'a, str>,
host: Option<Cow<'a, str>>,
mention_type: MentionType,
},
2023-10-05 19:21:23 +00:00
UnicodeEmoji(Cow<'a, str>),
ShortcodeEmoji(Cow<'a, str>),
2023-10-05 20:02:46 +00:00
Hashtag(Cow<'a, str>),
2023-10-01 21:04:32 +00:00
}
impl Token<'_> {
fn owned(&self) -> Token<'static> {
match self {
Token::PlainText(text) => Token::PlainText(Cow::Owned(text.clone().into_owned())),
Token::Sequence(tokens) => Token::Sequence(tokens.iter().map(Token::owned).collect()),
Token::Quote(inner) => Token::Quote(Box::new(inner.owned())),
Token::Small(inner) => Token::Small(Box::new(inner.owned())),
Token::BoldItalic(inner) => Token::BoldItalic(Box::new(inner.owned())),
Token::Bold(inner) => Token::Bold(Box::new(inner.owned())),
Token::Italic(inner) => Token::Italic(Box::new(inner.owned())),
Token::Center(inner) => Token::Center(Box::new(inner.owned())),
Token::Strikethrough(inner) => Token::Strikethrough(Box::new(inner.owned())),
Token::PlainTag(tag) => Token::PlainTag(Cow::Owned(tag.clone().into_owned())),
Token::InlineCode(code) => Token::InlineCode(Cow::Owned(code.clone().into_owned())),
Token::InlineMath(math) => Token::InlineMath(Cow::Owned(math.clone().into_owned())),
2023-10-05 17:09:26 +00:00
Token::UrlRaw(url) => Token::UrlRaw(Cow::Owned(url.clone().into_owned())),
Token::UrlNoEmbed(url) => Token::UrlNoEmbed(Cow::Owned(url.clone().into_owned())),
Token::Link { embed, label, href } => Token::Link {
embed: *embed,
2023-10-07 18:40:01 +00:00
label: Box::new(label.owned()),
2023-10-05 17:09:26 +00:00
href: Cow::Owned(href.clone().into_owned()),
},
Token::BlockCode { inner, lang } => Token::BlockCode {
lang: lang.as_ref().map(|l| Cow::Owned(l.clone().into_owned())),
inner: Cow::Owned(inner.clone().into_owned()),
},
Token::BlockMath(math) => Token::BlockMath(Cow::Owned(math.clone().into_owned())),
Token::Function {
name,
params,
inner,
} => Token::Function {
name: Cow::Owned(name.clone().into_owned()),
params: params
.iter()
.map(|(k, v)| {
(
Cow::Owned(k.clone().into_owned()),
v.as_ref().map(|val| Cow::Owned(val.clone().into_owned())),
)
})
.collect(),
inner: Box::new(inner.owned()),
},
2023-10-05 18:05:03 +00:00
Token::Mention {
name,
host,
mention_type,
} => Token::Mention {
name: Cow::Owned(name.clone().into_owned()),
host: host.as_ref().map(|v| Cow::Owned(v.clone().into_owned())),
mention_type: *mention_type,
},
2023-10-05 19:21:23 +00:00
Token::UnicodeEmoji(code) => Token::UnicodeEmoji(Cow::Owned(code.clone().into_owned())),
Token::ShortcodeEmoji(shortcode) => {
Token::ShortcodeEmoji(Cow::Owned(shortcode.clone().into_owned()))
}
2023-10-05 20:02:46 +00:00
Token::Hashtag(url) => Token::Hashtag(Cow::Owned(url.clone().into_owned())),
2023-10-01 21:04:32 +00:00
}
}
2023-10-05 22:17:52 +00:00
fn merged(&self) -> Token {
match self {
Token::Sequence(tokens) => {
let tokens_multi = tokens.iter().fold(Vec::new(), |mut acc, tok| {
if let Some(Token::PlainText(last)) = acc.last_mut() {
if let Token::PlainText(tok_text) = tok {
*last = Cow::from(last.to_string() + tok_text.as_ref());
return acc;
}
}
if let Token::Sequence(seq) = tok {
let items = seq.iter().map(Token::merged).flat_map(|t| match t {
Token::Sequence(seq) => Either::Left(seq.into_iter()),
other => Either::Right(std::iter::once(other)),
});
for item in items {
if let Some(Token::PlainText(last)) = acc.last_mut() {
if let Token::PlainText(tok_text) = item {
*last = Cow::from(last.to_string() + tok_text.as_ref());
continue;
}
}
acc.push(item);
}
return acc;
}
2023-10-05 22:17:52 +00:00
acc.push(tok.merged());
acc
});
if tokens_multi.len() == 1 {
return tokens_multi.into_iter().next().unwrap();
}
Token::Sequence(tokens_multi)
}
Token::Quote(inner) => Token::Quote(Box::new(inner.merged())),
Token::Small(inner) => Token::Small(Box::new(inner.merged())),
Token::BoldItalic(inner) => Token::BoldItalic(Box::new(inner.merged())),
Token::Bold(inner) => Token::Bold(Box::new(inner.merged())),
Token::Italic(inner) => Token::Italic(Box::new(inner.merged())),
Token::Center(inner) => Token::Center(Box::new(inner.merged())),
Token::Strikethrough(inner) => Token::Strikethrough(Box::new(inner.merged())),
2023-10-07 18:40:01 +00:00
Token::Link { embed, label, href } => Token::Link {
label: Box::new(label.merged()),
href: href.clone(),
embed: *embed,
},
2023-10-05 22:17:52 +00:00
Token::Function {
name,
params,
inner,
} => Token::Function {
name: name.clone(),
params: params.clone(),
inner: Box::new(inner.merged()),
},
other => other.clone(),
}
}
2023-10-01 21:04:32 +00:00
}
type Span<'a> = LocatedSpan<&'a str>;
trait SliceOffset {
fn up_to(&self, other: &Self) -> Self;
fn fragment_between<'a>(&self, other: &Self) -> &'a str
where
Self: 'a;
}
impl SliceOffset for Span<'_> {
fn up_to(&self, other: &Self) -> Self {
self.slice(..self.offset(other))
}
fn fragment_between<'a>(&self, other: &Self) -> &'a str
where
Self: 'a,
{
self.up_to(other).into_fragment()
}
}
#[inline]
fn boxing_token<'a>(func: impl Fn(Box<Token<'a>>) -> Token<'a>) -> impl Fn(Token<'a>) -> Token<'a> {
move |tokens| func(Box::new(tokens))
2023-10-01 21:04:32 +00:00
}
#[inline]
fn collect_sequence<'a, T>(
func: impl Fn(Vec<T>) -> Token<'a>,
transform: impl Fn(Token<'a>) -> Token<'a>,
) -> impl Fn(&mut dyn Iterator<Item = T>) -> Token<'a> {
move |tokens| transform(func(tokens.collect()))
}
#[inline]
fn collect_char_sequence<'a>(
2023-10-01 21:04:32 +00:00
func: impl Fn(Cow<'a, str>) -> Token<'a>,
) -> impl Fn(&mut dyn Iterator<Item = char>) -> Token<'a> {
move |chars| func(Cow::Owned(chars.collect()))
2023-10-01 21:04:32 +00:00
}
fn spliced<'a>(
segments: &[Span<'a>],
func: impl Fn(Span) -> IResult<Span, Token>,
parent: Span<'a>,
) -> IResult<Span<'a>, Token<'static>, nom::error::Error<Span<'a>>> {
let combined = segments
.iter()
.copied()
.map(Span::into_fragment)
.collect::<Vec<_>>()
.join("\n");
2023-10-01 21:04:32 +00:00
let cum_offset_combined = segments
.iter()
.scan(0, |acc, &x| {
*acc += x.len();
Some(*acc)
})
.collect::<Vec<_>>();
let current_seg = |input: Span| {
cum_offset_combined
.iter()
.enumerate()
.take_while(|(_, &o)| o > input.location_offset())
2023-10-01 21:04:32 +00:00
.map(|(i, o)| (segments[i], o))
.last()
};
type NE<E> = nom::Err<E>;
type NomError<'x> = nom::error::Error<Span<'x>>;
let quote_span = Span::new(&combined);
let (input, inner) = match func(quote_span) {
Ok((input, token)) => (input, token.owned()),
Err(e) => {
return match e {
NE::Error(e) => {
let offset_new = e.input.location_offset();
if let Some((seg_parent, offset_seg_new)) = current_seg(e.input) {
let offset = offset_new - offset_seg_new;
let offset_orig = offset + seg_parent.location_offset();
Err(NE::Error(NomError::new(
Span::new(&parent.into_fragment()[offset_orig..]),
e.code,
)))
} else {
// ???
Err(NE::Failure(NomError::new(parent, ErrorKind::Fail)))
}
}
NE::Failure(e) => Err(NE::Error(NomError::new(parent, e.code))),
NE::Incomplete(i) => Err(NE::Incomplete(i)),
};
}
};
let out = if let Some((seg_parent, offset_seg_new)) = current_seg(input) {
let offset = input.location_offset() - offset_seg_new;
let offset_orig = offset + seg_parent.location_offset();
parent.slice(offset_orig..)
} else {
parent
};
Ok((out, inner.owned()))
2023-10-01 21:04:32 +00:00
}
fn space(input: Span) -> IResult<Span, Token> {
2023-10-05 17:09:26 +00:00
let (input, frag) = recognize(alt((one_char('\u{0020}'), one_char('\u{3000}'), tab)))(input)?;
Ok((input, Token::PlainText(frag.into_fragment().into())))
2023-10-01 21:04:32 +00:00
}
struct Matcher<'a, 'b, T> {
matcher_inner: &'a (dyn Fn(Span<'b>) -> IResult<Span<'b>, T> + 'a),
collector: &'a (dyn Fn(&mut dyn Iterator<Item = T>) -> Token<'b> + 'a),
_phantom_closure: std::marker::PhantomData<&'a ()>,
_phantom_data: std::marker::PhantomData<&'b ()>,
_phantom_output: std::marker::PhantomData<fn() -> T>,
}
impl<'a, 'b, T> Matcher<'a, 'b, T> {
fn new(
matcher_inner: &'a (dyn Fn(Span<'b>) -> IResult<Span<'b>, T> + 'a),
collector: &'a (dyn Fn(&mut dyn Iterator<Item = T>) -> Token<'b> + 'a),
) -> Self {
Self {
matcher_inner,
collector,
_phantom_closure: std::marker::PhantomData,
_phantom_data: std::marker::PhantomData,
_phantom_output: std::marker::PhantomData,
}
}
}
impl<'a, 'b> Matcher<'a, 'b, Infallible> {
// Don't break this invariant, else a monster will come at night and eat all your socks
fn reject() -> Self {
Self {
matcher_inner: &fail::<_, Infallible, _>,
collector: &|_| unreachable!(),
_phantom_closure: std::marker::PhantomData,
_phantom_data: std::marker::PhantomData,
_phantom_output: std::marker::PhantomData,
}
}
}
2023-10-01 21:04:32 +00:00
struct Context;
impl Context {
2023-10-04 17:44:27 +00:00
#[inline]
2023-10-05 22:17:52 +00:00
const fn partial(
2023-10-01 21:04:32 +00:00
&self,
2023-10-05 22:17:52 +00:00
func: impl for<'a> Fn(&Self, Span<'a>) -> IResult<Span<'a>, Token<'a>> + 'static,
) -> impl for<'a> Fn(Span<'a>) -> IResult<Span<'a>, Token<'a>> + '_ {
2023-10-01 21:04:32 +00:00
move |input| func(self, input)
}
2023-10-05 22:17:52 +00:00
fn full<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
map(many1(self.partial(Self::full_single)), Token::Sequence)(input)
2023-10-01 21:04:32 +00:00
}
fn inline<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
2023-10-05 22:17:52 +00:00
map(many1(self.partial(Self::inline_single)), Token::Sequence)(input)
}
fn inline_label_safe<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
map(
many1(self.partial(Self::inline_label_safe_single)),
Token::Sequence,
)(input)
}
fn base_bold_italic<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
alt((
self.partial(Self::tag_bold_italic_asterisk),
self.partial(Self::tag_bold_italic_underscore),
self.partial(Self::tag_bold_asterisk),
self.partial(Self::tag_italic_asterisk),
self.partial(Self::tag_bold_underscore),
self.partial(Self::tag_italic_underscore),
))(input)
}
fn full_single<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let (input, token) = alt((
self.partial(Self::unicode_emoji),
2023-10-07 18:40:01 +00:00
alt((
self.partial(Self::tag_block_center),
self.partial(Self::tag_small),
self.partial(Self::tag_plain),
self.partial(Self::tag_bold),
self.partial(Self::tag_italic),
self.partial(Self::tag_strikethrough),
)),
2023-10-05 22:17:52 +00:00
self.partial(Self::url_no_embed),
self.partial(Self::base_bold_italic),
self.partial(Self::tag_block_code),
self.partial(Self::tag_inline_code),
self.partial(Self::tag_quote),
self.partial(Self::tag_block_math),
self.partial(Self::tag_inline_math),
self.partial(Self::tag_strikethrough_tilde),
self.partial(Self::tag_func),
self.partial(Self::tag_mention),
self.partial(Self::tag_hashtag),
self.partial(Self::shortcode_emoji),
2023-10-07 18:40:01 +00:00
self.partial(Self::link),
2023-10-05 22:17:52 +00:00
self.partial(Self::raw_url),
self.partial(Self::text),
))(input)?;
2023-10-01 21:04:32 +00:00
Ok((input, token))
}
2023-10-05 22:17:52 +00:00
fn inline_single<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let (input, token) = alt((
self.partial(Self::unicode_emoji),
self.partial(Self::tag_small),
self.partial(Self::tag_plain),
self.partial(Self::tag_bold),
self.partial(Self::tag_italic),
self.partial(Self::tag_strikethrough),
self.partial(Self::url_no_embed),
self.partial(Self::base_bold_italic),
self.partial(Self::tag_inline_code),
self.partial(Self::tag_inline_math),
self.partial(Self::tag_strikethrough_tilde),
self.partial(Self::tag_func),
self.partial(Self::tag_mention),
self.partial(Self::tag_hashtag),
self.partial(Self::shortcode_emoji),
2023-10-07 18:40:01 +00:00
self.partial(Self::link),
2023-10-05 22:17:52 +00:00
self.partial(Self::raw_url),
self.partial(Self::text),
))(input)?;
Ok((input, token))
}
fn inline_non_formatting_single<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let (input, token) = alt((
self.partial(Self::unicode_emoji),
self.partial(Self::url_no_embed),
self.partial(Self::tag_inline_code),
self.partial(Self::tag_inline_math),
self.partial(Self::tag_func),
self.partial(Self::tag_mention),
self.partial(Self::tag_hashtag),
self.partial(Self::shortcode_emoji),
self.partial(Self::raw_url),
self.partial(Self::text),
))(input)?;
Ok((input, token))
}
2023-10-05 22:17:52 +00:00
fn inline_label_safe_single<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let (input, token) = alt((
self.partial(Self::unicode_emoji),
self.partial(Self::tag_small),
self.partial(Self::tag_plain),
self.partial(Self::tag_bold),
self.partial(Self::tag_italic),
self.partial(Self::tag_strikethrough),
self.partial(Self::base_bold_italic),
self.partial(Self::tag_strikethrough_tilde),
self.partial(Self::tag_func),
self.partial(Self::shortcode_emoji),
self.partial(Self::text),
))(input)?;
2023-10-05 17:09:26 +00:00
Ok((input, token))
}
2023-10-01 21:04:32 +00:00
fn tag_quote<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let (input, leading_spaces) = tuple((opt(line_ending), opt(line_ending)))(input)?;
if let (None, None) = leading_spaces {
2023-10-05 22:17:52 +00:00
if input.get_column() != 1 {
2023-10-01 21:04:32 +00:00
return fail(input);
}
}
let quote_line = |input| tuple((tag(">"), opt(space), not_line_ending))(input);
let orig_input = input;
let (input, lines) = separated_list1(line_ending, quote_line)(input)?;
let quote_lines = lines
.into_iter()
.map(|(_, _, text)| text)
.collect::<Vec<_>>();
if quote_lines.len() == 1
&& quote_lines
.iter()
.map(Span::fragment)
.copied()
.any(&str::is_empty)
{
return fail(input);
}
let (_, inner) = spliced(&quote_lines, self.partial(Self::full), orig_input)?;
2023-10-01 21:04:32 +00:00
let (input, _) = tuple((opt(line_ending), opt(line_ending)))(input)?;
Ok((input, Token::Quote(Box::new(inner))))
}
fn tag_block_center<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let tag_start = &tag("<center>");
let tag_end = &tag("</center>");
let (input, _) = opt(line_ending)(input)?;
2023-10-05 22:17:52 +00:00
if input.get_column() != 1 {
return fail(input);
}
let (input, _) = tag_start(input)?;
let (input, _) = opt(line_ending)(input)?;
2023-10-05 22:17:52 +00:00
let (input, (center_seq, _)) = many_till(
self.partial(Self::inline_single),
tuple((opt(space1), opt(line_ending), tag_end)),
2023-10-05 22:17:52 +00:00
)(input)?;
Ok((
input,
boxing_token(Token::Center)(Token::Sequence(center_seq)),
))
}
2023-10-05 20:25:29 +00:00
fn tag_block_code<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let delim = &tag("```");
let (input, _) = opt(line_ending)(input)?;
2023-10-05 22:17:52 +00:00
if input.get_column() != 1 {
2023-10-05 20:25:29 +00:00
return fail(input);
}
let (input, _) = delim(input)?;
let (input, lang) = opt(map(
recognize(many1(tuple((not(delim), not_line_ending)))),
Span::into_fragment,
))(input)?;
let (input, _) = line_ending(input)?;
let (input, code) = map(
recognize(many1_count(tuple((
not(tuple((line_ending, delim))),
anychar,
)))),
Span::into_fragment,
)(input)?;
let (input, _) = line_ending(input)?;
let (input, _) = delim(input)?;
let (input, _) = many0(space)(input)?;
2023-10-05 22:17:52 +00:00
let (input, _) = not(not(line_ending))(input)?;
2023-10-05 20:25:29 +00:00
let (input, _) = opt(line_ending)(input)?;
Ok((
input,
Token::BlockCode {
lang: lang.map(<&str>::into),
inner: code.into(),
},
))
}
fn tag_block_math<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
2023-10-04 17:44:27 +00:00
let start = &tag("\\[");
let end = &tag("\\]");
let (input, _) = opt(line_ending)(input)?;
2023-10-05 22:17:52 +00:00
if input.get_column() != 1 {
return fail(input);
}
2023-10-04 17:44:27 +00:00
let (input, _) = start(input)?;
let (input, _) = opt(line_ending)(input)?;
let (input, math_span) = recognize(many1_count(tuple((
2023-10-04 17:44:27 +00:00
not(tuple((opt(line_ending), end))),
not_line_ending,
))))(input)?;
let (input, _) = opt(line_ending)(input)?;
2023-10-04 17:44:27 +00:00
let (input, _) = end(input)?;
let (input, _) = many0(space)(input)?;
let (input, _) = not(not_line_ending)(input)?;
let (input, _) = opt(line_ending)(input)?;
Ok((
input,
Token::BlockMath(Cow::Borrowed(math_span.into_fragment())),
))
}
#[inline]
fn tag_delimited<'a, 'b: 'a, T, S>(
2023-10-01 21:04:32 +00:00
&'a self,
opening_tag: impl Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
closing_tag: impl Fn(Span<'b>) -> IResult<Span<'b>, Span<'b>> + 'a,
2023-10-01 21:04:32 +00:00
escape: bool,
matcher: Matcher<'a, 'b, T>,
fallback: Matcher<'a, 'b, S>,
2023-10-01 21:04:32 +00:00
) -> impl Fn(Span<'b>) -> IResult<Span<'b>, Token<'b>> + '_ {
move |input| {
if escape {
if let Ok((input_escaped, (_, mark))) = tuple((tag("\\"), &opening_tag))(input) {
return Ok((
input_escaped,
Token::PlainText(Cow::Borrowed(mark.fragment())),
));
2023-10-01 21:04:32 +00:00
}
}
let begin = input;
let (post_open, _) = opening_tag(input)?;
let res = tuple((
many1(tuple((not(&closing_tag), &matcher.matcher_inner))),
&closing_tag,
2023-10-01 21:04:32 +00:00
))(post_open);
if let Err(nom::Err::Error(nom::error::Error {
input: input_past_err,
..
})) = res
{
let res_fallback = tuple((
many1(tuple((not(&closing_tag), &fallback.matcher_inner))),
&closing_tag,
))(post_open);
if res_fallback.is_err() {
return Ok((
input_past_err,
Token::PlainText(begin.fragment_between(&input_past_err).into()),
));
}
let (input, (inner, closing)) = res_fallback.unwrap();
let mut inner = inner.into_iter().map(|(_, t)| t);
2023-10-01 21:04:32 +00:00
return Ok((
input,
Token::Sequence(vec![
Token::PlainText(begin.fragment_between(&post_open).into()),
((fallback.collector)(&mut inner)),
Token::PlainText(closing.into_fragment().into()),
]),
2023-10-01 21:04:32 +00:00
));
}
let (input, (inner, _)) = res?;
let mut inner = inner.into_iter().map(|(_, t)| t);
2023-10-01 21:04:32 +00:00
Ok((input, (matcher.collector)(&mut inner)))
2023-10-01 21:04:32 +00:00
}
}
fn tag_func<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let (input, _) = tag("$[")(input)?;
let func_ident = |input| {
recognize(tuple((
many1_count(alt((alpha1, tag("_")))),
many0_count(alt((alphanumeric1, tag("_")))),
)))(input)
};
let param_value = recognize(many1_count(alt((
alphanumeric1,
tag("."),
tag("-"),
tag("_"),
))));
2023-10-05 22:17:52 +00:00
let (input, func_name) = map(func_ident, Span::into_fragment)(input)?;
let arg = tuple((func_ident, opt(tuple((tag("="), param_value)))));
let (input, args) =
opt(tuple((one_char('.'), separated_list1(one_char(','), arg))))(input)?;
let args_out = args.map_or_else(HashMap::new, |(_, items)| {
items
.into_iter()
.map(|(k, v)| {
(
Cow::from(k.into_fragment()),
v.map(|(_, val)| Cow::from(val.into_fragment())),
)
})
.collect::<HashMap<_, _>>()
});
2023-10-05 22:17:52 +00:00
let (input, _) = opt(space)(input)?;
2023-10-05 22:17:52 +00:00
let (input, (inner, _)) = many_till(self.partial(Self::inline_single), tag("]"))(input)?;
Ok((
input,
Token::Function {
name: Cow::from(func_name),
params: args_out,
2023-10-05 22:17:52 +00:00
inner: Box::new(Token::Sequence(inner)),
},
))
}
2023-10-05 20:12:51 +00:00
fn tag_plain<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let opening_tag = &tag("<small>");
let closing_tag = &tag("</small>");
let (input, _) = opening_tag(input)?;
let (input, text) = map(
recognize(many1(tuple((not_line_ending, not(closing_tag))))),
Span::into_fragment,
)(input)?;
let (input, _) = closing_tag(input)?;
Ok((input, Token::PlainTag(text.into())))
}
2023-10-01 21:04:32 +00:00
fn tag_small<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("<small>"),
tag("</small>"),
2023-10-01 21:04:32 +00:00
false,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::Small)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
// TODO: CommonMark flanking rules
fn tag_bold_italic_asterisk<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("***"),
tag("***"),
2023-10-01 21:04:32 +00:00
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
// TODO: CommonMark flanking rules
fn tag_bold_italic_underscore<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("___"),
tag("___"),
2023-10-01 21:04:32 +00:00
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::BoldItalic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
fn tag_bold<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("<b>"),
tag("</b>"),
2023-10-01 21:04:32 +00:00
false,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::Bold)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
// TODO: CommonMark flanking rules
fn tag_bold_asterisk<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("**"),
tag("**"),
2023-10-01 21:04:32 +00:00
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::Bold)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
// TODO: CommonMark flanking rules
fn tag_bold_underscore<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("__"),
tag("__"),
2023-10-01 21:04:32 +00:00
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::Bold)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
fn tag_italic<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("<i>"),
tag("</i>"),
2023-10-01 21:04:32 +00:00
false,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::Italic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
// TODO: CommonMark flanking rules
fn tag_italic_asterisk<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("*"),
tag("*"),
2023-10-01 21:04:32 +00:00
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::Italic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
// TODO: CommonMark flanking rules
fn tag_italic_underscore<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("_"),
tag("_"),
2023-10-01 21:04:32 +00:00
true,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::Italic)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
fn tag_strikethrough<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("<s>"),
tag("</s>"),
2023-10-01 21:04:32 +00:00
false,
Matcher::new(
&self.partial(Self::inline_single),
&collect_sequence(Token::Sequence, boxing_token(Token::Strikethrough)),
),
Matcher::new(
&self.partial(Self::inline_non_formatting_single),
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
// TODO: CommonMark flanking rules
fn tag_strikethrough_tilde<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("~~"),
tag("~~"),
2023-10-01 21:04:32 +00:00
true,
Matcher::new(
&move |input| {
map(
tuple(((not(line_ending)), self.partial(Self::inline_single))),
|(_, captured)| captured,
)(input)
},
&collect_sequence(Token::Sequence, boxing_token(Token::Strikethrough)),
),
Matcher::new(
&move |input| {
map(
tuple((
(not(line_ending)),
self.partial(Self::inline_non_formatting_single),
)),
|(_, captured)| captured,
)(input)
},
&collect_sequence(Token::Sequence, identity),
),
2023-10-01 21:04:32 +00:00
)(input)
}
fn tag_inline_code<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("`"),
|input| alt((tag("`"), tag("´")))(input),
2023-10-01 21:04:32 +00:00
true,
Matcher::new(
&move |input| {
map(
tuple((not(alt((tag("`"), tag("´"), line_ending))), anychar)),
|(_, captured)| captured,
)(input)
},
&collect_char_sequence(Token::InlineCode),
),
Matcher::reject(),
2023-10-01 21:04:32 +00:00
)(input)
}
fn tag_inline_math<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
self.tag_delimited(
tag("\\("),
tag("\\)"),
2023-10-01 21:04:32 +00:00
false,
Matcher::new(
&move |input| {
map(tuple((not(line_ending), anychar)), |(_, captured)| captured)(input)
},
&collect_char_sequence(Token::InlineMath),
),
Matcher::reject(),
2023-10-01 21:04:32 +00:00
)(input)
}
fn text<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
2023-10-05 22:17:52 +00:00
let (input, text) = map(recognize(anychar), Span::into_fragment)(input)?;
Ok((input, Token::PlainText(text.into())))
2023-10-01 21:04:32 +00:00
}
2023-10-05 17:09:26 +00:00
2023-10-05 22:17:52 +00:00
fn raw_url<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
2023-10-05 17:09:26 +00:00
let (input, url_span) = recognize(tuple((
protocol,
url_chars(|input| not(url_chars_base)(input), false),
)))(input)?;
let url = url_span.into_fragment();
let url_bytes = url.as_bytes();
// Strip punctuation at the end of sentences that might have been consumed as a part of the URL
let final_url = if matches!(url_bytes.last(), Some(b'.' | b',' | b'?')) {
url.slice(..url.len() - 1)
} else {
url
};
Ok((input, Token::UrlRaw(Cow::from(final_url))))
}
fn url_no_embed<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let (input, _) = tag("<")(input)?;
let (input, url_span) = recognize(tuple((protocol, url_chars(tag(">"), true))))(input)?;
let (input, _) = tag(">")(input)?;
2023-10-07 19:22:21 +00:00
Ok((
input,
Token::UrlNoEmbed(Cow::from(url_span.into_fragment())),
))
2023-10-05 17:09:26 +00:00
}
fn link<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let (input, no_embed) = opt(tag("?"))(input)?;
let (input, _) = tag("[")(input)?;
let (input, _) = not(tag("["))(input)?;
2023-10-07 18:40:01 +00:00
let (input, (label_tok, _)) =
many_till(self.partial(Self::inline_label_safe_single), tag("]("))(input)?;
let (input, url_span) = recognize(tuple((protocol, url_chars(tag(")"), true))))(input)?;
2023-10-05 17:09:26 +00:00
let (input, _) = tag(")")(input)?;
Ok((
input,
Token::Link {
2023-10-07 18:40:01 +00:00
label: Box::new(Token::Sequence(label_tok)),
2023-10-05 17:09:26 +00:00
href: url_span.into_fragment().into(),
embed: no_embed.is_none(),
},
))
}
2023-10-05 18:05:03 +00:00
2023-10-05 19:21:23 +00:00
fn unicode_emoji<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
let frag = input.fragment();
let Some(grapheme) = frag.graphemes(true).next() else {
return fail(input);
};
let grapheme = grapheme.trim_end_matches(|c| c == '\u{200c}' || c == '\u{200d}');
2023-10-05 19:21:23 +00:00
let emoji = emojis::get(grapheme);
if emoji.is_none() {
return fail(input);
}
Ok((
input.slice(grapheme.len()..),
Token::UnicodeEmoji(grapheme.into()),
))
}
2023-10-05 20:32:53 +00:00
fn shortcode_emoji<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
// TODO: Fail when preceded by alphanumerics
let (input, _) = tag(":")(input)?;
let (input, shortcode) = map(
recognize(many1(alt((alphanumeric1, recognize(one_of("_+-")))))),
Span::into_fragment,
)(input)?;
let (input, _) = tag(":")(input)?;
let (input, _) = not(alphanumeric1)(input)?;
Ok((input, Token::ShortcodeEmoji(shortcode.into())))
}
fn tag_mention<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
2023-10-05 18:05:03 +00:00
// TODO: Escaping and skip when preceded by alphanumerics
let tags = one_of("@!");
let (input, mention_type) = map(tags, |c| match c {
'@' => MentionType::User,
'!' => MentionType::Community,
_ => unreachable!(),
})(input)?;
let (input, name) = map(
recognize(many1(alt((alphanumeric1, recognize(one_of("-_")))))),
Span::into_fragment,
)(input)?;
2023-10-07 18:40:01 +00:00
let before = input;
let (_, host) = map(
2023-10-05 18:05:03 +00:00
opt(tuple((
tag("@"),
map(
2023-10-07 18:40:01 +00:00
recognize(many1(alt((alphanumeric1, recognize(one_of("-_.")))))),
2023-10-05 18:05:03 +00:00
Span::into_fragment,
),
))),
|maybe_tag_host| maybe_tag_host.map(|(_, host)| host),
)(input)?;
2023-10-07 18:40:01 +00:00
let host = host.map(|h| h.trim_end_matches(|c| matches!(c, '.' | '-' | '_')));
2023-10-05 18:05:03 +00:00
Ok((
2023-10-07 18:40:01 +00:00
host.map(|c| before.slice(c.len() + 1..)).unwrap_or(before),
2023-10-05 18:05:03 +00:00
Token::Mention {
mention_type,
name: name.into(),
host: host.map(|h| h.into()),
},
))
}
2023-10-05 20:02:46 +00:00
2023-10-05 22:17:52 +00:00
fn tag_hashtag<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token<'a>> {
2023-10-05 20:02:46 +00:00
// TODO: Skip when preceded by alphanumerics
let (input, _) = tag("#")(input)?;
let (input, hashtag_text) =
map(recognize(many1(hashtag_chars)), Span::into_fragment)(input)?;
Ok((input, Token::Hashtag(hashtag_text.into())))
}
}
#[inline]
fn hashtag_chars(input: Span) -> IResult<Span, Span> {
recognize(alt((
recognize(tuple((tag("("), hashtag_chars, tag(")")))),
recognize(tuple((tag("["), hashtag_chars, tag("]")))),
recognize(tuple((tag(""), hashtag_chars, tag("")))),
recognize(tuple((tag(""), hashtag_chars, tag("")))),
recognize(tuple((
not(space1),
not_line_ending,
not(one_of(".,:;!?#?/[]【】()「」()<>")),
anychar,
))),
)))(input)
2023-10-05 17:09:26 +00:00
}
#[inline]
fn protocol(input: Span) -> IResult<Span, Span> {
alt((tag("https://"), tag("http://")))(input)
}
#[inline]
fn url_chars_base(input: Span) -> IResult<Span, Span> {
2023-10-07 19:22:21 +00:00
recognize(alt((
alpha1,
recognize(tuple((tag("["), many_till(url_chars_base, tag("]"))))),
recognize(tuple((tag("("), many_till(url_chars_base, tag(")"))))),
recognize(one_of(".,_/:%#$&?!~=+-@")),
)))(input)
2023-10-05 17:09:26 +00:00
}
#[inline]
fn url_chars<'a, T: 'a>(
terminator: impl Fn(Span<'a>) -> IResult<Span<'a>, T> + 'a,
spaces: bool,
) -> impl FnMut(Span<'a>) -> IResult<Span<'a>, Span<'a>> + 'a {
let chars = tuple((
not(tuple((space1, eof))),
not(tuple((space1, tag("\"")))),
2023-10-07 19:22:21 +00:00
not(tuple((opt(space1), terminator))),
2023-10-05 17:09:26 +00:00
alt((url_chars_base, if spaces { space1 } else { fail })),
));
recognize(many1_count(chars))
}
#[cfg(test)]
mod test {
2023-10-05 22:17:52 +00:00
use crate::{url_chars, Context, Span, Token};
2023-10-05 17:09:26 +00:00
use nom::bytes::complete::tag;
2023-10-05 22:17:52 +00:00
use std::borrow::Cow;
use std::collections::HashMap;
2023-10-05 17:09:26 +00:00
fn parse_full(string: &str) -> Token {
Context.full(Span::new(string)).unwrap().1.merged().owned()
}
2023-10-05 17:09:26 +00:00
#[test]
fn parse_url_chars() {
assert_eq!(
2023-10-07 19:22:21 +00:00
url_chars(tag(")"), true)(Span::new(
"https://en.wikipedia.org/wiki/Sandbox_(computer_security))"
))
.unwrap()
.1
.into_fragment(),
"https://en.wikipedia.org/wiki/Sandbox_(computer_security)"
2023-10-05 17:09:26 +00:00
);
assert_eq!(
2023-10-07 19:22:21 +00:00
url_chars(tag(")"), true)(Span::new(
"https://en.wikipedia.org/wiki/Sandbox_(computer_security)))"
))
.unwrap()
.1
.into_fragment(),
"https://en.wikipedia.org/wiki/Sandbox_(computer_security)",
2023-10-05 17:09:26 +00:00
);
assert_eq!(
2023-10-07 19:22:21 +00:00
url_chars(tag(")"), true)(Span::new("https://cs.wikipedia.org/wiki/Among_Us "))
2023-10-05 17:09:26 +00:00
.unwrap()
.1
2023-10-07 19:22:21 +00:00
.into_fragment(),
"https://cs.wikipedia.org/wiki/Among_Us",
2023-10-05 17:09:26 +00:00
);
assert_eq!(
2023-10-07 19:22:21 +00:00
url_chars(tag(")"), true)(Span::new("https://cs.wikipedia.org/wiki/Among Us )"))
2023-10-05 17:09:26 +00:00
.unwrap()
.1
2023-10-07 19:22:21 +00:00
.into_fragment(),
"https://cs.wikipedia.org/wiki/Among Us"
2023-10-05 17:09:26 +00:00
);
assert_eq!(
2023-10-07 19:22:21 +00:00
url_chars(tag(")"), false)(Span::new("https://en.wikipedia.org/wiki/Among Us )"))
2023-10-05 17:09:26 +00:00
.unwrap()
.1
2023-10-07 19:22:21 +00:00
.into_fragment(),
"https://en.wikipedia.org/wiki/Among"
2023-10-05 17:09:26 +00:00
);
}
2023-10-05 19:21:23 +00:00
#[test]
fn parse_formatting() {
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"~~stikethrough~~"#),
Token::Strikethrough(Box::new(Token::PlainText("stikethrough".into()))),
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"**bold**"#),
Token::Bold(Box::new(Token::PlainText("bold".into()))),
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"*italic*"#),
Token::Italic(Box::new(Token::PlainText("italic".into()))),
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"not code `code` also not code"#),
Token::Sequence(vec![
Token::PlainText("not code ".into()),
Token::InlineCode("code".into()),
Token::PlainText(" also not code".into())
]),
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"not code `code` also `not code"#),
Token::Sequence(vec![
Token::PlainText("not code ".into()),
Token::InlineCode("code".into()),
Token::PlainText(" also `not code".into())
]),
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"not code `*not bold*` also not code"#),
Token::Sequence(vec![
Token::PlainText("not code ".into()),
Token::InlineCode("*not bold*".into()),
Token::PlainText(" also not code".into())
]),
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"***bold italic***"#),
Token::BoldItalic(Box::new(Token::PlainText("bold italic".into())))
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"<b><i>bold italic</i></b>"#),
Token::Bold(Box::new(Token::Italic(Box::new(Token::PlainText(
"bold italic".into()
2023-10-07 19:26:25 +00:00
)))))
);
}
2023-10-05 22:17:52 +00:00
#[test]
fn parse_complex() {
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(
r#"<center>centered
🦋🏳
text</center>"#
),
Token::Center(Box::new(Token::Sequence(vec![
Token::PlainText("centered\n".into()),
Token::UnicodeEmoji("🦋".into()),
Token::UnicodeEmoji("🏳️‍⚧️".into()),
Token::PlainText("\ntext".into())
2023-10-07 19:26:25 +00:00
])))
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(
r#"> <center>centered
> 👩🏽🤝👩🏼
> text</center>"#
),
Token::Quote(Box::new(Token::Center(Box::new(Token::Sequence(vec![
Token::PlainText("centered\n".into()),
Token::UnicodeEmoji("👩🏽‍🤝‍👩🏼".into()),
Token::PlainText("\ntext".into())
]))))),
);
2023-10-05 22:17:52 +00:00
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"$[x2 $[sparkle 🥺]💜$[spin.y,speed=5s ❤️]🦊]"#),
2023-10-05 22:17:52 +00:00
Token::Function {
name: "x2".into(),
params: HashMap::new(),
inner: Box::new(Token::Sequence(vec![
Token::Function {
name: "sparkle".into(),
params: HashMap::new(),
inner: Box::new(Token::UnicodeEmoji("🥺".into())),
},
Token::UnicodeEmoji("💜".into()),
Token::Function {
name: "spin".into(),
params: {
let mut params = HashMap::new();
params.insert("y".into(), None);
params.insert("speed".into(), Some("5s".into()));
params
},
inner: Box::new(Token::UnicodeEmoji("❤️".into())),
},
Token::UnicodeEmoji("🦊".into()),
]))
},
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full(r#"<b>bold @tag1 <i> @tag2 </b>italic</i>"#),
Token::Sequence(vec![
Token::PlainText("<b>bold ".into()),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag1".into(),
host: None
},
Token::PlainText(" <i> ".into()),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag2".into(),
host: None
},
Token::PlainText(" </b>italic</i>".into())
]),
);
assert_eq!(
parse_full(
r#"
> test
> <i>
> italic
> </i>
>> Nested quote
"#
2023-10-07 19:26:25 +00:00
),
Token::Quote(Box::new(Token::Sequence(vec![
Token::PlainText("test\n".into()),
Token::Italic(Box::new(Token::PlainText("\nitalic\n".into()))),
Token::Quote(Box::new(Token::PlainText("Nested quote".into())))
]))),
);
2023-10-05 22:17:52 +00:00
}
2023-10-07 18:40:01 +00:00
#[test]
fn parse_link() {
assert_eq!(
parse_full("Link test: [label](https://example.com)"),
Token::Sequence(vec![
Token::PlainText("Link test: ".into()),
Token::Link {
label: Box::new(Token::PlainText("label".into())),
href: "https://example.com".into(),
embed: true
}
])
);
2023-10-07 19:22:21 +00:00
assert_eq!(
parse_full("<https://example.com>"),
Token::UrlNoEmbed("https://example.com".into())
);
// Adjacent links okay
assert_eq!(
parse_full("<https://example.com/><https://awawa.gay/>"),
Token::Sequence(vec![
Token::UrlNoEmbed("https://example.com/".into()),
Token::UrlNoEmbed("https://awawa.gay/".into())
])
);
2023-10-07 18:40:01 +00:00
assert_eq!(
parse_full("Link test: ?[label](https://awawa.gay)"),
Token::Sequence(vec![
Token::PlainText("Link test: ".into()),
Token::Link {
label: Box::new(Token::PlainText("label".into())),
href: "https://awawa.gay".into(),
embed: false
}
])
);
2023-10-07 19:22:21 +00:00
assert_eq!(
parse_full("Link test: ?[label](https://awawa.gay)test"),
Token::Sequence(vec![
Token::PlainText("Link test: ".into()),
Token::Link {
label: Box::new(Token::PlainText("label".into())),
href: "https://awawa.gay".into(),
embed: false
},
Token::PlainText("test".into())
])
);
assert_eq!(
parse_full("Link test: (?[label](https://awawa.gay))"),
Token::Sequence(vec![
Token::PlainText("Link test: (".into()),
Token::Link {
label: Box::new(Token::PlainText("label".into())),
href: "https://awawa.gay".into(),
embed: false
},
Token::PlainText(")".into())
])
);
2023-10-07 18:40:01 +00:00
assert_eq!(
parse_full("Link test: ?[label](https://awawa.gay"), // Missing closing bracket
Token::Sequence(vec![
Token::PlainText("Link test: ?[label](".into()),
Token::UrlRaw("https://awawa.gay".into()),
])
);
}
#[test]
fn parse_mention() {
assert_eq!(
parse_full("@tag"),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag".into(),
host: None
}
);
assert_eq!(
parse_full("hgsjlkdsa @tag fgahjsdkd"),
Token::Sequence(vec![
Token::PlainText("hgsjlkdsa ".into()),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag".into(),
host: None
},
Token::PlainText(" fgahjsdkd".into())
])
);
assert_eq!(
parse_full("hgsjlkdsa @tag@ fgahjsdkd"),
Token::Sequence(vec![
Token::PlainText("hgsjlkdsa ".into()),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag".into(),
host: None
},
Token::PlainText("@ fgahjsdkd".into())
])
);
assert_eq!(
parse_full("aaaa @tag@domain bbbbb"),
Token::Sequence(vec![
Token::PlainText("aaaa ".into()),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag".into(),
host: Some("domain".into())
},
Token::PlainText(" bbbbb".into())
])
);
assert_eq!(
parse_full("test @tag@domain, test"),
Token::Sequence(vec![
Token::PlainText("test ".into()),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag".into(),
host: Some("domain".into())
},
Token::PlainText(", test".into())
])
);
assert_eq!(
parse_full("test @tag@domain.gay. test"),
Token::Sequence(vec![
Token::PlainText("test ".into()),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag".into(),
host: Some("domain.gay".into())
},
Token::PlainText(". test".into())
])
);
assert_eq!(
parse_full("test @tag@domain? test"),
Token::Sequence(vec![
Token::PlainText("test ".into()),
Token::Mention {
mention_type: crate::MentionType::User,
name: "tag".into(),
host: Some("domain".into())
},
Token::PlainText("? test".into())
])
);
assert_eq!(
parse_full("test !tag@domain.com test"),
Token::Sequence(vec![
Token::PlainText("test ".into()),
Token::Mention {
mention_type: crate::MentionType::Community,
name: "tag".into(),
host: Some("domain.com".into())
},
Token::PlainText(" test".into())
])
);
}
2023-10-05 19:21:23 +00:00
#[test]
fn parse_emoji() {
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full("🥺💜❤️🦊"),
Token::Sequence(
vec!["🥺", "💜", "❤️", "🦊"]
.into_iter()
.map(<&str as Into<Cow<_>>>::into)
.map(Token::UnicodeEmoji)
.collect::<Vec<_>>()
2023-10-07 19:26:25 +00:00
)
);
2023-10-05 19:21:23 +00:00
// Trans flag, ZWJ
2023-10-05 22:17:52 +00:00
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full("\u{1f3f3}\u{0fe0f}\u{0200d}\u{026a7}\u{0fe0f}"),
Token::UnicodeEmoji("\u{1f3f3}\u{0fe0f}\u{0200d}\u{026a7}\u{0fe0f}".into())
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full("\u{0200d}\u{1f3f3}\u{0fe0f}"),
Token::Sequence(vec![
Token::PlainText("\u{0200d}".into()), // ZWJ
Token::UnicodeEmoji("\u{1f3f3}\u{0fe0f}".into()), // White flag
2023-10-07 19:26:25 +00:00
])
);
// Trans flag, ZWNJ
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full("\u{1f3f3}\u{0fe0f}\u{0200c}\u{026a7}\u{0fe0f}"),
Token::Sequence(vec![
Token::UnicodeEmoji("\u{1f3f3}\u{0fe0f}".into()), // White flag
Token::PlainText("\u{0200c}".into()), // ZWNJ
Token::UnicodeEmoji("\u{026a7}\u{0fe0f}".into()) // Trans symbol
2023-10-07 19:26:25 +00:00
])
);
assert_eq!(
2023-10-07 19:26:25 +00:00
parse_full("\u{1f3f3}\u{0fe0f}\u{0200d}\u{0200d}\u{0200d}"),
Token::Sequence(vec![
Token::UnicodeEmoji("\u{1f3f3}\u{0fe0f}".into()), // White flag
Token::PlainText("\u{0200d}\u{0200d}\u{0200d}".into()), // ZWJ
2023-10-07 19:26:25 +00:00
])
2023-10-05 22:17:52 +00:00
);
2023-10-05 19:21:23 +00:00
}
2023-10-01 21:04:32 +00:00
}