Compare commits

...

3 Commits

Author SHA1 Message Date
Natty 6b96f60b43
MMM: Fixed nesting counting in increase_nesting 2023-12-31 17:18:27 +01:00
Natty ce8cb5aad0
MMM: More forgiving math parsing 2023-12-30 20:02:25 +01:00
Natty 9c8489ced7
MMM: Correctly parse empty strings
Surely I wouldn't forget to do that, right?
2023-12-30 19:23:32 +01:00
1 changed files with 51 additions and 23 deletions

View File

@ -718,40 +718,54 @@ impl Context {
}
pub fn full<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
map(many1(self.partial(Self::full_single)), Token::Sequence)(input)
map(
many_till(self.partial(Self::full_single), eof).map(|v| v.0),
Token::Sequence,
)(input)
}
pub fn inline<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
map(many1(self.partial(Self::inline_single)), Token::Sequence)(input)
map(
many_till(self.partial(Self::inline_single), eof).map(|v| v.0),
Token::Sequence,
)(input)
}
pub fn inline_label_safe<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
map(
many1(self.partial(Self::inline_label_safe_single)),
many_till(self.partial(Self::inline_label_safe_single), eof).map(|v| v.0),
Token::Sequence,
)(input)
}
fn inline_profile_fields<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
map(
many1(alt((
self.partial(Self::unicode_emoji),
self.partial(Self::tag_mention),
self.partial(Self::tag_hashtag),
self.partial(Self::raw_url),
self.partial(Self::tag_raw_text),
))),
many_till(
alt((
self.partial(Self::unicode_emoji),
self.partial(Self::tag_mention),
self.partial(Self::tag_hashtag),
self.partial(Self::raw_url),
self.partial(Self::tag_raw_text),
)),
eof,
)
.map(|v| v.0),
Token::Sequence,
)(input)
}
fn inline_ui<'a>(&self, input: Span<'a>) -> IResult<Span<'a>, Token> {
map(
many1(alt((
self.partial(Self::unicode_emoji),
self.partial(Self::shortcode_emoji),
self.partial(Self::tag_raw_text),
))),
many_till(
alt((
self.partial(Self::unicode_emoji),
self.partial(Self::shortcode_emoji),
self.partial(Self::tag_raw_text),
)),
eof,
)
.map(|v| v.0),
Token::Sequence,
)(input)
}
@ -972,14 +986,13 @@ impl Context {
let (input, _) = start(input)?;
let (input, _) = opt(line_ending)(input)?;
let (input, _) = opt(space1_unicode)(input)?;
let (input, math_span) = recognize(many1_count(tuple((
not(tuple((opt(line_ending), end))),
not_line_ending,
))))(input)?;
let (input, math_span) = map(
many_till(anychar, tuple((opt(space1_unicode), opt(line_ending), end))),
|v| v.0,
)(input)?;
let (input, _) = opt(line_ending)(input)?;
let (input, _) = end(input)?;
// Trailing whitespace after the closing delim
let (input, _) = opt(space1_unicode)(input)?;
// If we got this far, the next character should be a line ending
@ -988,7 +1001,7 @@ impl Context {
Ok((
input,
Token::BlockMath(math_span.into_fragment().to_string()),
Token::BlockMath(math_span.into_iter().collect::<String>()),
))
}
@ -1609,7 +1622,10 @@ impl Context {
}
input.extra.depth += 1;
func.parse(input)
func.parse(input).map(|mut v| {
v.0.extra.depth -= 1;
v
})
}
}
@ -1709,6 +1725,13 @@ mod test {
.merged()
}
#[test]
fn parse_empty() {
let ctx = Context::default();
assert_eq!(parse_full(""), Token::Sequence(vec![]));
}
#[test]
fn parse_url_chars() {
let ctx = Context::default();
@ -1905,6 +1928,11 @@ a^2 + b^2 = c^2
Token::BlockMath("a^2 + b^2 = c^2".to_string())
);
assert_eq!(
parse_full(r"\[ x^2 + y^2 = z^2 \]"),
Token::BlockMath("x^2 + y^2 = z^2".to_string())
);
assert_eq!(
parse_full(
r#"<center>centered