feat: introduce Token struct

This commit is contained in:
PoiScript 2023-11-20 14:56:38 +08:00
parent 9004de9930
commit 6c27a9257f
No known key found for this signature in database
GPG key ID: 22C2B1249D99985E
23 changed files with 586 additions and 262 deletions

View file

@ -2,7 +2,7 @@ use nom::{
branch::alt,
bytes::complete::tag,
character::complete::{digit1, line_ending, space0},
combinator::{eof, map, opt},
combinator::{eof, map, opt, recognize},
sequence::tuple,
IResult,
};
@ -26,9 +26,7 @@ pub fn clock_node(input: Input) -> IResult<Input, GreenElement, ()> {
space0,
double_arrow_token,
space0,
digit1,
colon_token,
digit1,
recognize(tuple((digit1, colon_token, digit1))),
))),
space0,
alt((line_ending, eof)),
@ -41,13 +39,11 @@ pub fn clock_node(input: Input) -> IResult<Input, GreenElement, ()> {
b.text(clock);
b.ws(ws_);
b.push(timestamp);
if let Some((ws, double_arrow, ws_, hour, colon, minute)) = duration {
if let Some((ws, double_arrow, ws_, time)) = duration {
b.ws(ws);
b.push(double_arrow);
b.ws(ws_);
b.text(hour);
b.push(colon);
b.text(minute);
b.text(time);
}
b.ws(ws__);
b.nl(nl);
@ -125,9 +121,7 @@ fn parse() {
WHITESPACE@53..54 " "
DOUBLE_ARROW@54..56 "=>"
WHITESPACE@56..58 " "
TEXT@58..59 "1"
COLON@59..60 ":"
TEXT@60..62 "00"
TEXT@58..62 "1:00"
NEW_LINE@62..63 "\n"
BLANK_LINE@63..64 "\n"
"###

View file

@ -141,8 +141,7 @@ impl<'a> Iterator for ElementPositions<'a> {
let previous = self.pos;
self.pos = iter
.next()
.map(|i| i + self.pos)
.unwrap_or_else(|| self.input.s.len());
.map_or_else(|| self.input.s.len(), |i| i + self.pos);
debug_assert!(
previous < self.pos && self.pos <= self.input.s.len(),

View file

@ -48,7 +48,6 @@ fn parse() {
let to_link = to_ast::<Link>(link_node);
let link = to_link("[[#id]]");
assert_eq!(link.path().as_ref().map(|x| x.text()), Some("#id"));
insta::assert_debug_snapshot!(
link.syntax,
@r###"

View file

@ -30,10 +30,7 @@ pub fn macros_node(input: Input) -> IResult<Input, GreenElement, ()> {
children.push(l_curly3);
children.push(name.text_token());
if let Some((l_parens, argument, r_parens)) = argument {
children.push(node(
MACROS_ARGUMENT,
[l_parens, argument.text_token(), r_parens],
));
children.extend([l_parens, argument.text_token(), r_parens]);
}
children.push(r_curly3);
node(MACROS, children)
@ -64,10 +61,9 @@ fn test() {
MACROS@0..22
L_CURLY3@0..3 "{{{"
TEXT@3..16 "one_arg_macro"
MACROS_ARGUMENT@16..19
L_PARENS@16..17 "("
TEXT@17..18 "1"
R_PARENS@18..19 ")"
L_PARENS@16..17 "("
TEXT@17..18 "1"
R_PARENS@18..19 ")"
R_CURLY3@19..22 "}}}"
"###
);
@ -78,10 +74,9 @@ fn test() {
MACROS@0..25
L_CURLY3@0..3 "{{{"
TEXT@3..16 "two_arg_macro"
MACROS_ARGUMENT@16..22
L_PARENS@16..17 "("
TEXT@17..21 "1, 2"
R_PARENS@21..22 ")"
L_PARENS@16..17 "("
TEXT@17..21 "1, 2"
R_PARENS@21..22 ")"
R_CURLY3@22..25 "}}}"
"###
);
@ -92,10 +87,9 @@ fn test() {
MACROS@0..28
L_CURLY3@0..3 "{{{"
TEXT@3..16 "two_arg_macro"
MACROS_ARGUMENT@16..25
L_PARENS@16..17 "("
TEXT@17..24 "1\\,a, 2"
R_PARENS@24..25 ")"
L_PARENS@16..17 "("
TEXT@17..24 "1\\,a, 2"
R_PARENS@24..25 ")"
R_CURLY3@25..28 "}}}"
"###
);

View file

@ -189,7 +189,6 @@ pub enum SyntaxKind {
FN_REF,
LATEX_FRAGMENT,
MACROS,
MACROS_ARGUMENT,
SNIPPET,
TARGET,
BOLD,