diff --git a/development.md b/development.md index 2810a48..bcd343c 100644 --- a/development.md +++ b/development.md @@ -30,6 +30,7 @@ curl -q https://orgmode.org/worg/org-faq.org --output ./benches/org-faq.org curl -q https://orgmode.org/worg/org-hacks.org --output ./benches/org-hacks.org curl -q https://orgmode.org/worg/org-release-notes.org --output ./benches/org-release-notes.org curl -q https://orgmode.org/worg/org-syntax.org --output ./benches/org-syntax.org +curl -q https://raw.githubusercontent.com/bzg/org-mode/main/doc/org-manual.org --output ./benches/org-manual.org cargo bench --bench parse ``` diff --git a/examples/html-slugify.rs b/examples/html-slugify.rs index f930830..996a263 100644 --- a/examples/html-slugify.rs +++ b/examples/html-slugify.rs @@ -9,7 +9,6 @@ use orgize::{ rowan::WalkEvent, Org, }; -use rowan::NodeOrToken; use slugify::slugify; use std::cmp::min; use std::env::args; @@ -34,10 +33,7 @@ impl Traverser for MyHtmlHandler { slugify!(&title) )); for elem in headline.title() { - match elem { - NodeOrToken::Node(node) => self.node(node, ctx), - NodeOrToken::Token(token) => self.token(token, ctx), - } + self.element(elem, ctx); } self.0.push_str(format!("")); } @@ -50,7 +46,7 @@ impl Traverser for MyHtmlHandler { special_block quote_block center_block verse_block comment_block example_block export_block source_block babel_call clock cookie radio_target drawer dyn_block fn_def fn_ref macros snippet timestamp target fixed_width org_table org_table_row org_table_cell latex_fragment - latex_environment entity line_break superscript subscript + latex_environment entity line_break superscript subscript keyword property_drawer } } diff --git a/src/export/forward.rs b/src/export/forward.rs index dd3f7ab..2bd742a 100644 --- a/src/export/forward.rs +++ b/src/export/forward.rs @@ -7,7 +7,7 @@ /// ast::Headline, /// export::{HtmlExport, TraversalContext, Traverser}, /// forward_handler, -/// rowan::{ast::AstNode, WalkEvent, NodeOrToken}, +/// rowan::{ast::AstNode, WalkEvent}, /// Org, /// }; /// use slugify::slugify; @@ -33,10 +33,7 @@ /// slugify!(&title) /// )); /// for elem in headline.title() { -/// match elem { -/// NodeOrToken::Node(node) => self.node(node, ctx), -/// NodeOrToken::Token(token) => self.token(token, ctx), -/// } +/// self.element(elem, ctx); /// } /// self.0.push_str(format!("")); /// } @@ -49,7 +46,7 @@ /// special_block quote_block center_block verse_block comment_block example_block export_block /// source_block babel_call clock cookie radio_target drawer dyn_block fn_def fn_ref macros /// snippet timestamp target fixed_width org_table org_table_row org_table_cell latex_fragment -/// latex_environment entity line_break superscript subscript +/// latex_environment entity line_break superscript subscript keyword property_drawer /// } /// } /// @@ -209,6 +206,12 @@ macro_rules! forward_handler { (@method $handler:ty, subscript) => { forward_handler!(@method $handler, subscript, WalkEvent<&$crate::ast::Subscript>); }; + (@method $handler:ty, keyword) => { + forward_handler!(@method $handler, keyword, WalkEvent<&$crate::ast::Keyword>); + }; + (@method $handler:ty, property_drawer) => { + forward_handler!(@method $handler, property_drawer, WalkEvent<&$crate::ast::PropertyDrawer>); + }; (@method $handler:ty, $x:ident) => { std::compile_error!(std::concat!(std::stringify!($x), " is not a method")); }; diff --git a/src/export/html.rs b/src/export/html.rs index 3970819..5a44b70 100644 --- a/src/export/html.rs +++ b/src/export/html.rs @@ -125,10 +125,7 @@ impl Traverser for HtmlExport { WalkEvent::Enter(item) => { self.output += "
"; for elem in item.tag() { - match elem { - NodeOrToken::Node(n) => self.node(n, ctx), - NodeOrToken::Token(t) => self.token(t, ctx), - } + self.element(elem, ctx); } self.output += "
"; } @@ -379,10 +376,7 @@ impl Traverser for HtmlExport { let level = min(headline.level(), 6); let _ = write!(&mut self.output, ""); for elem in headline.title() { - match elem { - NodeOrToken::Node(node) => self.node(node, ctx), - NodeOrToken::Token(token) => self.token(token, ctx), - } + self.element(elem, ctx); } let _ = write!(&mut self.output, ""); } @@ -534,4 +528,14 @@ impl Traverser for HtmlExport { WalkEvent::Leave(_) => self.output += "", } } + + #[tracing::instrument(skip(self, ctx))] + fn keyword(&mut self, _event: WalkEvent<&Keyword>, ctx: &mut TraversalContext) { + ctx.skip(); + } + + #[tracing::instrument(skip(self, ctx))] + fn property_drawer(&mut self, _event: WalkEvent<&PropertyDrawer>, ctx: &mut TraversalContext) { + ctx.skip() + } } diff --git a/src/export/traverse.rs b/src/export/traverse.rs index 2413751..4c1464a 100644 --- a/src/export/traverse.rs +++ b/src/export/traverse.rs @@ -143,10 +143,16 @@ pub trait Traverser { LINE_BREAK => traverse!(LineBreak, line_break), SUPERSCRIPT => traverse!(Superscript, superscript), SUBSCRIPT => traverse!(Subscript, subscript), + KEYWORD => traverse!(Keyword, keyword), + PROPERTY_DRAWER => traverse!(PropertyDrawer, property_drawer), BLOCK_CONTENT | LIST_ITEM_CONTENT => traverse_children!(node), - kind => debug_assert!(!kind.is_element() && !kind.is_object()), + kind => debug_assert!( + !kind.is_element() && !kind.is_object(), + "{:?} is not handled", + kind + ), } } @@ -258,4 +264,8 @@ pub trait Traverser { fn superscript(&mut self, event: WalkEvent<&Superscript>, ctx: &mut TraversalContext); /// Called when entering or leaving `Subscript` node fn subscript(&mut self, event: WalkEvent<&Subscript>, ctx: &mut TraversalContext); + /// Called when entering or leaving `Keyword` node + fn keyword(&mut self, event: WalkEvent<&Keyword>, ctx: &mut TraversalContext); + /// Called when entering or leaving `PropertyDrawer` node + fn property_drawer(&mut self, event: WalkEvent<&PropertyDrawer>, ctx: &mut TraversalContext); } diff --git a/src/syntax/headline.rs b/src/syntax/headline.rs index 118a1b5..d4ff787 100644 --- a/src/syntax/headline.rs +++ b/src/syntax/headline.rs @@ -133,11 +133,8 @@ fn headline_stars(input: Input) -> IResult { if level == 0 { return Err(nom::Err::Error(())); } - // followed by eof, new line, or whitespace - else if matches!( - bytes.get(level), - None | Some(b'\n') | Some(b'\r') | Some(b' ') - ) { + // headline stars must be followed by space + else if matches!(bytes.get(level), Some(b' ')) { Ok(input.take_split(level)) } else { Err(nom::Err::Error(())) @@ -232,7 +229,7 @@ fn headline_priority_node(input: Input) -> IResult(headline_node); @@ -307,6 +304,16 @@ fn parse() { NEW_LINE@11..12 "\n" "### ); + + let config = &ParseConfig::default(); + + assert!(headline_node(("_ ", config).into()).is_err()); + assert!(headline_node(("*", config).into()).is_err()); + assert!(headline_node((" * ", config).into()).is_err()); + assert!(headline_node(("**", config).into()).is_err()); + assert!(headline_node(("**\n", config).into()).is_err()); + assert!(headline_node(("**\r", config).into()).is_err()); + assert!(headline_node(("**\t", config).into()).is_err()); } #[test] diff --git a/src/syntax/list.rs b/src/syntax/list.rs index c12d473..dd9716b 100644 --- a/src/syntax/list.rs +++ b/src/syntax/list.rs @@ -2,7 +2,7 @@ use memchr::{memchr, memchr2}; use nom::{ branch::alt, bytes::complete::{tag, take}, - character::complete::{alphanumeric1, digit1, space0}, + character::complete::{alphanumeric1, digit1, space0, space1}, combinator::{cond, map, opt, recognize, verify}, sequence::{preceded, tuple}, AsBytes, IResult, InputLength, InputTake, @@ -10,7 +10,7 @@ use nom::{ use super::{ combinator::{ - at_token, blank_lines, colon2_token, l_bracket_token, line_starts_iter, node, + at_token, blank_lines, colon2_token, eol_or_eof, l_bracket_token, line_starts_iter, node, r_bracket_token, GreenElement, }, element::element_node, @@ -82,25 +82,23 @@ fn list_item_node<'a>( preceded(digit1, tag(".")), preceded(digit1, tag(")")), )), - space0, + alt((space1, eol_or_eof)), )))(input)?; - // bullet must ends with whitespace, - if !(bullet - .s - .bytes() - .last() - .map(|b| b == b' ' || b == b'\t') - .unwrap_or(true) - // or input should be a line end - || input - .s - .bytes() - .next() - .map(|b| b == b'\r' || b == b'\n') - .unwrap_or(true)) - { - return Err(nom::Err::Error(())); + if input.is_empty() { + return Ok(( + input, + ( + false, + node( + LIST_ITEM, + [ + indent.token(LIST_ITEM_INDENT), + bullet.token(LIST_ITEM_BULLET), + ], + ), + ), + )); } let is_ordered = bullet.s.starts_with(|c: char| c.is_ascii_digit()); @@ -289,8 +287,6 @@ fn parse() { LIST_ITEM@0..2 LIST_ITEM_INDENT@0..0 "" LIST_ITEM_BULLET@0..2 "1)" - LIST_ITEM_CONTENT@2..2 - PARAGRAPH@2..2 "### ); @@ -301,8 +297,6 @@ fn parse() { LIST_ITEM@0..2 LIST_ITEM_INDENT@0..0 "" LIST_ITEM_BULLET@0..2 "+ " - LIST_ITEM_CONTENT@2..2 - PARAGRAPH@2..2 "### ); @@ -312,10 +306,7 @@ fn parse() { LIST@0..2 LIST_ITEM@0..2 LIST_ITEM_INDENT@0..0 "" - LIST_ITEM_BULLET@0..1 "-" - LIST_ITEM_CONTENT@1..2 - PARAGRAPH@1..2 - BLANK_LINE@1..2 "\n" + LIST_ITEM_BULLET@0..2 "-\n" "### ); @@ -594,6 +585,13 @@ fn parse() { "### ); + to_list("- "); + to_list("-\t"); + to_list("-\r"); + to_list("-\t\n"); + to_list("-\r\n"); + to_list("-"); + let config = &ParseConfig::default(); assert!(list_node(("-a", config).into()).is_err());