chore: add orgize-{cli,common,lsp} package

orgize-common/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+name = "orgize-common"
+version.workspace = true
+authors.workspace = true
+repository.workspace = true
+edition.workspace = true
+license.workspace = true
+description = "Shared code between orgize-lsp and orgize-lsp."
+
+[dependencies]
+anyhow = "1.0.75"
+jetscii = "0.5.3"
+nom = "7.1.3"
+orgize = { path = "../orgize" }
+resolve-path = "0.1.0"
+tracing = "0.1.40"

orgize-common/src/detangle.rs

@@ -0,0 +1,110 @@
+use orgize::{
+    ast::{Headline, SourceBlock},
+    rowan::ast::AstNode,
+    SyntaxKind,
+};
+use resolve_path::PathResolveExt;
+use std::{fs, path::PathBuf};
+
+use crate::{
+    header_argument::{header_argument, property_drawer, property_keyword},
+    utils::language_comments,
+};
+
+pub fn detangle(
+    block: SourceBlock,
+    file_path: &PathBuf,
+) -> anyhow::Result<Option<(usize, usize, String)>> {
+    let arg1 = block.parameters().unwrap_or_default();
+    let arg2 = property_drawer(block.syntax()).unwrap_or_default();
+    let arg3 = property_keyword(block.syntax()).unwrap_or_default();
+    let language = block.language().unwrap_or_default();
+
+    let tangle = header_argument(&arg1, &arg2, &arg3, ":tangle", "no");
+
+    if tangle == "no" {
+        return Ok(None);
+    }
+
+    let comments = header_argument(&arg1, &arg2, &arg3, ":comments", "no");
+
+    let parent = block
+        .syntax()
+        .ancestors()
+        .find(|n| n.kind() == SyntaxKind::HEADLINE || n.kind() == SyntaxKind::DOCUMENT);
+
+    let nth = parent
+        .as_ref()
+        .and_then(|n| n.children().position(|c| &c == block.syntax()))
+        .unwrap_or(1);
+
+    let headline_title = parent.and_then(Headline::cast).map(|headline| {
+        headline
+            .title()
+            .fold(String::new(), |a, n| a + &n.to_string())
+    });
+
+    let dest_path = tangle.try_resolve_in(file_path)?.to_path_buf();
+
+    let content = fs::read_to_string(dest_path)?;
+
+    let Some(begin) = block
+        .syntax()
+        .children()
+        .find(|n| n.kind() == SyntaxKind::BLOCK_CONTENT)
+    else {
+        return Ok(None);
+    };
+
+    let text_range = begin.text_range();
+
+    if comments == "yes" || comments == "link" || comments == "noweb" || comments == "both" {
+        let begin_comments = format!(
+            "[[file:{path}::*{title}][{title}:{nth}]]",
+            title = headline_title.as_deref().unwrap_or("No heading"),
+            path = file_path.to_string_lossy(),
+        );
+        let end_comments = format!(
+            "{title}:{nth} ends here",
+            title = headline_title.as_deref().unwrap_or("No heading"),
+        );
+
+        let mut block_content = String::new();
+
+        for line in content
+            .lines()
+            .skip_while(|line| trim_comments(line, &language).unwrap_or_default() != begin_comments)
+            .skip(1)
+        {
+            if trim_comments(line, &language).unwrap_or_default() == end_comments {
+                return Ok(Some((
+                    text_range.start().into(),
+                    text_range.end().into(),
+                    block_content,
+                )));
+            } else {
+                block_content += line;
+                block_content += "\n";
+            }
+        }
+
+        tracing::warn!(
+            "Cannot found contents wrapped by comments for code block {path}*{title}:{nth}.",
+            title = headline_title.as_deref().unwrap_or("No heading"),
+            path = file_path.to_string_lossy(),
+        );
+
+        return Ok(None);
+    }
+
+    Ok(Some((
+        text_range.start().into(),
+        text_range.end().into(),
+        content,
+    )))
+}
+
+fn trim_comments<'a>(input: &'a str, language: &str) -> Option<&'a str> {
+    let (begin, end) = language_comments(language)?;
+    Some(input.trim().strip_prefix(begin)?.strip_suffix(end)?.trim())
+}

orgize-common/src/execute_src_block.rs

@@ -0,0 +1,142 @@
+use orgize::{
+    ast::{AffiliatedKeyword, SourceBlock},
+    rowan::ast::AstNode,
+    SyntaxKind,
+};
+use std::{fs::File, io::Write, iter::once, path::Path, process};
+
+use crate::{
+    header_argument::{header_argument, property_drawer, property_keyword},
+    utils::language_execute_command,
+};
+
+#[derive(Debug)]
+enum Format {
+    Code,
+    List,
+    Verbatim,
+    Html,
+    Latex,
+    Raw,
+}
+
+pub fn execute(block: SourceBlock, path: &Path) -> anyhow::Result<Option<(usize, usize, String)>> {
+    let arg1 = block.parameters().unwrap_or_default();
+    let arg2 = property_drawer(block.syntax()).unwrap_or_default();
+    let arg3 = property_keyword(block.syntax()).unwrap_or_default();
+    let language = block.language().unwrap_or_default();
+    let results = header_argument(&arg1, &arg2, &arg3, ":results", "no");
+
+    if results == "no" {
+        return Ok(None);
+    }
+
+    let Some(command) = language_execute_command(&language) else {
+        anyhow::bail!("{language:?} is not supported.")
+    };
+
+    let mut segs = results.split(&[' ', '\t']).filter(|x| !x.is_empty());
+
+    let format = match (segs.next(), segs.next()) {
+        (Some("output"), Some("code")) | (Some("code"), None) => Format::Code,
+        (Some("output"), Some("list")) | (Some("list"), None) => Format::List,
+        (Some("output"), Some("scalar"))
+        | (Some("scalar"), None)
+        | (Some("output"), Some("verbatim"))
+        | (Some("verbatim"), None) => Format::Verbatim,
+        (Some("output"), Some("html")) | (Some("html"), None) => Format::Html,
+        (Some("output"), Some("latex")) | (Some("latex"), None) => Format::Latex,
+        (Some("output"), Some("raw")) | (Some("raw"), None) => Format::Raw,
+        (Some("value"), _) => anyhow::bail!("{language:?} is not supported."),
+        _ => return Ok(None),
+    };
+
+    let results = collect_output(command, &block.value(), format, path)?;
+
+    if let Some((start, end)) = find_existing_results(&block) {
+        Ok(Some((start, end, results)))
+    } else {
+        let start = block.end() as usize;
+        Ok(Some((start, start, format!("\n#+RESULTS:\n{}\n", results))))
+    }
+}
+
+fn collect_output(
+    command: &str,
+    value: &str,
+    format: Format,
+    path: &Path,
+) -> anyhow::Result<String> {
+    let path = path.join("orgize-temporary");
+
+    let mut file = File::create(&path)?;
+
+    file.write_all(value.as_bytes())?;
+
+    let output = process::Command::new(command).arg(path).output()?;
+
+    let output = String::from_utf8_lossy(&output.stdout);
+
+    match format {
+        Format::Code => Ok(once("#+begin_src")
+            .chain(output.lines())
+            .chain(once("#+end_src"))
+            .fold(String::new(), |acc, line| acc + line + "\n")),
+
+        Format::Html => Ok(once("#+begin_export html")
+            .chain(output.lines())
+            .chain(once("#+end_export"))
+            .fold(String::new(), |acc, line| acc + line + "\n")),
+
+        Format::Latex => Ok(once("#+begin_export latex")
+            .chain(output.lines())
+            .chain(once("#+end_export"))
+            .fold(String::new(), |acc, line| acc + line + "\n")),
+
+        Format::List => Ok(output
+            .lines()
+            .fold(String::new(), |acc, line| acc + "- " + line + "\n")),
+
+        Format::Verbatim => Ok(output
+            .lines()
+            .fold(String::new(), |acc, line| acc + ": " + line + "\n")),
+
+        Format::Raw => Ok(output.to_string()),
+    }
+}
+
+fn find_existing_results(block: &SourceBlock) -> Option<(usize, usize)> {
+    let results = block
+        .syntax()
+        .next_sibling()
+        .filter(|n| {
+            matches!(
+                n.kind(),
+                SyntaxKind::ORG_TABLE
+                    | SyntaxKind::FIXED_WIDTH
+                    | SyntaxKind::LIST
+                    | SyntaxKind::SOURCE_BLOCK
+                    | SyntaxKind::EXPORT_BLOCK
+            )
+        })
+        .filter(|n| {
+            n.children()
+                .filter_map(AffiliatedKeyword::cast)
+                .any(|k| k.key().eq_ignore_ascii_case("results"))
+        })?;
+
+    let mut iter = results
+        .children_with_tokens()
+        .skip_while(|n| n.kind() == SyntaxKind::AFFILIATED_KEYWORD)
+        .take_while(|n| n.kind() != SyntaxKind::BLANK_LINE);
+
+    let first = iter.next();
+
+    let last = iter.last();
+
+    let start = first.as_ref().map(|n| n.text_range().start())?;
+
+    let end = last.or(first).map(|x| x.text_range().end())?;
+
+    Some((start.into(), end.into()))
+}

orgize-common/src/formatting.rs

@@ -0,0 +1,133 @@
+use orgize::{
+    ast::Rule,
+    export::{Container, Event, TraversalContext, Traverser},
+    rowan::ast::AstNode,
+    Org, SyntaxKind, SyntaxNode,
+};
+
+pub fn formatting(org: &Org) -> Vec<(usize, usize, String)> {
+    let mut format = FormattingTraverser { edits: vec![] };
+
+    org.traverse(&mut format);
+
+    format.edits
+}
+
+struct FormattingTraverser {
+    edits: Vec<(usize, usize, String)>,
+}
+
+impl Traverser for FormattingTraverser {
+    fn event(&mut self, event: Event, _: &mut TraversalContext) {
+        match event {
+            Event::Rule(rule) => {
+                format_rule(&rule, &mut self.edits);
+                format_blank_lines(rule.syntax(), &mut self.edits);
+            }
+            Event::Clock(clock) => {
+                format_blank_lines(clock.syntax(), &mut self.edits);
+            }
+
+            Event::Enter(Container::Document(document)) => {
+                format_blank_lines(document.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::Paragraph(paragraph)) => {
+                format_blank_lines(paragraph.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::List(list)) => {
+                format_blank_lines(list.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::OrgTable(table)) => {
+                format_blank_lines(table.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::SpecialBlock(block)) => {
+                format_blank_lines(block.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::QuoteBlock(block)) => {
+                format_blank_lines(block.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::CenterBlock(block)) => {
+                format_blank_lines(block.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::VerseBlock(block)) => {
+                format_blank_lines(block.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::CommentBlock(block)) => {
+                format_blank_lines(block.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::ExampleBlock(block)) => {
+                format_blank_lines(block.syntax(), &mut self.edits);
+            }
+            Event::Enter(Container::ExportBlock(block)) => {
+                format_blank_lines(block.syntax(), &mut self.edits);
+            }
+
+            _ => {}
+        }
+    }
+}
+
+fn format_rule(rule: &Rule, edits: &mut Vec<(usize, usize, String)>) {
+    let node = rule.syntax();
+
+    for token in node.children_with_tokens().filter_map(|e| e.into_token()) {
+        if token.kind() == SyntaxKind::WHITESPACE && !token.text().is_empty() {
+            edits.push((
+                token.text_range().start().into(),
+                token.text_range().end().into(),
+                "".into(),
+            ));
+        }
+
+        if token.kind() == SyntaxKind::TEXT && token.text().len() != 5 {
+            edits.push((
+                token.text_range().start().into(),
+                token.text_range().end().into(),
+                "-----".into(),
+            ));
+        }
+
+        if token.kind() == SyntaxKind::NEW_LINE && token.text() != "\n" {
+            edits.push((
+                token.text_range().start().into(),
+                token.text_range().end().into(),
+                "\n".into(),
+            ));
+        }
+    }
+}
+
+fn format_blank_lines(node: &SyntaxNode, edits: &mut Vec<(usize, usize, String)>) {
+    let mut blank_lines = node
+        .children_with_tokens()
+        .filter_map(|e| e.into_token())
+        .filter(|n| n.kind() == SyntaxKind::BLANK_LINE);
+
+    if let Some(line) = blank_lines.next() {
+        if line.text() != "\n" {
+            edits.push((
+                line.text_range().start().into(),
+                line.text_range().end().into(),
+                "\n".into(),
+            ));
+        }
+    }
+
+    match (blank_lines.next(), blank_lines.last()) {
+        (Some(first), Some(last)) => {
+            edits.push((
+                first.text_range().start().into(),
+                last.text_range().end().into(),
+                "".into(),
+            ));
+        }
+        (Some(first), None) => {
+            edits.push((
+                first.text_range().start().into(),
+                first.text_range().end().into(),
+                "".into(),
+            ));
+        }
+        _ => {}
+    }
+}

orgize-common/src/header_argument.rs

@@ -0,0 +1,111 @@
+use jetscii::Substring;
+use nom::{
+    bytes::complete::take_while1,
+    character::complete::{space0, space1},
+    InputTake,
+};
+use orgize::{
+    ast::{Headline, Keyword, Token},
+    rowan::ast::AstNode,
+    SyntaxKind, SyntaxNode,
+};
+
+pub fn header_argument<'a>(
+    arg1: &'a str,
+    arg2: &'a str,
+    arg3: &'a str,
+    key: &str,
+    default: &'static str,
+) -> &'a str {
+    extract_header_args(arg1, key)
+        .or_else(|_| extract_header_args(arg2, key))
+        .or_else(|_| extract_header_args(arg3, key))
+        .unwrap_or(default)
+}
+
+pub fn property_keyword(node: &SyntaxNode) -> Option<Token> {
+    node.ancestors()
+        .find(|n| n.kind() == SyntaxKind::DOCUMENT)
+        .and_then(|n| n.first_child())
+        .filter(|n| n.kind() == SyntaxKind::SECTION)
+        .and_then(|n| {
+            n.children()
+                .filter_map(Keyword::cast)
+                .filter(|kw| kw.key().eq_ignore_ascii_case("PROPERTY"))
+                .map(|kw| kw.value())
+                .find(|value| value.trim_start().starts_with("header-args "))
+        })
+}
+
+pub fn property_drawer(node: &SyntaxNode) -> Option<Token> {
+    node.ancestors()
+        .find_map(Headline::cast)
+        .and_then(|hdl| hdl.properties())
+        .and_then(|drawer| drawer.get("header-args"))
+}
+
+pub fn extract_header_args<'a>(input: &'a str, key: &str) -> Result<&'a str, nom::Err<()>> {
+    let mut i = input;
+
+    while !i.is_empty() {
+        let (input, _) = space0(i)?;
+        let (input, name) = take_while1(|c| c != ' ' && c != '\t')(input)?;
+
+        if !name.eq_ignore_ascii_case(key) {
+            debug_assert!(input.len() < i.len(), "{} < {}", input.len(), i.len());
+            i = input;
+            continue;
+        }
+
+        let (input, _) = space1(input)?;
+
+        if let Some(idx) = Substring::new(" :")
+            .find(input)
+            .or_else(|| Substring::new("\t:").find(input))
+        {
+            let idx = input[0..idx]
+                .rfind(|c| c != ' ' && c != '\t')
+                .map(|i| i + 1)
+                .unwrap_or(idx);
+
+            let (_, value) = input.take_split(idx);
+
+            return Ok(value.trim());
+        } else {
+            return Ok(input.trim());
+        }
+    }
+
+    Err(nom::Err::Error(()))
+}
+
+#[test]
+fn parse_header_args() {
+    assert!(extract_header_args("", ":tangle").is_err());
+    assert!(extract_header_args(" :noweb yes", ":tangle1").is_err());
+    assert!(extract_header_args(":tangle", ":tangle").is_err());
+
+    assert_eq!(extract_header_args(":tangle  ", ":tangle").unwrap(), "");
+
+    assert_eq!(
+        extract_header_args(":tangle emacs.d/init.el", ":tangle").unwrap(),
+        "emacs.d/init.el"
+    );
+    assert_eq!(
+        extract_header_args(" :tangle emacs.d/init.el", ":tangle").unwrap(),
+        "emacs.d/init.el"
+    );
+    assert_eq!(
+        extract_header_args(" :tangle emacs.d/init.el  :noweb yes", ":tangle").unwrap(),
+        "emacs.d/init.el"
+    );
+    assert_eq!(
+        extract_header_args(" :noweb yes :tangle emacs.d/init.el", ":tangle").unwrap(),
+        "emacs.d/init.el"
+    );
+
+    assert_eq!(
+        extract_header_args(":results output code", ":results").unwrap(),
+        "output code"
+    );
+}

orgize-common/src/lib.rs

@@ -0,0 +1,13 @@
+mod detangle;
+mod execute_src_block;
+mod formatting;
+mod header_argument;
+mod tangle;
+mod utils;
+
+pub use detangle::detangle;
+pub use execute_src_block::execute;
+pub use formatting::formatting;
+pub use header_argument::*;
+pub use tangle::tangle;
+pub use utils::headline_slug;

orgize-common/src/tangle.rs

@@ -0,0 +1,130 @@
+// TODO: :noweb support
+
+use orgize::{
+    ast::{Headline, SourceBlock},
+    rowan::{ast::AstNode, Direction},
+    SyntaxKind,
+};
+use resolve_path::PathResolveExt;
+use std::fmt::Write;
+use std::path::PathBuf;
+
+use crate::{
+    header_argument::{header_argument, property_drawer, property_keyword},
+    utils::language_comments,
+};
+
+pub fn tangle(
+    block: SourceBlock,
+    path: &PathBuf,
+) -> anyhow::Result<Option<(PathBuf, Option<u32>, String, bool)>> {
+    let arg1 = block.parameters().unwrap_or_default();
+    let arg2 = property_drawer(block.syntax()).unwrap_or_default();
+    let arg3 = property_keyword(block.syntax()).unwrap_or_default();
+    let language = block.language().unwrap_or_default();
+
+    let tangle = header_argument(&arg1, &arg2, &arg3, ":tangle", "no");
+
+    if tangle == "no" {
+        return Ok(None);
+    }
+
+    let comments = header_argument(&arg1, &arg2, &arg3, ":comments", "no");
+    let padline = header_argument(&arg1, &arg2, &arg3, ":padline", "no");
+    let shebang = header_argument(&arg1, &arg2, &arg3, ":shebang", "no");
+    let mode = header_argument(
+        &arg1,
+        &arg2,
+        &arg3,
+        ":tangle-mode",
+        if shebang == "yea" { "o755" } else { "no" },
+    );
+    let is_mkdir = header_argument(&arg1, &arg2, &arg3, ":mkdir", "no");
+
+    let parent = block
+        .syntax()
+        .ancestors()
+        .find(|n| n.kind() == SyntaxKind::HEADLINE || n.kind() == SyntaxKind::DOCUMENT);
+
+    let nth = parent
+        .as_ref()
+        .and_then(|n| n.children().position(|c| &c == block.syntax()))
+        .unwrap_or(1);
+
+    let headline_title = parent.and_then(Headline::cast).map(|headline| {
+        headline
+            .title()
+            .fold(String::new(), |a, n| a + &n.to_string())
+    });
+
+    let path = tangle.try_resolve_in(path)?.to_path_buf();
+
+    let mut permission = None;
+    let mut content = String::new();
+
+    if mode != "no"
+        && mode.len() == 4
+        && mode.starts_with('o')
+        && mode.bytes().skip(1).all(|b| (b'0'..=b'7').contains(&b))
+    {
+        permission = u32::from_str_radix(&mode[1..], 8).ok();
+    }
+
+    if shebang != "no" && !shebang.is_empty() {
+        content.push_str(shebang);
+    }
+
+    if comments == "org" || comments == "both" {
+        if let Some((begin, end)) = language_comments(&language) {
+            let start = block
+                .syntax()
+                .siblings(Direction::Prev)
+                .skip(1) // skip self
+                .take_while(|n| n.kind() != SyntaxKind::SOURCE_BLOCK)
+                .last();
+
+            for sibling in start
+                .into_iter()
+                .flat_map(|start| start.siblings(Direction::Next))
+                .take_while(|node| node != block.syntax())
+            {
+                for line in sibling.to_string().lines() {
+                    if line.is_empty() {
+                        let _ = writeln!(content);
+                    } else {
+                        let _ = writeln!(content, "{begin} {line} {end}");
+                    }
+                }
+            }
+        }
+    }
+
+    if comments == "yes" || comments == "link" || comments == "noweb" || comments == "both" {
+        if let Some((begin, end)) = language_comments(&language) {
+            let _ = writeln!(
+                content,
+                "{begin} [[file:{path}::*{title}][{title}:{nth}]] {end}",
+                title = headline_title.as_deref().unwrap_or("No heading"),
+                path = path.to_string_lossy(),
+            );
+        }
+    }
+
+    content.push_str(&block.value());
+
+    if padline != "no" {
+        let _ = writeln!(content);
+    }
+
+    if comments == "yes" || comments == "link" || comments == "noweb" || comments == "both" {
+        if let Some((begin, end)) = language_comments(&language) {
+            let _ = writeln!(
+                content,
+                "{begin} {title}:{nth} ends here {end}",
+                title = headline_title.as_deref().unwrap_or("No heading"),
+            );
+        }
+    }
+
+    Ok(Some((path, permission, content, is_mkdir != "no")))
+}

orgize-common/src/utils.rs

@@ -0,0 +1,35 @@
+use orgize::ast::Headline;
+
+pub fn language_comments(language: &str) -> Option<(&str, &str)> {
+    match language {
+        "c" | "cpp" | "c++" | "go" | "js" | "javascript" | "ts" | "typescript" | "rust"
+        | "vera" | "jsonc" => Some(("//", "")),
+        "toml" | "tml" | "yaml" | "yml" | "conf" | "gitconfig" | "conf-toml" | "sh" | "shell"
+        | "bash" | "zsh" | "fish" => Some(("#", "")),
+        "lua" | "sql" => Some(("--", "")),
+        "lisp" | "emacs-lisp" | "elisp" => Some((";;", "")),
+        "xml" | "html" | "svg" => Some(("<!--", "-->")),
+        _ => None,
+    }
+}
+
+pub fn language_execute_command(language: &str) -> Option<&str> {
+    match language {
+        "js" | "javascript" => Some("node"),
+        "sh" | "bash" => Some("bash"),
+        "py" | "python" => Some("python"),
+        "fish" => Some("fish"),
+        _ => None,
+    }
+}
+
+pub fn headline_slug(headline: &Headline) -> String {
+    headline.title().fold(String::new(), |mut acc, elem| {
+        for ch in elem.to_string().chars() {
+            if ch.is_ascii_graphic() {
+                acc.push(ch);
+            }
+        }
+        acc
+    })
+}