sphinx_ultra/
parser.rs

1use anyhow::Result;
2use log::debug;
3use pulldown_cmark::{Event, Parser as MarkdownParser, Tag};
4use regex::Regex;
5use std::collections::HashMap;
6use std::path::Path;
7
8use crate::config::BuildConfig;
9use crate::document::{
10    CrossReference, Document, DocumentContent, MarkdownContent, MarkdownNode, RstContent,
11    RstDirective, RstNode, TocEntry,
12};
13use crate::utils;
14
15pub struct Parser {
16    rst_directive_regex: Regex,
17    cross_ref_regex: Regex,
18}
19
20impl Parser {
21    pub fn new(_config: &BuildConfig) -> Result<Self> {
22        // Directive names follow the docutils pattern: alphanumeric plus
23        // internal ._+:- (covers `code-block` and domain forms like `py:function`).
24        let rst_directive_regex = Regex::new(r"^\s*\.\.\s+([a-zA-Z][a-zA-Z0-9._+:-]*)::\s*(.*?)$")?;
25        let cross_ref_regex = Regex::new(r":(\w+):`([^`]+)`")?;
26
27        Ok(Self {
28            rst_directive_regex,
29            cross_ref_regex,
30        })
31    }
32
33    pub fn parse(&self, file_path: &Path, content: &str) -> Result<Document> {
34        let output_path = self.get_output_path(file_path)?;
35        let mut document = Document::new(file_path.to_path_buf(), output_path);
36
37        // Set source modification time
38        document.source_mtime = utils::get_file_mtime(file_path)?;
39
40        // Determine file type and parse accordingly
41        let extension = file_path
42            .extension()
43            .and_then(|ext| ext.to_str())
44            .unwrap_or("");
45
46        match extension {
47            "rst" => {
48                document.content = self.parse_rst(content)?;
49            }
50            "md" => {
51                document.content = self.parse_markdown(content)?;
52            }
53            _ => {
54                document.content = DocumentContent::PlainText(content.to_string());
55            }
56        }
57
58        // Extract title from content
59        document.title = self.extract_title(&document.content);
60
61        // Extract table of contents
62        document.toc = self.extract_toc(&document.content);
63
64        // Extract cross-references
65        document.cross_refs = self.extract_cross_refs(content);
66
67        debug!(
68            "Parsed document: {} ({} chars)",
69            file_path.display(),
70            content.len()
71        );
72
73        Ok(document)
74    }
75
76    fn parse_rst(&self, content: &str) -> Result<DocumentContent> {
77        let mut nodes = Vec::new();
78        let mut directives = Vec::new();
79        // docutils assigns section levels by order of first use of each
80        // adornment style, not by a fixed character table.
81        let mut adornment_order: Vec<char> = Vec::new();
82        let lines: Vec<&str> = content.lines().collect();
83
84        let mut i = 0;
85        while i < lines.len() {
86            let line = lines[i];
87            let trimmed = line.trim();
88
89            if trimmed.is_empty() {
90                i += 1;
91                continue;
92            }
93
94            // Check for RST directive
95            if let Some(captures) = self.rst_directive_regex.captures(line) {
96                let directive_name = captures.get(1).unwrap().as_str();
97                let directive_args = captures.get(2).unwrap().as_str();
98
99                let (directive, consumed_lines) =
100                    self.parse_rst_directive(&lines[i..], directive_name, directive_args, i + 1)?;
101
102                directives.push(directive.clone());
103                nodes.push(RstNode::Directive {
104                    name: directive.name,
105                    args: directive.args,
106                    options: directive.options,
107                    content: directive.content,
108                    line: i + 1,
109                });
110
111                i += consumed_lines;
112                continue;
113            }
114
115            // Check for title (underlined with =, -, ~, etc.)
116            if i + 1 < lines.len() {
117                let next_line = lines[i + 1];
118                if !next_line.trim().is_empty()
119                    && next_line.chars().all(|c| "=-~^\"'*+#<>".contains(c))
120                    && next_line.len() >= trimmed.len()
121                {
122                    let adornment = next_line.chars().next().unwrap();
123                    let level = match adornment_order.iter().position(|&c| c == adornment) {
124                        Some(pos) => pos + 1,
125                        None => {
126                            adornment_order.push(adornment);
127                            adornment_order.len()
128                        }
129                    };
130                    nodes.push(RstNode::Title {
131                        text: trimmed.to_string(),
132                        level,
133                        line: i + 1,
134                    });
135
136                    i += 2;
137                    continue;
138                }
139            }
140
141            // Check for code block (indented text after ::)
142            if line.ends_with("::") {
143                let (code_content, consumed_lines) = self.parse_code_block(&lines[i + 1..]);
144                nodes.push(RstNode::CodeBlock {
145                    language: None,
146                    content: code_content,
147                    line: i + 1,
148                });
149                i += consumed_lines + 1;
150                continue;
151            }
152
153            // Default to paragraph
154            let (paragraph_content, consumed_lines) = self.parse_paragraph(&lines[i..]);
155            nodes.push(RstNode::Paragraph {
156                content: paragraph_content,
157                line: i + 1,
158            });
159            i += consumed_lines;
160        }
161
162        Ok(DocumentContent::RestructuredText(RstContent {
163            raw: content.to_string(),
164            ast: nodes,
165            directives,
166        }))
167    }
168
169    fn parse_markdown(&self, content: &str) -> Result<DocumentContent> {
170        let mut nodes = Vec::new();
171        let parser = MarkdownParser::new(content);
172        let current_line = 1;
173
174        for event in parser {
175            match event {
176                Event::Start(Tag::Heading { .. }) => {
177                    // We'll handle this in the text event
178                }
179                Event::End(_) => {
180                    // Handle end tags generically
181                }
182                Event::Start(Tag::Paragraph) => {
183                    // Start of paragraph
184                }
185                Event::Start(Tag::CodeBlock(_)) => {
186                    // Start of code block
187                }
188                Event::Text(text) => {
189                    // Handle text content based on context
190                    nodes.push(MarkdownNode::Paragraph {
191                        content: text.to_string(),
192                        line: current_line,
193                    });
194                }
195                Event::Code(_code) => {
196                    // Inline code
197                }
198                _ => {
199                    // Handle other events as needed
200                }
201            }
202        }
203
204        Ok(DocumentContent::Markdown(MarkdownContent {
205            raw: content.to_string(),
206            ast: nodes,
207            front_matter: None, // TODO: Parse YAML front matter
208        }))
209    }
210
211    fn parse_rst_directive(
212        &self,
213        lines: &[&str],
214        name: &str,
215        args: &str,
216        start_line: usize,
217    ) -> Result<(RstDirective, usize)> {
218        let mut options = HashMap::new();
219        let mut content = String::new();
220        let mut consumed_lines = 1;
221        let mut i = 1;
222
223        // Parse options (lines starting with :option:)
224        while i < lines.len() {
225            let line = lines[i];
226            if line.trim().is_empty() {
227                i += 1;
228                consumed_lines += 1;
229                continue;
230            }
231
232            if let Some(stripped) = line
233                .strip_prefix("   :")
234                .or_else(|| line.strip_prefix("\t:"))
235            {
236                // This is an option
237                if let Some(colon_pos) = stripped.find(':') {
238                    let option_name = &stripped[..colon_pos];
239                    let option_value = stripped[colon_pos + 1..].trim();
240                    options.insert(option_name.to_string(), option_value.to_string());
241                }
242                i += 1;
243                consumed_lines += 1;
244            } else if line.starts_with("   ") || line.starts_with("\t") {
245                // This is content
246                break;
247            } else {
248                // End of directive
249                break;
250            }
251        }
252
253        // Parse content (indented lines)
254        while i < lines.len() {
255            let line = lines[i];
256            if line.starts_with("   ") || line.starts_with('\t') {
257                // Dedent one indentation unit without byte-slicing (a tab is one
258                // byte; `&line[3..]` panicked on short tab-indented lines).
259                let dedented = line
260                    .strip_prefix("   ")
261                    .or_else(|| line.strip_prefix('\t'))
262                    .unwrap_or(line);
263                content.push_str(dedented);
264                content.push('\n');
265                i += 1;
266                consumed_lines += 1;
267            } else if line.trim().is_empty() {
268                content.push('\n');
269                i += 1;
270                consumed_lines += 1;
271            } else {
272                break;
273            }
274        }
275
276        let directive = RstDirective {
277            name: name.to_string(),
278            args: if args.is_empty() {
279                Vec::new()
280            } else {
281                vec![args.to_string()]
282            },
283            options,
284            content: content.trim_end().to_string(),
285            line: start_line,
286        };
287
288        Ok((directive, consumed_lines))
289    }
290
291    fn parse_code_block(&self, lines: &[&str]) -> (String, usize) {
292        let mut content = String::new();
293        let mut consumed_lines = 0;
294
295        for line in lines {
296            if line.starts_with("   ") || line.starts_with("\t") || line.trim().is_empty() {
297                content.push_str(line);
298                content.push('\n');
299                consumed_lines += 1;
300            } else {
301                break;
302            }
303        }
304
305        (content.trim().to_string(), consumed_lines)
306    }
307
308    fn parse_paragraph(&self, lines: &[&str]) -> (String, usize) {
309        let mut content = String::new();
310        let mut consumed_lines = 0;
311
312        for line in lines {
313            let trimmed = line.trim();
314            if trimmed.is_empty() {
315                break;
316            }
317
318            content.push_str(trimmed);
319            content.push(' ');
320            consumed_lines += 1;
321        }
322
323        (content.trim().to_string(), consumed_lines)
324    }
325
326    fn extract_title(&self, content: &DocumentContent) -> String {
327        match content {
328            DocumentContent::RestructuredText(rst) => {
329                for node in &rst.ast {
330                    if let RstNode::Title { text, level: 1, .. } = node {
331                        return text.clone();
332                    }
333                }
334            }
335            DocumentContent::Markdown(md) => {
336                for node in &md.ast {
337                    if let MarkdownNode::Heading { text, level: 1, .. } = node {
338                        return text.clone();
339                    }
340                }
341            }
342            DocumentContent::PlainText(_) => {}
343        }
344
345        "Untitled".to_string()
346    }
347
348    fn extract_toc(&self, content: &DocumentContent) -> Vec<TocEntry> {
349        let mut toc = Vec::new();
350
351        match content {
352            DocumentContent::RestructuredText(rst) => {
353                for node in &rst.ast {
354                    if let RstNode::Title { text, level, line } = node {
355                        let anchor = text.to_lowercase().replace(' ', "-");
356                        toc.push(TocEntry::new(text.clone(), *level, anchor, *line));
357                    }
358                }
359            }
360            DocumentContent::Markdown(md) => {
361                for node in &md.ast {
362                    if let MarkdownNode::Heading { text, level, line } = node {
363                        let anchor = text.to_lowercase().replace(' ', "-");
364                        toc.push(TocEntry::new(text.clone(), *level, anchor, *line));
365                    }
366                }
367            }
368            DocumentContent::PlainText(_) => {}
369        }
370
371        toc
372    }
373
374    fn extract_cross_refs(&self, content: &str) -> Vec<CrossReference> {
375        let mut cross_refs = Vec::new();
376
377        for (line_num, line) in content.lines().enumerate() {
378            for captures in self.cross_ref_regex.captures_iter(line) {
379                let ref_type = captures.get(1).unwrap().as_str();
380                let target = captures.get(2).unwrap().as_str();
381
382                cross_refs.push(CrossReference {
383                    ref_type: ref_type.to_string(),
384                    target: target.to_string(),
385                    text: None,
386                    line_number: line_num + 1,
387                });
388            }
389        }
390
391        cross_refs
392    }
393
394    fn get_output_path(&self, source_path: &Path) -> Result<std::path::PathBuf> {
395        let mut output_path = source_path.to_path_buf();
396        output_path.set_extension("html");
397        Ok(output_path)
398    }
399}
400
401#[cfg(test)]
402mod tests {
403    use super::*;
404    use crate::config::BuildConfig;
405
406    fn parse_rst_ast(content: &str) -> Vec<RstNode> {
407        let parser = Parser::new(&BuildConfig::default()).unwrap();
408        match parser.parse_rst(content).unwrap() {
409            DocumentContent::RestructuredText(rst) => rst.ast,
410            _ => unreachable!(),
411        }
412    }
413
414    #[test]
415    fn hyphenated_directive_is_recognized() {
416        let ast = parse_rst_ast(".. code-block:: python\n\n   x = 1\n");
417        assert!(
418            ast.iter()
419                .any(|n| matches!(n, RstNode::Directive { name, .. } if name == "code-block")),
420            "code-block must parse as a directive, got: {ast:?}"
421        );
422    }
423
424    #[test]
425    fn domain_directive_is_recognized() {
426        let ast = parse_rst_ast(".. py:function:: foo(x)\n\n   Does foo.\n");
427        assert!(
428            ast.iter()
429                .any(|n| matches!(n, RstNode::Directive { name, .. } if name == "py:function")),
430            "py:function must parse as a directive, got: {ast:?}"
431        );
432    }
433
434    #[test]
435    fn tab_indented_directive_content_does_not_panic() {
436        let ast = parse_rst_ast(".. note::\n\n\tshort\n");
437        let content = ast.iter().find_map(|n| match n {
438            RstNode::Directive { content, .. } => Some(content.clone()),
439            _ => None,
440        });
441        assert!(
442            content.expect("directive parsed").contains("short"),
443            "tab-indented content must be captured"
444        );
445    }
446
447    #[test]
448    fn equals_underline_gets_level_one() {
449        let parser = Parser::new(&BuildConfig::default()).unwrap();
450        let content_ast = parser.parse_rst("Title\n=====\n\nBody.\n").unwrap();
451        if let DocumentContent::RestructuredText(ref rst) = content_ast {
452            assert!(
453                rst.ast
454                    .iter()
455                    .any(|n| matches!(n, RstNode::Title { level: 1, .. })),
456                "first adornment style must be level 1, got: {:?}",
457                rst.ast
458            );
459        }
460        assert_eq!(parser.extract_title(&content_ast), "Title");
461    }
462
463    #[test]
464    fn adornment_levels_by_order_of_first_use() {
465        let ast = parse_rst_ast("One\n===\n\nTwo\n---\n\nAlso One\n========\n");
466        let levels: Vec<usize> = ast
467            .iter()
468            .filter_map(|n| match n {
469                RstNode::Title { level, .. } => Some(*level),
470                _ => None,
471            })
472            .collect();
473        assert_eq!(levels, vec![1, 2, 1], "levels follow order of first use");
474    }
475}
sphinx_ultra/parser.rs

sphinx_ultra/
parser.rs
