sphinx_ultra/rst/
lines.rs

1//! Line preprocessing: docutils `statemachine.string2lines(tab_width=8,
2//! convert_whitespace=True)` equivalent, with byte-span mapping back to the
3//! original source (docutils loses this; we keep it for node spans).
4//!
5//! Probe-verified semantics (docutils 0.22.4):
6//! - `\v` / `\f` become single spaces (convert_whitespace).
7//! - Lines split on `\n`, `\r\n`, `\r` (plus Python `splitlines` exotics:
8//!   `\x1c`-`\x1e`, `\u{85}`, `\u{2028}`, `\u{2029}`).
9//! - Tabs expand to the next multiple-of-8 column (character columns).
10//! - Trailing whitespace stripped per line.
11//! - One processed line == one source line, so message line numbers are
12//!   `processed index + 1`.
13
14use std::ops::Deref;
15
16#[derive(Debug, Clone, PartialEq, Eq)]
17pub struct ProcessedLine {
18    pub text: String,
19    /// Byte range of the original line content (terminator excluded).
20    pub src_start: u32,
21    pub src_end: u32,
22}
23
24impl ProcessedLine {
25    /// Leading-space count (text is already tab-expanded and rstripped).
26    pub fn indent(&self) -> usize {
27        self.text.len() - self.text.trim_start_matches(' ').len()
28    }
29
30    pub fn is_blank(&self) -> bool {
31        self.text.is_empty()
32    }
33}
34
35#[derive(Debug, Clone, Default)]
36pub struct Lines(Vec<ProcessedLine>);
37
38impl Deref for Lines {
39    type Target = [ProcessedLine];
40
41    fn deref(&self) -> &[ProcessedLine] {
42        &self.0
43    }
44}
45
46fn is_line_boundary(c: char) -> bool {
47    matches!(
48        c,
49        '\n' | '\r' | '\x1c' | '\x1d' | '\x1e' | '\u{85}' | '\u{2028}' | '\u{2029}'
50    )
51}
52
53fn process_line(raw: &str) -> String {
54    // convert_whitespace (\v, \f -> space), then expandtabs(8), then rstrip.
55    let mut expanded = String::with_capacity(raw.len());
56    let mut col = 0usize;
57    for c in raw.chars() {
58        match c {
59            '\t' => {
60                let next_stop = (col / 8 + 1) * 8;
61                for _ in col..next_stop {
62                    expanded.push(' ');
63                }
64                col = next_stop;
65            }
66            '\x0b' | '\x0c' => {
67                expanded.push(' ');
68                col += 1;
69            }
70            _ => {
71                expanded.push(c);
72                col += 1;
73            }
74        }
75    }
76    expanded.trim_end().to_string()
77}
78
79impl Lines {
80    pub fn new(source: &str) -> Lines {
81        let mut lines = Vec::new();
82        let bytes_len = source.len();
83        let mut line_start = 0usize;
84        let mut chars = source.char_indices().peekable();
85        while let Some((i, c)) = chars.next() {
86            if is_line_boundary(c) {
87                lines.push(ProcessedLine {
88                    text: process_line(&source[line_start..i]),
89                    src_start: line_start as u32,
90                    src_end: i as u32,
91                });
92                // \r\n is one boundary
93                if c == '\r' {
94                    if let Some(&(_, '\n')) = chars.peek() {
95                        chars.next();
96                    }
97                }
98                line_start = match chars.peek() {
99                    Some(&(j, _)) => j,
100                    None => bytes_len,
101                };
102            }
103        }
104        if line_start < bytes_len {
105            lines.push(ProcessedLine {
106                text: process_line(&source[line_start..]),
107                src_start: line_start as u32,
108                src_end: bytes_len as u32,
109            });
110        }
111        Lines(lines)
112    }
113}
114
115#[cfg(test)]
116mod tests {
117    use super::*;
118
119    #[test]
120    fn tabs_expand_to_8_col_stops() {
121        assert_eq!(Lines::new("a\tb")[0].text, "a       b");
122        assert_eq!(Lines::new("\ta")[0].text, "        a");
123        assert_eq!(Lines::new("ab\tc")[0].text, "ab      c");
124        assert_eq!(Lines::new("abcdefgh\tz")[0].text, "abcdefgh        z");
125        assert_eq!(Lines::new("x\ty\tz")[0].text, "x       y       z");
126    }
127
128    #[test]
129    fn trailing_whitespace_stripped_and_spans_map_to_source() {
130        let src = "one  \ntwo";
131        let l = Lines::new(src);
132        assert_eq!(l[0].text, "one");
133        assert_eq!(
134            &src[l[0].src_start as usize..l[0].src_end as usize],
135            "one  "
136        );
137        assert_eq!(l[1].text, "two");
138        assert_eq!(&src[l[1].src_start as usize..l[1].src_end as usize], "two");
139    }
140
141    #[test]
142    fn vertical_tab_and_formfeed_become_spaces() {
143        assert_eq!(Lines::new("a\x0bb\x0cc")[0].text, "a b c");
144    }
145
146    #[test]
147    fn crlf_and_cr_split_without_stray_cr() {
148        let l = Lines::new("one\r\ntwo\rthree");
149        assert_eq!(l.len(), 3);
150        assert_eq!(l[0].text, "one");
151        assert_eq!(l[1].text, "two");
152        assert_eq!(l[2].text, "three");
153    }
154
155    #[test]
156    fn trailing_newline_produces_no_empty_last_line() {
157        assert_eq!(Lines::new("a\n").len(), 1);
158        let l = Lines::new("a\n\n");
159        assert_eq!(l.len(), 2);
160        assert!(l[1].is_blank());
161    }
162
163    #[test]
164    fn indent_counts_leading_spaces() {
165        let l = Lines::new("    four\n\tone-tab");
166        assert_eq!(l[0].indent(), 4);
167        assert_eq!(l[1].indent(), 8);
168    }
169}
sphinx_ultra/rst/lines.rs

sphinx_ultra/rst/
lines.rs
