Skip to main content

sphinx_ultra/
builder.rs

1use anyhow::Result;
2use log::{debug, info};
3use rayon::prelude::*;
4use std::borrow::Cow;
5use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
6use std::path::{Path, PathBuf};
7use std::sync::{Arc, Mutex};
8use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
9
10use crate::cache::BuildCache;
11use crate::config::BuildConfig;
12use crate::doctree::Doctree;
13use crate::document::Document;
14use crate::env;
15use crate::env::dependencies as env_dependencies;
16use crate::env::genindex as env_genindex;
17use crate::env::metadata as env_metadata;
18use crate::env::numbers as env_numbers;
19use crate::env::py_domain as env_py_domain;
20use crate::env::resolve as env_resolve;
21use crate::env::std_domain as env_std;
22use crate::env::toctree as env_toctree;
23use crate::env::toctree::{ConsistencyLevel, ToctreeWarningKind};
24use crate::env::BuildEnvironment;
25use crate::error::{BuildErrorReport, BuildWarning, ErrorType, WarningType};
26use crate::extensions::{ExtensionLoader, SphinxApp};
27use crate::intersphinx::{self, HttpConfig, Intersphinx, LoadRequest, UreqFetcher};
28use crate::inventory::InventoryFile;
29use crate::matching;
30use crate::parser::Parser;
31use crate::utils;
32use crate::utils::py_repr_str;
33
34/// Subdirectory of the cache dir holding one bincode doctree per document.
35/// It lives inside the `.config-fingerprint`-governed cache directory, so a
36/// configuration change wipes these along with everything else.
37const DOCTREE_SUBDIR: &str = "doctrees";
38
39/// Magic prefix identifying a persisted doctree file ("sphinx-ultra
40/// doctree"). Together with [`DOCTREE_FORMAT_VERSION`] it forms the 8-byte
41/// header [`SphinxBuilder::store_doctree`] writes ahead of the bincode blob.
42const DOCTREE_MAGIC: &[u8; 4] = b"SUDT";
43
44/// Format version of the per-document doctree files.
45///
46/// The blob itself is bincode, which has no field-presence framing and no
47/// self-description: bytes written by an older build decode *successfully*
48/// into a plausible-looking doctree and are then silently mis-read. This
49/// word is what turns that into an honest cache miss
50/// ([`SphinxBuilder::load_doctree`] returns `None` for a missing or
51/// mismatched version, and the document is re-read).
52///
53/// Bump it whenever previously written blobs would be misread, namely:
54/// - the serialized shape changes — a field added to or removed from
55///   `Doctree`/`Node`/`Attrs`, a different `AttrValue` variant set, or a
56///   different bincode configuration;
57/// - the *meaning* of what the parser stores changes while the shape does
58///   not. Wave 4's index-entry attribute moving from `AttrValue::Str` to
59///   `AttrValue::List` is the worked example: both variants decode, and an
60///   old blob then harvests the wrong index entries.
61///
62/// Version 2: wave 4.5's provenance change — `Span` gained a `line` field
63/// (and its byte range now indexes the parser's processed source text),
64/// so version-1 blobs no longer decode as written.
65const DOCTREE_FORMAT_VERSION: u32 = 2;
66
67/// Bytes of the [`DOCTREE_MAGIC`] + [`DOCTREE_FORMAT_VERSION`] header.
68const DOCTREE_HEADER_LEN: usize = DOCTREE_MAGIC.len() + std::mem::size_of::<u32>();
69
70/// Sphinx's `root_doc` default (`config.py`), used when the configuration
71/// leaves it unset.
72const DEFAULT_ROOT_DOC: &str = "index";
73
74/// One document's read-phase output.
75///
76/// This is the brief's `ReadResult { document, doctree, registry }` with the
77/// registry riding `document.registry` instead of sitting beside it: a
78/// cache hit skips parsing, so the only honest source for that document's
79/// registry is the one persisted with the cached `Document` (see
80/// [`Document::registry`]). Splitting it out would mean either a second
81/// copy or an empty stand-in on every cache hit.
82struct ReadResult {
83    /// Root-relative docname (`docname_of_path`).
84    docname: String,
85    document: Document,
86    doctree: Doctree,
87    /// Read completion time in microseconds since the epoch — what Sphinx
88    /// stores in `env.all_docs[docname]` (`builders/__init__.py:665`).
89    ///
90    /// `None` for a document this build did *not* read: its rendered output
91    /// and its doctree came back from the cache, and everything it
92    /// contributed to the environment — its read time included — is
93    /// whatever the build that did read it left behind.
94    read_time_us: Option<u64>,
95}
96
97#[derive(Debug, Clone)]
98pub struct BuildStats {
99    pub files_processed: usize,
100    pub files_skipped: usize,
101    pub build_time: Duration,
102    pub output_size_mb: f64,
103    pub cache_hits: usize,
104    pub errors: usize,
105    pub warnings: usize,
106    pub warning_details: Vec<BuildWarning>,
107    pub error_details: Vec<BuildErrorReport>,
108}
109
110pub struct SphinxBuilder {
111    config: BuildConfig,
112    source_dir: PathBuf,
113    output_dir: PathBuf,
114    cache: BuildCache,
115    parser: Parser,
116    parallel_jobs: usize,
117    incremental: bool,
118    warnings: Arc<Mutex<Vec<BuildWarning>>>,
119    errors: Arc<Mutex<Vec<BuildErrorReport>>>,
120    #[allow(dead_code)]
121    sphinx_app: Option<SphinxApp>,
122    #[allow(dead_code)]
123    extension_loader: ExtensionLoader,
124    /// Persisted build state (toctree graph, section/figure numbering, std
125    /// domain data, ...). Loaded from the cache dir's `env.bin` if present
126    /// and current; otherwise a fresh, empty environment.
127    ///
128    /// It steers the build: [`BuildEnvironment::get_outdated_files`] decides
129    /// which documents this build reads, the merge phase folds those (and
130    /// only those) back in, and the resolve phase saves it again. A document
131    /// that is not read keeps every contribution the build that read it
132    /// made.
133    env: BuildEnvironment,
134    /// docname -> the pseudo-XML of that document's *resolved* doctree, as
135    /// the resolve phase left it (Sphinx's `get_and_resolve_doctree`
136    /// output). Kept for [`Self::snapshot_env`], which is what the
137    /// environment-oracle differential diffs; the write phase does not
138    /// consume doctrees yet.
139    resolved: Mutex<BTreeMap<String, String>>,
140    /// Sphinx's `IndexBuilder`, fed one resolved document at a time by the
141    /// resolve phase (where Sphinx's `write_doc_serialized` feeds it) and
142    /// frozen into `searchindex.js` by [`Self::generate_search_index`].
143    search_index: Mutex<crate::search::SphinxIndexBuilder>,
144    /// The general index this build assembled (`IndexEntries.create_index`),
145    /// kept beside [`Self::resolved`] for the same reason: it is build
146    /// output derived from the environment plus the builder's own uri
147    /// scheme, not environment state.
148    genindex: Mutex<Vec<env_genindex::IndexGroup>>,
149    /// The Python module index (`PythonModuleIndex.generate`), build output
150    /// exactly like [`Self::genindex`] — sphinx assembles it while the HTML
151    /// builder writes the `py-modindex` page.
152    py_modindex: Mutex<env_py_domain::PyModindex>,
153    /// The cross-project inventories `intersphinx_mapping` names, loaded
154    /// once per build. Empty (and inert) unless a mapping is configured.
155    intersphinx: Intersphinx,
156}
157
158/// [`BuildConfig`] fields deliberately kept *out* of the cache fingerprint.
159///
160/// Both are operational switches that steer how this run reports
161/// diagnostics; neither changes the content the cache holds, and Sphinx
162/// treats neither as an environment-invalidating change:
163///
164/// * `nitpicky` (`-n`) is a `Config` value whose rebuild class is `''`
165///   (`sphinx/config.py:272`), and `_config_status` only reports
166///   `CONFIG_CHANGED` for values whose rebuild class is `'env'`
167///   (`sphinx/environment/__init__.py:366-369`) — so `-n` never invalidates
168///   Sphinx's environment.
169/// * `fail_on_warning` (`-W`, Sphinx's `warningiserror`) is not a `Config`
170///   value at all: it is a `sphinx-build` argument and can never take part
171///   in a config comparison.
172///
173/// Including them here would be worse than a spurious full rebuild. A
174/// fingerprint mismatch wipes the whole cache directory (documents,
175/// `doctrees/`, `env.bin`, `__intersphinx_cache__`), and read-phase
176/// diagnostics are only emitted for documents a build actually reads — so
177/// merely adding `-W` would force a cold read that re-emits every
178/// read-phase warning and fails, while the *next* identical `-W` run would
179/// be warm, emit nothing, and pass. Everything else stays in, `tags`
180/// included: tags select `only::` branches and therefore change parse
181/// output.
182const EXCLUDED_FROM_FINGERPRINT: [&str; 2] = ["fail_on_warning", "nitpicky"];
183
184/// The extensions discovery treats as documents, in precedence order.
185///
186/// Sphinx drives this from `source_suffix` (default `{'.rst': ...}`); this
187/// crate still hard-codes the triple M1's discovery filter has always used,
188/// which `crate::rst`'s `SOURCE_SUFFIXES` mirrors for toctree entries. The
189/// *order* is load-bearing: it decides which file wins when two of them map
190/// to the same docname, the way `source_suffix`'s order decides Sphinx's
191/// `doc2path` fallback.
192const DISCOVERY_SUFFIXES: [&str; 3] = ["rst", "md", "txt"];
193
194/// Whether Sphinx's *default* `source_suffix` — `{'.rst':
195/// 'restructuredtext'}` (`config.py:243`) — covers `path`.
196///
197/// This crate's discovery is deliberately wider than that, and the two
198/// places where the difference would otherwise reach the user's warning
199/// stream both gate on this predicate: the orphan check in
200/// [`SphinxBuilder::resolve_phase`] and the docname-collision report in
201/// [`SphinxBuilder::dedup_by_docname`]. Neither may fail a `-W` build over
202/// a file Sphinx would not have read in the first place.
203fn is_default_source_suffix(path: &Path) -> bool {
204    path.extension().is_some_and(|extension| extension == "rst")
205}
206
207/// `path`'s position in [`DISCOVERY_SUFFIXES`], or `None` if it is not a
208/// document at all.
209fn suffix_rank(path: &Path) -> Option<usize> {
210    let extension = path.extension()?.to_string_lossy();
211    DISCOVERY_SUFFIXES
212        .iter()
213        .position(|suffix| *suffix == extension)
214}
215
216/// blake3 over the configuration minus [`EXCLUDED_FROM_FINGERPRINT`].
217///
218/// The value is serialized through `serde_json::Value` before hashing, which
219/// also makes the digest order-independent: `Value::Object` is a `BTreeMap`,
220/// so every map in the configuration is emitted in sorted-key order no
221/// matter what order the source type iterates in.
222fn config_fingerprint(config: &BuildConfig) -> Result<String> {
223    let mut value = serde_json::to_value(config)?;
224    if let Some(map) = value.as_object_mut() {
225        for key in EXCLUDED_FROM_FINGERPRINT {
226            map.remove(key);
227        }
228    }
229    Ok(blake3::hash(serde_json::to_string(&value)?.as_bytes())
230        .to_hex()
231        .to_string())
232}
233
234/// Render the Sphinx 9.1 `.buildinfo` file.
235pub(crate) fn sphinx_build_info_contents(config: &BuildConfig, tags: &[String]) -> Result<String> {
236    let config_value = sphinx_html_config(config);
237    let config_hash = stable_hash(&config_value);
238    let mut sphinx_tags = ["builder_html", "format_html", "html"]
239        .into_iter()
240        .map(str::to_string)
241        .collect::<Vec<_>>();
242    sphinx_tags.extend(tags.iter().cloned());
243    sphinx_tags.sort();
244    sphinx_tags.dedup();
245    let tags_hash = stable_hash(&StableValue::List(
246        sphinx_tags.into_iter().map(StableValue::String).collect(),
247    ));
248    let newline = if cfg!(windows) { "\r\n" } else { "\n" };
249
250    Ok(format!(
251        "# Sphinx build info version 1{newline}\
252         # This file records the configuration used when building these files. When it is not found, a full rebuild will be done.{newline}\
253         config: {config_hash}{newline}\
254         tags: {tags_hash}{newline}"
255    ))
256}
257
258/// A small value model for Sphinx's `stable_hash`. Its collection branches
259/// deliberately hash child values first, sort those hashes, then hash the
260/// Python-style list representation — this is the algorithm in
261/// `sphinx.util._serialise`, not a JSON or Rust-debug serialization.
262#[derive(Clone)]
263enum StableValue {
264    None,
265    Bool(bool),
266    String(String),
267    List(Vec<StableValue>),
268    Dict(BTreeMap<String, StableValue>),
269    Tuple(Vec<StableValue>),
270    Raw(String),
271}
272
273fn stable_hash(value: &StableValue) -> String {
274    match value {
275        // Sphinx first replaces a dict with the sorted hashes of its items,
276        // then runs the list branch over that result once more. The second
277        // pass is easy to miss because lists and tuples only have one pass.
278        StableValue::Dict(values) => stable_collection(values.iter().map(|(key, value)| {
279            StableValue::String(stable_hash(&StableValue::Tuple(vec![
280                StableValue::String(key.clone()),
281                value.clone(),
282            ])))
283        })),
284        StableValue::List(values) | StableValue::Tuple(values) => {
285            stable_collection(values.iter().cloned())
286        }
287        StableValue::None => md5_hex("None"),
288        StableValue::Bool(value) => md5_hex(if *value { "True" } else { "False" }),
289        StableValue::String(value) | StableValue::Raw(value) => md5_hex(value),
290    }
291}
292
293fn stable_collection(values: impl Iterator<Item = StableValue>) -> String {
294    let mut hashes: Vec<String> = values.map(|value| stable_hash(&value)).collect();
295    hashes.sort();
296    let repr = hashes
297        .iter()
298        .map(|hash| format!("'{hash}'"))
299        .collect::<Vec<_>>()
300        .join(", ");
301    md5_hex(&format!("[{repr}]"))
302}
303
304fn stable_hash_json(value: &serde_json::Value) -> StableValue {
305    match value {
306        serde_json::Value::Null => StableValue::None,
307        serde_json::Value::Bool(value) => StableValue::Bool(*value),
308        serde_json::Value::Number(value) => StableValue::Raw(value.to_string()),
309        serde_json::Value::String(value) => StableValue::String(value.clone()),
310        serde_json::Value::Array(values) => {
311            StableValue::List(values.iter().map(stable_hash_json).collect())
312        }
313        serde_json::Value::Object(values) => StableValue::Dict(
314            values
315                .iter()
316                .map(|(key, value)| (key.clone(), stable_hash_json(value)))
317                .collect(),
318        ),
319    }
320}
321
322fn stable_strings(values: impl IntoIterator<Item = String>) -> StableValue {
323    StableValue::List(values.into_iter().map(StableValue::String).collect())
324}
325
326fn stable_dict(entries: impl IntoIterator<Item = (String, StableValue)>) -> StableValue {
327    StableValue::Dict(entries.into_iter().collect())
328}
329
330/// `BuildInfo.__init__`: `{c.name: c.value for c in config.filter({'html'})}`
331/// — every configuration value of the `html` rebuild category, as Sphinx
332/// holds it after `config-inited`.
333///
334/// A key `conf.py` set is hashed as the literal it was set to
335/// ([`BuildConfig::conf_values`]); every other key is hashed at Sphinx's
336/// default, including the defaults Sphinx computes from other values
337/// (`html_title`, `html_short_title`, `singlehtml_sidebars`,
338/// `qthelp_basename`). `html_context` is the one key read from the typed
339/// field, because `-A` adds to it the way `-D html_context.name=value` does.
340///
341/// `html_css_files`/`html_js_files` are hashed as the `(filename, attrs)`
342/// tuples `convert_html_css_files` rewrites them into at `config-inited`.
343/// The builder's `init_css_files` later adds a `priority` to those `attrs`,
344/// but `BuildInfo` has been computed by then, so the hash never sees it.
345fn sphinx_html_config(config: &BuildConfig) -> StableValue {
346    let conf = |key: &str| config.conf_values.get(key).map(stable_hash_json);
347    let none = StableValue::None;
348    let string = |value: &str| StableValue::String(value.to_string());
349    let empty_list = || StableValue::List(Vec::new());
350    let empty_dict = || stable_dict([]);
351
352    // `copyright` and `project_copyright` are two names for one value
353    // (`Config.__setattr__`); `copyright`'s default is `project_copyright`.
354    let copyright = conf("copyright")
355        .or_else(|| conf("project_copyright"))
356        .unwrap_or_else(|| string(""));
357    let html_title = conf("html_title").unwrap_or_else(|| {
358        StableValue::String(format!(
359            "{} {} documentation",
360            config.project,
361            config.release.as_deref().unwrap_or("")
362        ))
363    });
364    let html_short_title = conf("html_short_title").unwrap_or_else(|| html_title.clone());
365    let html_sidebars = conf("html_sidebars").unwrap_or_else(empty_dict);
366    let singlehtml_sidebars = conf("singlehtml_sidebars").unwrap_or_else(|| html_sidebars.clone());
367    let qthelp_basename = conf("qthelp_basename")
368        .unwrap_or_else(|| StableValue::String(make_filename(&config.project)));
369    let html_context = StableValue::Dict(
370        config
371            .html_context
372            .iter()
373            .map(|(key, value)| (key.clone(), stable_hash_json(value)))
374            .collect(),
375    );
376
377    let mut values = BTreeMap::new();
378    let defaults = [
379        ("pygments_style", none.clone()),
380        ("templates_path", empty_list()),
381        ("template_bridge", none.clone()),
382        ("modindex_common_prefix", empty_list()),
383        ("html_theme", string("alabaster")),
384        ("html_theme_path", empty_list()),
385        ("html_theme_options", empty_dict()),
386        ("html_style", none.clone()),
387        ("html_logo", none.clone()),
388        ("html_favicon", none.clone()),
389        ("html_static_path", empty_list()),
390        ("html_extra_path", empty_list()),
391        ("html_last_updated_fmt", none.clone()),
392        ("html_last_updated_use_utc", StableValue::Bool(false)),
393        ("html_additional_pages", empty_dict()),
394        ("html_domain_indices", StableValue::Bool(true)),
395        ("html_permalinks", StableValue::Bool(true)),
396        ("html_permalinks_icon", string("¶")),
397        ("html_use_index", StableValue::Bool(true)),
398        ("html_split_index", StableValue::Bool(false)),
399        ("html_copy_source", StableValue::Bool(true)),
400        ("html_show_sourcelink", StableValue::Bool(true)),
401        ("html_sourcelink_suffix", string(".txt")),
402        ("html_use_opensearch", string("")),
403        ("html_file_suffix", none.clone()),
404        ("html_link_suffix", none.clone()),
405        ("html_show_copyright", StableValue::Bool(true)),
406        ("html_show_search_summary", StableValue::Bool(true)),
407        ("html_show_sphinx", StableValue::Bool(true)),
408        ("html_output_encoding", string("utf-8")),
409        ("html_compact_lists", StableValue::Bool(true)),
410        ("html_secnumber_suffix", string(". ")),
411        ("html_search_language", none.clone()),
412        ("html_search_options", empty_dict()),
413        ("html_scaled_image_link", StableValue::Bool(true)),
414        ("html_baseurl", string("")),
415        ("html_codeblock_linenos_style", string("inline")),
416        ("html4_writer", StableValue::Bool(false)),
417        (
418            "mathjax_path",
419            string("https://cdn.jsdelivr.net/npm/mathjax@4/tex-mml-chtml.js"),
420        ),
421        ("mathjax_options", empty_dict()),
422        (
423            "mathjax_inline",
424            stable_strings(["\\(".to_string(), "\\)".to_string()]),
425        ),
426        (
427            "mathjax_display",
428            stable_strings(["\\[".to_string(), "\\]".to_string()]),
429        ),
430        ("mathjax_config", none.clone()),
431        ("mathjax2_config", none.clone()),
432        ("mathjax3_config", none.clone()),
433        ("mathjax4_config", none.clone()),
434        ("mathjax_config_path", string("")),
435        ("htmlhelp_file_suffix", none.clone()),
436        ("htmlhelp_link_suffix", none.clone()),
437        ("qthelp_namespace", none.clone()),
438        ("qthelp_theme", string("nonav")),
439        ("qthelp_theme_options", empty_dict()),
440    ];
441    for (key, default) in defaults {
442        values.insert(key.to_string(), conf(key).unwrap_or(default));
443    }
444    let computed = [
445        ("project_copyright", copyright.clone()),
446        ("copyright", copyright),
447        ("html_title", html_title),
448        ("html_short_title", html_short_title),
449        ("html_sidebars", html_sidebars),
450        ("singlehtml_sidebars", singlehtml_sidebars),
451        ("qthelp_basename", qthelp_basename),
452        ("html_context", html_context),
453        ("html_css_files", html_asset_files(config, "html_css_files")),
454        ("html_js_files", html_asset_files(config, "html_js_files")),
455    ];
456    for (key, value) in computed {
457        values.insert(key.to_string(), value);
458    }
459    StableValue::Dict(values)
460}
461
462/// `convert_html_css_files`/`convert_html_js_files` (`builders/html`,
463/// `config-inited`): a bare filename becomes `(filename, {})`, a
464/// `(filename, attrs)` pair stays, and anything else is dropped with a
465/// warning.
466fn html_asset_files(config: &BuildConfig, key: &str) -> StableValue {
467    let Some(serde_json::Value::Array(entries)) = config.conf_values.get(key) else {
468        return StableValue::List(Vec::new());
469    };
470    StableValue::List(
471        entries
472            .iter()
473            .filter_map(|entry| match entry {
474                serde_json::Value::String(filename) => Some(StableValue::Tuple(vec![
475                    StableValue::String(filename.clone()),
476                    stable_dict([]),
477                ])),
478                serde_json::Value::Array(pair) if pair.len() == 2 => {
479                    Some(StableValue::Tuple(vec![
480                        stable_hash_json(&pair[0]),
481                        stable_hash_json(&pair[1]),
482                    ]))
483                }
484                _ => None,
485            })
486            .collect(),
487    )
488}
489
490/// `sphinx.util.osutil.make_filename`: everything outside `[a-zA-Z0-9_-]`
491/// stripped, `sphinx` when nothing is left. `qthelp_basename`'s default.
492fn make_filename(name: &str) -> String {
493    let filename: String = name
494        .chars()
495        .filter(|c| c.is_ascii_alphanumeric() || matches!(c, '_' | '-'))
496        .collect();
497    if filename.is_empty() {
498        "sphinx".to_string()
499    } else {
500        filename
501    }
502}
503
504fn md5_hex(input: &str) -> String {
505    use md5::{Digest, Md5};
506    format!("{:x}", Md5::digest(input.as_bytes()))
507}
508
509impl SphinxBuilder {
510    pub fn new(config: BuildConfig, source_dir: PathBuf, output_dir: PathBuf) -> Result<Self> {
511        // -d/doctree_dir relocates the cache (sphinx-build's doctree dir).
512        let cache_dir = config
513            .doctree_dir
514            .clone()
515            .unwrap_or_else(|| output_dir.join(".sphinx-ultra-cache"));
516        // A change to any *content-bearing* configuration value invalidates
517        // cached documents (they were rendered under the old configuration);
518        // the two purely operational flags in `BuildConfig` are excluded —
519        // see [`config_fingerprint`].
520        let config_fingerprint = config_fingerprint(&config)?;
521        let cache = BuildCache::new(
522            cache_dir,
523            config.max_cache_size_mb,
524            config.cache_expiration_hours,
525            &config_fingerprint,
526        )?;
527
528        // Reuse whatever environment survived the fingerprint-wipe check
529        // above (BuildCache::new already discarded it if the config
530        // changed); a first build or an incompatible/corrupt env.bin both
531        // fall back to a fresh, empty environment.
532        let env = BuildEnvironment::load(cache.cache_dir()).unwrap_or_default();
533
534        // Canonicalize source_dir so it matches the canonicalized absolute paths
535        // returned by matching::get_matching_files; without this, relative
536        // --source paths (including the default ".") fail strip_prefix later.
537        let source_dir = crate::utils::canonicalize_simplified(&source_dir).unwrap_or(source_dir);
538
539        let parser = Parser::new(&config)?.with_srcdir(source_dir.clone());
540
541        let parallel_jobs = config.parallel_jobs.unwrap_or_else(|| {
542            std::thread::available_parallelism()
543                .map(|n| n.get())
544                .unwrap_or(4)
545        });
546
547        // Initialize Sphinx app with extensions
548        let mut sphinx_app = SphinxApp::new(config.clone())?;
549        let mut extension_loader = ExtensionLoader::new()?;
550
551        // Load configured extensions
552        for extension_name in &config.extensions {
553            match extension_loader.load_extension(extension_name) {
554                Ok(extension) => {
555                    if let Err(e) = sphinx_app.add_extension(extension) {
556                        log::warn!("Failed to add extension '{}': {}", extension_name, e);
557                    }
558                }
559                Err(e) => {
560                    log::warn!("Failed to load extension '{}': {}", extension_name, e);
561                }
562            }
563        }
564
565        Ok(Self {
566            config,
567            source_dir,
568            output_dir,
569            cache,
570            parser,
571            parallel_jobs,
572            incremental: false,
573            warnings: Arc::new(Mutex::new(Vec::new())),
574            errors: Arc::new(Mutex::new(Vec::new())),
575            sphinx_app: Some(sphinx_app),
576            extension_loader,
577            env,
578            resolved: Mutex::new(BTreeMap::new()),
579            search_index: Mutex::new(crate::search::SphinxIndexBuilder::new()),
580            genindex: Mutex::new(Vec::new()),
581            py_modindex: Mutex::new(env_py_domain::PyModindex::default()),
582            intersphinx: Intersphinx::default(),
583        })
584    }
585
586    /// Read every inventory `intersphinx_mapping` names — Sphinx's
587    /// `load_mappings`, which it runs at `builder-inited`, before the read
588    /// phase (`ext/intersphinx/__init__.py:80`).
589    ///
590    /// Local inventory locations are resolved against the source directory
591    /// and re-read on every build; remote ones go through [`UreqFetcher`]
592    /// and are cached under the (fingerprint-wiped) cache directory, so a
593    /// configuration change discards them along with everything else.
594    /// Fails where Sphinx raises `ConfigError` from `load_mappings` — an
595    /// entry that survived normalisation but violates
596    /// `_IntersphinxProject`'s invariants — which aborts the build with the
597    /// same config-error exit code an invalid mapping gets at config time.
598    fn load_intersphinx_inventories(&mut self) -> Result<()> {
599        if self.config.intersphinx_mapping.is_empty() {
600            return Ok(());
601        }
602        let http = HttpConfig {
603            tls_verify: self.config.tls_verify,
604            tls_cacerts: self.config.tls_cacerts.clone(),
605            user_agent: self.config.user_agent.clone(),
606            timeout: self.config.intersphinx_timeout,
607        };
608        let now = SystemTime::now()
609            .duration_since(UNIX_EPOCH)
610            .map(|since| since.as_secs() as i64)
611            .unwrap_or(0);
612        let outcome = intersphinx::load_mappings(
613            &LoadRequest {
614                mapping: &self.config.intersphinx_mapping,
615                srcdir: &self.source_dir,
616                cache_dir: Some(self.cache.cache_dir().join(intersphinx::CACHE_DIR_NAME)),
617                cache_limit: self.config.intersphinx_cache_limit,
618                now,
619                http: &http,
620            },
621            &UreqFetcher,
622        )?;
623        for message in outcome.infos {
624            info!("{message}");
625        }
626        for message in outcome.warnings {
627            // Sphinx logs these without a location and without a type, so
628            // they render as a bare `WARNING: ...` — but they still count
629            // toward the build's warning total and toward `-W`.
630            self.add_warning(BuildWarning::new(
631                PathBuf::new(),
632                None,
633                message,
634                WarningType::Other,
635            ));
636        }
637        self.intersphinx = Intersphinx {
638            data: outcome.data,
639            disabled_reftypes: self
640                .config
641                .intersphinx_disabled_reftypes
642                .iter()
643                .cloned()
644                .collect(),
645            resolve_self: self.config.intersphinx_resolve_self.clone(),
646        };
647        Ok(())
648    }
649
650    pub fn set_parallel_jobs(&mut self, jobs: usize) {
651        self.parallel_jobs = jobs;
652    }
653
654    pub fn enable_incremental(&mut self) {
655        self.incremental = true;
656    }
657
658    /// Discard the saved environment before building (sphinx-build `-E`).
659    ///
660    /// Sphinx's `-E` is `freshenv=True`: the pickled environment is not
661    /// loaded at all, and the fresh one it builds instead reports every
662    /// document as new. Emptying the cache directory is the same statement
663    /// about the *other* half of the persisted state (documents and
664    /// doctrees), and dropping the already-loaded environment here is what
665    /// keeps the two halves saying the same thing.
666    pub fn fresh_env(&mut self) -> Result<()> {
667        self.cache.clear()?;
668        self.env = BuildEnvironment::default();
669        Ok(())
670    }
671
672    /// Add a warning to the collection
673    #[allow(dead_code)]
674    pub fn add_warning(&self, warning: BuildWarning) {
675        self.warnings.lock().unwrap().push(warning);
676    }
677
678    /// Add an error to the collection
679    #[allow(dead_code)]
680    pub fn add_error(&self, error: BuildErrorReport) {
681        self.errors.lock().unwrap().push(error);
682    }
683
684    /// Check if warnings should be treated as errors
685    #[allow(dead_code)]
686    pub fn should_fail_on_warning(&self) -> bool {
687        self.config.fail_on_warning
688    }
689
690    pub async fn clean(&mut self) -> Result<()> {
691        if self.output_dir.exists() {
692            tokio::fs::remove_dir_all(&self.output_dir).await?;
693        }
694        // A clean build must not reuse documents cached before the clean
695        // (the on-disk cache lived inside the output dir we just removed),
696        // nor the environment that was loaded from it.
697        self.cache.clear()?;
698        self.env = BuildEnvironment::default();
699        Ok(())
700    }
701
702    /// Run the build: read → merge → resolve → write, then validation.
703    ///
704    /// The four phases mirror Sphinx's own split (`builders/__init__.py`):
705    ///
706    /// - **read** ([`Self::read_phase`], parallel): parse every *outdated*
707    ///   source file into a `Document` + doctree, and recover the rest from
708    ///   the cache.
709    /// - **merge** ([`Self::merge_phase`], sequential, docname-ordered):
710    ///   fold each re-read document's output into the [`BuildEnvironment`] —
711    ///   Sphinx's `merge_info_from` plus the collectors it dispatches — and
712    ///   persist that document's doctree once those have mutated it.
713    /// - **resolve** ([`Self::resolve_phase`]): whole-project state that
714    ///   needs every document read first, then persist the environment.
715    /// - **write** ([`Self::write_phase`]): emit the output files.
716    ///
717    /// `&mut self` only so the environment can be moved out and back; the
718    /// phase methods themselves take `&self` and the environment by
719    /// reference.
720    pub async fn build(&mut self) -> Result<BuildStats> {
721        let start_time = Instant::now();
722        info!("Starting build process...");
723
724        // Ensure output directory exists
725        tokio::fs::create_dir_all(&self.output_dir).await?;
726
727        // Discover all source files
728        let source_files = self.discover_source_files().await?;
729        info!("Discovered {} source files", source_files.len());
730
731        self.load_intersphinx_inventories()?;
732
733        let mut env = std::mem::take(&mut self.env);
734        let to_read = self.plan_read(&env, &source_files);
735        let mut read_results = self.read_phase(&source_files, &to_read)?;
736
737        self.merge_phase(&mut env, &mut read_results);
738        self.resolve_phase(&mut env, &read_results);
739        self.env = env;
740
741        let files_skipped = read_results
742            .iter()
743            .filter(|result| result.read_time_us.is_none())
744            .count();
745
746        // Keep documents in discovery order (the merge phase iterates a
747        // docname-sorted view of its own): the write and validation phases
748        // below produce warnings in this order, which is user-visible. The
749        // doctrees ride along for their source tables, which the
750        // validation pass needs to spell an included file's path.
751        let (processed_docs, doctrees): (Vec<Document>, Vec<Doctree>) = read_results
752            .into_iter()
753            .map(|result| (result.document, result.doctree))
754            .unzip();
755
756        self.write_phase(&processed_docs);
757
758        // Directive/role validation runs in every build unless disabled
759        if self.config.validate_directives {
760            self.validate_directives_and_roles(&processed_docs, &doctrees);
761        }
762
763        // Generate cross-references and indices
764        self.generate_indices(&processed_docs).await?;
765
766        // Copy static assets
767        self.copy_static_assets().await?;
768
769        // Generate sitemap and search index
770        self.generate_search_index().await?;
771        tokio::fs::write(
772            self.output_dir.join(".buildinfo"),
773            sphinx_build_info_contents(&self.config, &self.config.tags)?,
774        )
775        .await?;
776
777        let build_time = start_time.elapsed();
778        let output_size = utils::calculate_directory_size(&self.output_dir).await?;
779
780        let warnings = self.warnings.lock().unwrap();
781        let errors = self.errors.lock().unwrap();
782
783        let stats = BuildStats {
784            files_processed: processed_docs.len(),
785            files_skipped,
786            build_time,
787            output_size_mb: output_size as f64 / 1024.0 / 1024.0,
788            cache_hits: self.cache.hit_count(),
789            errors: errors.len(),
790            warnings: warnings.len(),
791            warning_details: warnings.clone(),
792            error_details: errors.clone(),
793        };
794
795        info!("Build completed in {:?}", build_time);
796        Ok(stats)
797    }
798
799    async fn discover_source_files(&self) -> Result<Vec<PathBuf>> {
800        // Use pattern-based file discovery like Sphinx
801        let include_patterns = &self.config.include_patterns;
802        let exclude_patterns = &self.config.exclude_patterns;
803
804        // Add built-in exclude patterns for common build artifacts and hidden files
805        let mut all_exclude_patterns = exclude_patterns.clone();
806        all_exclude_patterns.extend_from_slice(&[
807            "_build/**".to_string(),
808            "__pycache__/**".to_string(),
809            ".git/**".to_string(),
810            ".svn/**".to_string(),
811            ".hg/**".to_string(),
812            ".*/**".to_string(), // Skip all hidden directories
813            "Thumbs.db".to_string(),
814            ".DS_Store".to_string(),
815        ]);
816
817        match matching::get_matching_files(
818            &self.source_dir,
819            include_patterns,
820            &all_exclude_patterns,
821        ) {
822            // Sphinx's Project.discover keeps only files with a configured
823            // source suffix, regardless of include_patterns
824            Ok(files) => Ok(self.dedup_by_docname(
825                files
826                    .into_iter()
827                    .filter(|path| self.is_source_file(path))
828                    .collect(),
829            )),
830            Err(e) => {
831                log::warn!(
832                    "Pattern matching failed, falling back to simple discovery: {}",
833                    e
834                );
835                // Fallback to old method if pattern matching fails
836                let mut files = Vec::new();
837                self.discover_files_sync(&self.source_dir, &mut files)?;
838                Ok(self.dedup_by_docname(files))
839            }
840        }
841    }
842
843    /// Sphinx's `Project.discover` collision handling
844    /// (`project.py:64-79`): two files that map to one docname are not two
845    /// documents. Sphinx keeps whichever its directory walk yields first,
846    /// warns `multiple files found for the document "%s"` once, and names
847    /// the file it kept; this crate keeps the one whose suffix comes first
848    /// in [`DISCOVERY_SUFFIXES`], because a walk order is not something a
849    /// build should be allowed to depend on.
850    ///
851    /// Everything downstream of discovery is keyed by docname alone — the
852    /// persisted doctree, `env.all_docs`, `plan_read`'s source map, the
853    /// output path — so admitting both files meant the rendered page
854    /// alternated between them across builds, the loser's edits never
855    /// invalidated anything, and two threads wrote one output path at once.
856    ///
857    /// The *warning* is scoped the same way the orphan check is
858    /// ([`is_default_source_suffix`]): Sphinx's default `source_suffix` is
859    /// `.rst` alone, so a collision involving at most one `.rst` file is
860    /// not a collision Sphinx can see — it never discovered the `.md`/`.txt`
861    /// sibling at all, and builds `page.rst` beside `page.md` cleanly. Only
862    /// a collision between two files Sphinx would *both* have read is
863    /// reported; anything else drops its loser silently, so a wider
864    /// discovery set can never fail a `-W` build that Sphinx passes. No two
865    /// same-stem `.rst` files can exist in one directory, so today the
866    /// warning is unreachable by construction; it becomes reachable when
867    /// `source_suffix` turns configurable and a project names a second
868    /// restructuredtext suffix (wave 6, with MyST).
869    ///
870    /// Two deliberate deviations from Sphinx's message, both because
871    /// Sphinx's own rendering is an artifact rather than a contract: the
872    /// listed files are the colliding *documents* in the order this
873    /// function ranked them (Sphinx globs `docname.*`, so its list picks up
874    /// non-source neighbours and comes out in filesystem order), and the
875    /// kept path is rendered as a plain Python string repr (Sphinx's `%r`
876    /// hits `_StrPath.__repr__` and prints `_StrPath('/path')`).
877    fn dedup_by_docname(&self, files: Vec<PathBuf>) -> Vec<PathBuf> {
878        let mut by_docname: BTreeMap<String, Vec<PathBuf>> = BTreeMap::new();
879        for path in &files {
880            by_docname
881                .entry(self.docname_of_path(path))
882                .or_default()
883                .push(path.clone());
884        }
885
886        let mut dropped: BTreeSet<PathBuf> = BTreeSet::new();
887        for (docname, mut candidates) in by_docname {
888            if candidates.len() < 2 {
889                continue;
890            }
891            candidates.sort_by(|a, b| {
892                suffix_rank(a)
893                    .cmp(&suffix_rank(b))
894                    .then_with(|| a.as_path().cmp(b.as_path()))
895            });
896            // Only a collision Sphinx could see is worth a diagnostic.
897            if candidates
898                .iter()
899                .filter(|path| is_default_source_suffix(path))
900                .count()
901                > 1
902            {
903                let listed: Vec<String> = candidates
904                    .iter()
905                    .map(|path| {
906                        path.strip_prefix(&self.source_dir)
907                            .unwrap_or(path)
908                            .display()
909                            .to_string()
910                    })
911                    .collect();
912                self.add_warning(BuildWarning::new(
913                    // Logged with no `location`, so it prints bare.
914                    PathBuf::new(),
915                    None,
916                    format!(
917                        "multiple files found for the document \"{docname}\": {}\nUse {} for the build.",
918                        listed.join(", "),
919                        py_repr_str(&candidates[0].display().to_string()),
920                    ),
921                    WarningType::Other,
922                ));
923            }
924            dropped.extend(candidates.into_iter().skip(1));
925        }
926
927        if dropped.is_empty() {
928            return files;
929        }
930        // Discovery order is otherwise preserved: only the losers go.
931        files
932            .into_iter()
933            .filter(|path| !dropped.contains(path))
934            .collect()
935    }
936
937    /// Fallback file discovery for when pattern matching fails
938    fn discover_files_sync(&self, dir: &Path, files: &mut Vec<PathBuf>) -> Result<()> {
939        for entry in std::fs::read_dir(dir)? {
940            let entry = entry?;
941            let path = entry.path();
942
943            if path.is_dir() {
944                // Skip hidden directories and build artifacts
945                if let Some(name) = path.file_name() {
946                    if name.to_string_lossy().starts_with('.')
947                        || name == "_build"
948                        || name == "__pycache__"
949                    {
950                        continue;
951                    }
952                }
953
954                self.discover_files_sync(&path, files)?;
955            } else if self.is_source_file(&path) {
956                files.push(path);
957            }
958        }
959        Ok(())
960    }
961
962    /// Whether discovery treats `path` as a document
963    /// ([`DISCOVERY_SUFFIXES`]).
964    fn is_source_file(&self, path: &Path) -> bool {
965        suffix_rank(path).is_some()
966    }
967
968    /// Which documents this build has to read
969    /// ([`BuildEnvironment::get_outdated_files`]), and the `updating
970    /// environment:` line Sphinx prints about it.
971    ///
972    /// A non-incremental build reads everything, `sphinx-build -a` included
973    /// — that flag turns the document cache off here (see the mapping in
974    /// `main.rs`), where sphinx would still have read incrementally. Reading
975    /// more than sphinx does is slower, never wrong.
976    fn plan_read(&self, env: &BuildEnvironment, files: &[PathBuf]) -> BTreeSet<String> {
977        // `env.doc2path` for the documents this build discovered, and
978        // `env.found_docs` as its key set.
979        let sources: BTreeMap<String, PathBuf> = files
980            .iter()
981            .map(|path| (self.docname_of_path(path), path.clone()))
982            .collect();
983        let found: BTreeSet<String> = sources.keys().cloned().collect();
984
985        if !self.incremental {
986            debug!(
987                "Not an incremental build: reading all {} files",
988                found.len()
989            );
990            return found;
991        }
992
993        let outdated = env.get_outdated_files(
994            &found,
995            // A content-bearing configuration change wipes the cache
996            // directory whole (`.config-fingerprint`), so it reaches this
997            // point as an empty environment as well; saying it out loud
998            // keeps the two statements from drifting apart. Operational
999            // flags (`-W`, `-n`) are excluded from the fingerprint
1000            // (`EXCLUDED_FROM_FINGERPRINT`) and therefore never land here.
1001            self.cache.config_changed(),
1002            &env::FileTimes {
1003                source_modified_us: &|docname| {
1004                    sources.get(docname).and_then(|path| modified_us(path))
1005                },
1006                doctree_exists: &|docname| self.doctree_path(docname).is_file(),
1007                dependency_modified_us: &modified_us,
1008            },
1009        );
1010
1011        // Sphinx's `updating environment: %s added, %s changed, %s removed`
1012        // (`builders/__init__.py:493-497`), minus the `[reason]` prefix: the
1013        // whole-configuration fingerprint this crate uses cannot tell "new
1014        // config" from "config changed".
1015        info!(
1016            "updating environment: {} added, {} changed, {} removed",
1017            outdated.added.len(),
1018            outdated.changed.len(),
1019            outdated.removed.len()
1020        );
1021
1022        let mut to_read = outdated.to_read();
1023
1024        // Deliberate divergence: the toctrees that pointed at a deleted
1025        // document are read again.
1026        //
1027        // Sphinx does not do this — it resolves toctree entries a second
1028        // time while *writing* each page (`adapters/toctree.py`), which is
1029        // where its "toctree contains reference to nonexisting document"
1030        // warning comes from on a rebuild. This crate resolves entries once,
1031        // in the parser, so leaving the container unread would make the
1032        // deletion silent until the next cold build. Re-reading it is how a
1033        // read-time resolver keeps an incremental build's diagnostics equal
1034        // to a cold one's; it costs one re-parse per container, and only
1035        // when a document actually disappears.
1036        for removed in &outdated.removed {
1037            for container in env.files_to_rebuild.get(removed).into_iter().flatten() {
1038                if found.contains(container) {
1039                    to_read.insert(container.clone());
1040                }
1041            }
1042        }
1043
1044        to_read
1045    }
1046
1047    /// Read phase: parse every outdated source file in parallel, and
1048    /// recover the rest from the cache.
1049    ///
1050    /// One file failing must not abort the build: failures become
1051    /// `BuildErrorReport`s (and a non-zero exit) while the rest continue.
1052    /// Results keep the discovery order of `files`.
1053    fn read_phase(&self, files: &[PathBuf], to_read: &BTreeSet<String>) -> Result<Vec<ReadResult>> {
1054        info!(
1055            "Processing {} files with {} parallel jobs",
1056            files.len(),
1057            self.parallel_jobs
1058        );
1059
1060        // Sphinx's `env.found_docs`: known before any file is parsed, and
1061        // needed *during* the parse so `toctree` entries resolve.
1062        let found_docs = Arc::new(
1063            files
1064                .iter()
1065                .map(|path| self.docname_of_path(path))
1066                .collect::<BTreeSet<String>>(),
1067        );
1068
1069        // Configure rayon thread pool
1070        let pool = rayon::ThreadPoolBuilder::new()
1071            .num_threads(self.parallel_jobs)
1072            .build()?;
1073
1074        let results: Vec<(PathBuf, Result<ReadResult>)> = pool.install(|| {
1075            files
1076                .par_iter()
1077                .map(|file_path| {
1078                    let docname = self.docname_of_path(file_path);
1079                    let outdated = to_read.contains(&docname);
1080                    (
1081                        file_path.clone(),
1082                        self.read_one_file(file_path, docname, &found_docs, outdated),
1083                    )
1084                })
1085                .collect()
1086        });
1087
1088        let mut read_results = Vec::with_capacity(results.len());
1089        for (file_path, result) in results {
1090            match result {
1091                Ok(read) => read_results.push(read),
1092                Err(e) => {
1093                    self.errors.lock().unwrap().push(BuildErrorReport::new(
1094                        file_path,
1095                        None,
1096                        format!("{e:#}"),
1097                        ErrorType::ParseError,
1098                    ));
1099                }
1100            }
1101        }
1102
1103        Ok(read_results)
1104    }
1105
1106    /// One document's read-phase result.
1107    ///
1108    /// `outdated` decides *how*: an outdated document is parsed (its cache
1109    /// entry, however valid, describes a document the environment has
1110    /// already been told to forget), an up-to-date one is recovered whole
1111    /// from the cache — rendered page and persisted doctree — and is not
1112    /// merged into the environment again. A recovery that fails is not
1113    /// fatal: the document is parsed instead, which is honest work rather
1114    /// than a hit, and the cache counts it as the miss it is.
1115    fn read_one_file(
1116        &self,
1117        file_path: &Path,
1118        docname: String,
1119        found_docs: &Arc<BTreeSet<String>>,
1120        outdated: bool,
1121    ) -> Result<ReadResult> {
1122        let relative_path = file_path.strip_prefix(&self.source_dir)?;
1123        debug!("Processing file: {}", relative_path.display());
1124
1125        // The write phase still writes an unread document's page — skipping
1126        // the write is how cached pages went missing from the output tree.
1127        if !outdated && self.incremental {
1128            if let Ok(file_mtime) = utils::get_file_mtime(file_path) {
1129                let hit = self.cache.get_document_with(file_path, |cached| {
1130                    if cached.source_mtime < file_mtime || cached.html.is_empty() {
1131                        return None;
1132                    }
1133                    // A cached document whose doctree file is gone, or was
1134                    // written in a format this build no longer reads, is not
1135                    // usable: the resolve phase needs that doctree, and
1136                    // inventing an empty one would quietly drop the
1137                    // document's toc, titles and toctrees. Re-parse instead
1138                    // — this counts as a cache miss.
1139                    self.load_doctree(&docname)
1140                });
1141                if let Some((document, doctree)) = hit {
1142                    debug!("Using cached version of {}", relative_path.display());
1143                    return Ok(ReadResult {
1144                        docname,
1145                        document,
1146                        doctree,
1147                        read_time_us: None,
1148                    });
1149                }
1150            }
1151        }
1152
1153        // Read and parse the file
1154        let content = std::fs::read_to_string(file_path)?;
1155        let parsed =
1156            self.parser
1157                .parse_full(file_path, &content, &docname, Some(Arc::clone(found_docs)))?;
1158        let mut document = parsed.document;
1159
1160        // Simple document rendering (placeholder). Done here, in the read
1161        // phase, because the rendered HTML is part of what the incremental
1162        // cache stores; the write phase only puts it on disk.
1163        let rendered_html = format!(
1164            "<html><body>{}</body></html>",
1165            html_escape::encode_text(&document.content.to_string())
1166        );
1167        document.html = rendered_html;
1168
1169        // The doctree is *not* persisted here: the domain hooks the merge
1170        // phase runs still mutate it (the index domain removes a node whose
1171        // entries do not validate), and what a later build loads has to be
1172        // the tree this build resolved. See [`Self::merge_phase`].
1173
1174        // Cache the document
1175        if self.incremental {
1176            self.cache.store_document(file_path, &document)?;
1177        }
1178
1179        Ok(ReadResult {
1180            docname,
1181            document,
1182            doctree: parsed.doctree,
1183            read_time_us: Some(now_micros()),
1184        })
1185    }
1186
1187    /// Merge phase: fold the read phase's per-document output into the
1188    /// environment, in docname order.
1189    ///
1190    /// Sequential and deterministic, mirroring Sphinx's `merge_info_from`
1191    /// (`environment/__init__.py:421`) and the collectors it dispatches to:
1192    /// `all_docs`, the title collector, and the toctree collector.
1193    ///
1194    /// Only documents this build actually **read** are merged. Sphinx's
1195    /// `Builder._read_serial` clears a document immediately before reading
1196    /// it and touches nothing else; a document that was not outdated keeps
1197    /// every contribution the build that read it made, which is the whole
1198    /// point of the environment being persistent.
1199    ///
1200    /// This phase also **persists** each re-read document's doctree, at the
1201    /// end of its merge: the domain hooks below mutate the tree, and a
1202    /// later build must load what they left, not what the parser produced.
1203    fn merge_phase(&self, env: &mut BuildEnvironment, results: &mut [ReadResult]) {
1204        env.root_doc = self
1205            .config
1206            .root_doc
1207            .clone()
1208            .unwrap_or_else(|| DEFAULT_ROOT_DOC.to_string());
1209
1210        // Documents that vanished since the saved environment was written
1211        // (Sphinx's `removed` set) must not leave stale state behind. The
1212        // set is taken from what the read phase came back with rather than
1213        // from `get_outdated_files`: it is the same set plus any document
1214        // that failed to read, whose recorded state is equally worthless.
1215        let present: HashSet<&str> = results.iter().map(|r| r.docname.as_str()).collect();
1216        let stale: Vec<String> = env
1217            .all_docs
1218            .keys()
1219            .filter(|docname| !present.contains(docname.as_str()))
1220            .cloned()
1221            .collect();
1222        for docname in stale {
1223            env.clear_doc(&docname);
1224        }
1225
1226        let mut ordered: Vec<usize> = (0..results.len()).collect();
1227        ordered.sort_by(|a, b| results[*a].docname.cmp(&results[*b].docname));
1228
1229        // Sphinx's `env.doc2path`: the source file a docname was read from.
1230        // Documents this build did not read fall back to the conventional
1231        // `<srcdir>/<docname>.rst`, the same shape `doc2path` synthesizes
1232        // from `source_suffix`.
1233        //
1234        // Owned rather than borrowed from `results`, which this loop holds
1235        // mutably: the index domain *removes* an `index` node whose entries
1236        // do not validate.
1237        let paths: HashMap<String, PathBuf> = results
1238            .iter()
1239            .map(|result| (result.docname.clone(), result.document.source_path.clone()))
1240            .collect();
1241        let doc2path = |docname: &str| -> PathBuf {
1242            paths
1243                .get(docname)
1244                .cloned()
1245                .unwrap_or_else(|| self.source_dir.join(format!("{docname}.rst")))
1246        };
1247
1248        for index in ordered {
1249            let result = &mut results[index];
1250            // A document this build did not read contributes nothing: what
1251            // it contributed last time is still in the environment, and
1252            // still correct.
1253            let Some(read_time_us) = result.read_time_us else {
1254                continue;
1255            };
1256            let docname = result.docname.clone();
1257            let docname = docname.as_str();
1258            // A re-read replaces this document's state wholesale — without
1259            // the clear, the `extend`-shaped fields (toctree_includes)
1260            // would accumulate duplicates across rebuilds.
1261            env.clear_doc(docname);
1262
1263            env.all_docs.insert(docname.to_string(), read_time_us);
1264
1265            let title = env_toctree::document_title(&result.doctree);
1266            // Sphinx's longtitle differs from the title only for documents
1267            // carrying an explicit `title` attribute, which nothing produces
1268            // yet (`collectors/title.py:27`).
1269            env.longtitles.insert(docname.to_string(), title.clone());
1270            env.titles.insert(docname.to_string(), title);
1271
1272            env.metadata.insert(
1273                docname.to_string(),
1274                env_metadata::document_metadata(&result.doctree),
1275            );
1276
1277            // The files this document pulls in, which is what makes it
1278            // outdated when one of *them* changes: image uris from the
1279            // doctree walk plus the parse-recorded include targets.
1280            env_dependencies::process_doc(
1281                env,
1282                docname,
1283                &result.doctree,
1284                &self.source_dir,
1285                &result.document.registry.dependencies,
1286            );
1287
1288            // The docnames it textually includes (`env.note_included`
1289            // replayed from the parse records): the orphan check in
1290            // `check_consistency` is the one consumer.
1291            let included: std::collections::BTreeSet<String> =
1292                result.document.registry.included.iter().cloned().collect();
1293            if !included.is_empty() {
1294                env.included.insert(docname.to_string(), included);
1295            }
1296
1297            let (toc, num_entries) = env_toctree::build_toc(&result.doctree, docname);
1298            // Each toctree node copied into the toc is noted, in the order
1299            // it was copied (which is the order Sphinx notes them in).
1300            for toctree in env_toctree::toctree_copies(&toc) {
1301                env_toctree::note_toctree(env, docname, toctree);
1302            }
1303            env.tocs.insert(docname.to_string(), toc);
1304            env.toc_num_entries.insert(docname.to_string(), num_entries);
1305
1306            // `TocTree.parse_content` calls `env.note_reread()` for every
1307            // entry that names a document the project does not have
1308            // (`directives/other.py`): such a document is re-read on every
1309            // build, so that the day the missing target appears its toctree
1310            // takes it up — and stops warning about it. `clear_doc` above
1311            // dropped the previous read's claim, so a document that no
1312            // longer has a dangling entry is no longer re-read either.
1313            if result.document.toctrees.iter().any(|toctree| {
1314                toctree
1315                    .warnings
1316                    .iter()
1317                    .any(|warning| warning.kind == ToctreeWarningKind::MissingDocument)
1318            }) {
1319                env.reread_always.insert(docname.to_string());
1320            }
1321
1322            // The document's toctree diagnostics, produced when its entries
1323            // were resolved. Sphinx logs them during the read phase, which
1324            // walks documents in this same sorted order.
1325            self.report_parse_warnings(&result.document, &result.doctree);
1326
1327            // The domains' read-phase hooks, dispatched in the order
1328            // `_DomainsContainer._process_doc` walks them — `c, changeset,
1329            // citation, cpp, index, js, math, py, rst, std`, so `index`
1330            // before `std` with `py` in between — and after the parse
1331            // diagnostics above, which Sphinx logs while reading.
1332            // `PythonDomain` defines no `process_doc` hook, so its slot in
1333            // that walk is a no-op: the py registrations (and their
1334            // duplicate warnings, which are parse-time in Sphinx and
1335            // interleave with std's in document order) replay inside
1336            // `env_std::process_doc`'s parse-time pass below. Warning
1337            // locations come from node spans and the doctree's source
1338            // table, not the document text.
1339            let mut index_warnings = Vec::new();
1340            env_genindex::process_doc(
1341                env,
1342                docname,
1343                &mut result.doctree,
1344                &result.document.source_path,
1345                &mut index_warnings,
1346            );
1347            for warning in index_warnings {
1348                self.add_warning(warning);
1349            }
1350
1351            let mut std_warnings = Vec::new();
1352            env_std::process_doc(
1353                env,
1354                &env_std::DocumentSource {
1355                    docname,
1356                    doctree: &result.doctree,
1357                    registry: &result.document.registry,
1358                    path: &result.document.source_path,
1359                },
1360                &doc2path,
1361                &mut std_warnings,
1362            );
1363            for warning in std_warnings {
1364                self.add_warning(warning);
1365            }
1366
1367            // Persist the doctree *after* the hooks above have had it, the
1368            // way Sphinx writes its pickle after the read's transforms and
1369            // `doctree-read` handlers run (`builders/__init__.py:632-671`;
1370            // `IndexDomain.process_doc` at `domains/index.py:47-60` is the
1371            // one that mutates here, removing an `index` node whose entries
1372            // did not validate). Persisting the parser's tree instead would
1373            // hand the next build a document this build already rejected
1374            // part of — and the two builds would resolve different trees.
1375            //
1376            // Failing to write it is reported like any other per-document
1377            // failure, but the file is removed first: a stale doctree left
1378            // beside a fresh `all_docs` entry would look current to the next
1379            // build, whereas a missing one simply makes the document
1380            // outdated and re-read.
1381            if let Err(e) = self.store_doctree(docname, &result.doctree) {
1382                let _ = std::fs::remove_file(self.doctree_path(docname));
1383                self.errors.lock().unwrap().push(BuildErrorReport::new(
1384                    result.document.source_path.clone(),
1385                    None,
1386                    format!("{e:#}"),
1387                    ErrorType::Other,
1388                ));
1389            }
1390        }
1391    }
1392
1393    /// Surface one document's parse-time diagnostics: `TocTree.parse_content`'s
1394    /// warnings and the `logger.warning` calls other directives make
1395    /// (`RegistryExport::log_warnings`). Both are carried on the parse
1396    /// records rather than raised as they happen, so that a cache hit — which
1397    /// skips the parse entirely — still reproduces them.
1398    ///
1399    /// Each stream replays in its own record sequence — the order the parse
1400    /// produced. (The old stable sort by line only reproduced document
1401    /// order while every record came from one source; an included file's
1402    /// warnings would be shuffled into the includer's. A warning's line is
1403    /// display data, not an ordering key.) A document carrying both kinds
1404    /// emits all toctree warnings before all log warnings rather than
1405    /// interleaved by position — the same cross-category simplification
1406    /// `std_domain::process_doc` documents.
1407    ///
1408    /// `doctree` supplies the source table: a warning raised inside an
1409    /// included file — a toctree's `location=toctree` or a log warning's
1410    /// `location=node` — renders that file's path, not the document's.
1411    fn report_parse_warnings(&self, document: &Document, doctree: &Doctree) {
1412        let mut ordered: Vec<BuildWarning> = Vec::new();
1413        for toctree in &document.toctrees {
1414            for warning in &toctree.warnings {
1415                let warning_type = match warning.kind {
1416                    ToctreeWarningKind::MissingDocument => WarningType::MissingToctreeRef,
1417                    ToctreeWarningKind::EmptyGlob | ToctreeWarningKind::PatternError => {
1418                        WarningType::EmptyToctree
1419                    }
1420                    ToctreeWarningKind::DuplicateEntry => WarningType::Other,
1421                };
1422                let source_path = doctree
1423                    .sources
1424                    .get(warning.source as usize)
1425                    .map(PathBuf::from)
1426                    .unwrap_or_else(|| document.source_path.clone());
1427                ordered.push(
1428                    BuildWarning::new(
1429                        source_path,
1430                        Some(warning.line as usize),
1431                        warning.message.clone(),
1432                        warning_type,
1433                    )
1434                    .with_category(warning.category.clone()),
1435                );
1436            }
1437        }
1438        for warning in &document.registry.log_warnings {
1439            // `rendered_path` reproduces sphinx's tuple-location doc2path
1440            // append (the doubled `.rst.rst` quirk) for the records that
1441            // carry it — see `ParseLogWarning::rendered_path` for WHY.
1442            let source_path = doctree
1443                .sources
1444                .get(warning.source as usize)
1445                .map(|path| PathBuf::from(warning.rendered_path(path)))
1446                .unwrap_or_else(|| document.source_path.clone());
1447            // Sphinx logs these with no `type`/`subtype`, so they render
1448            // with no `[category]` suffix.
1449            ordered.push(BuildWarning::new(
1450                source_path,
1451                Some(warning.line as usize),
1452                warning.message.clone(),
1453                WarningType::Other,
1454            ));
1455        }
1456        let mut warnings = self.warnings.lock().unwrap();
1457        warnings.extend(ordered);
1458    }
1459
1460    /// Resolve phase: whole-project state that only exists once every
1461    /// document has been read, then persist the environment.
1462    ///
1463    /// Runs the numbering passes (`TocTreeCollector.get_updated_docs`, which
1464    /// Sphinx dispatches through `env-get-updated` right after the read
1465    /// phase) and Sphinx's post-read consistency checks over the finished
1466    /// toctree graph (`env.check_consistency()`), then saves the environment
1467    /// — Sphinx's own end-of-read-phase step (`builders/__init__.py:420`).
1468    ///
1469    /// Every document is resolved, not only the ones this build read: the
1470    /// write phase emits every page (see [`Self::write_phase`]), and a page
1471    /// is written from a doctree resolved against the environment as it
1472    /// stands *now* — a document that was not re-read can still have gained
1473    /// a section number, or lost the target of one of its references.
1474    ///
1475    /// Failing to save the environment is **not** a build failure: the
1476    /// cache directory is optional infrastructure, the output this build
1477    /// produced is valid without it, and the only consequence is that the
1478    /// next build starts cold. It is reported and the build goes on.
1479    fn resolve_phase(&self, env: &mut BuildEnvironment, results: &[ReadResult]) {
1480        info!("Resolving build environment");
1481
1482        let sources: HashMap<&str, &Path> = results
1483            .iter()
1484            .map(|result| {
1485                (
1486                    result.docname.as_str(),
1487                    result.document.source_path.as_path(),
1488                )
1489            })
1490            .collect();
1491
1492        self.number_phase(env, results);
1493        // Sphinx's default `source_suffix` is `.rst` alone, so a `.md`/`.txt`
1494        // this crate's wider discovery admitted is not a document Sphinx
1495        // would warn about being orphaned. A docname with no source in this
1496        // build (nothing was read for it) is treated as one, which is the
1497        // pre-existing behavior.
1498        let orphan_candidate = |docname: &str| {
1499            sources
1500                .get(docname)
1501                .is_none_or(|path| is_default_source_suffix(path))
1502        };
1503        for message in env_toctree::check_consistency(env, &orphan_candidate) {
1504            // Sphinx logs these with `location=docname`, which renders as
1505            // the document's source path with no line number.
1506            let source = sources
1507                .get(message.docname.as_str())
1508                .map(|path| path.to_path_buf())
1509                .unwrap_or_else(|| PathBuf::from(&message.docname));
1510            match message.level {
1511                ConsistencyLevel::Warning => self.warnings.lock().unwrap().push(
1512                    BuildWarning::new(source, None, message.message, WarningType::OrphanedDocument)
1513                        .with_category(message.category),
1514                ),
1515                // Sphinx uses `logger.info` for the multiple-parents note,
1516                // so it must stay out of the warning count (and out of -W).
1517                ConsistencyLevel::Info => info!("{}: {}", source.display(), message.message),
1518            }
1519        }
1520
1521        self.xref_phase(env, results);
1522        self.genindex_phase(env, &sources);
1523        self.py_modindex_phase(env);
1524
1525        if let Err(e) = env.save(self.cache.cache_dir()) {
1526            log::warn!(
1527                "Could not save the build environment to {}: {e:#} — this build's \
1528                 output is complete, but the next one will start from scratch",
1529                self.cache.cache_dir().display()
1530            );
1531        }
1532    }
1533
1534    /// Assemble the general index (`IndexEntries.create_index`).
1535    ///
1536    /// Sphinx runs this from the HTML builder's `write_genindex`, so a
1537    /// `dummy` build never reaches it — but the environment oracle calls it
1538    /// explicitly, right after the build and before snapshotting warnings
1539    /// (see `tools/gen_env_fixture.py`), which is why this runs last and
1540    /// unconditionally rather than from the writer.
1541    fn genindex_phase(&self, env: &BuildEnvironment, sources: &HashMap<&str, &Path>) {
1542        // The oracle's dummy builder answers `get_relative_uri('genindex',
1543        // docname)` with `''` for every document, making each target a bare
1544        // `#<target_id>` — the same honest answer [`Self::xref_phase`] gives
1545        // until the HTML writer supplies its own uri scheme.
1546        let rel_uri = |_docname: &str| Some(String::new());
1547        let mut messages = Vec::new();
1548        let groups = env_genindex::create_index(env, &rel_uri, &mut messages);
1549        for message in messages {
1550            // `location=docname`: the document's source path, no line.
1551            let source = sources
1552                .get(message.docname.as_str())
1553                .map(|path| path.to_path_buf())
1554                .unwrap_or_else(|| PathBuf::from(&message.docname));
1555            self.add_warning(message.into_warning(&source));
1556        }
1557        *self.genindex.lock().unwrap() = groups;
1558    }
1559
1560    /// Assemble the Python module index (`PythonModuleIndex.generate`).
1561    ///
1562    /// Like [`Self::genindex_phase`], Sphinx only runs this from an HTML
1563    /// build (`write_domain_indices`) — the environment oracle calls
1564    /// `generate()` explicitly after its dummy build, so this too runs
1565    /// unconditionally at the end of the resolve phase. It raises no
1566    /// diagnostics of its own.
1567    fn py_modindex_phase(&self, env: &BuildEnvironment) {
1568        *self.py_modindex.lock().unwrap() =
1569            env_py_domain::generate_modindex(&env.py, &self.config.modindex_common_prefix);
1570    }
1571
1572    /// Cross-reference resolution (`ReferencesResolver`, run per document as
1573    /// Sphinx writes it — after numbering, which `:numref:` reads).
1574    ///
1575    /// Each document is resolved over a *copy* of its doctree, exactly like
1576    /// Sphinx's `get_and_resolve_doctree`, and the result is kept as
1577    /// pseudo-XML for [`Self::snapshot_env`]. Documents are visited in
1578    /// docname order, which is the order Sphinx's write loop uses and
1579    /// therefore the order its warnings come out in.
1580    fn xref_phase(&self, env: &BuildEnvironment, results: &[ReadResult]) {
1581        let in_memory: HashMap<&str, &Doctree> = results
1582            .iter()
1583            .map(|result| (result.docname.as_str(), &result.doctree))
1584            .collect();
1585        let load_doctree = |docname: &str| -> Option<Cow<'_, Doctree>> {
1586            match in_memory.get(docname) {
1587                Some(doctree) => Some(Cow::Borrowed(*doctree)),
1588                None => self.load_doctree(docname).map(Cow::Owned),
1589            }
1590        };
1591        // The oracle builds with sphinx's dummy builder, whose
1592        // `get_target_uri` is `''`; nothing consumes a resolved doctree's
1593        // URIs yet (the write phase renders from `Document.html`), so this
1594        // is the one honest answer until the HTML writer supplies its own.
1595        let relative_uri = |_from: &str, _to: &str| String::new();
1596        let resolver = env_resolve::Resolver {
1597            env,
1598            numfig: self.config.numfig,
1599            numfig_format: &self.config.numfig_format,
1600            doctree: &load_doctree,
1601            relative_uri: &relative_uri,
1602            intersphinx: &self.intersphinx,
1603        };
1604        let nitpick = env_resolve::NitpickConfig {
1605            nitpicky: self.config.nitpicky,
1606            ignore: &self.config.nitpick_ignore,
1607            ignore_regex: &self.config.nitpick_ignore_regex,
1608        };
1609
1610        let mut ordered: Vec<&ReadResult> = results.iter().collect();
1611        ordered.sort_by(|a, b| a.docname.cmp(&b.docname));
1612
1613        // A second `build()` on the same builder must not keep the previous
1614        // one's documents (one of them may since have been deleted).
1615        self.resolved.lock().unwrap().clear();
1616        *self.search_index.lock().unwrap() = crate::search::SphinxIndexBuilder::new();
1617        let mut unresolvable_domain_refs = 0usize;
1618        for result in ordered {
1619            let mut doctree = result.doctree.clone();
1620            let resolution = env_resolve::resolve_document(
1621                &resolver,
1622                &nitpick,
1623                &result.docname,
1624                &mut doctree,
1625                &result.document.source_path,
1626            );
1627            unresolvable_domain_refs += resolution.unresolvable_domain_refs;
1628            for warning in resolution.warnings {
1629                self.add_warning(warning);
1630            }
1631
1632            // Sphinx feeds the search index from exactly this resolved
1633            // doctree (`write_doc_serialized` → `index_page`), with the
1634            // longtitle as plain text and the source-relative filename.
1635            let title = env
1636                .longtitles
1637                .get(&result.docname)
1638                .map(crate::doctree::Node::astext)
1639                .unwrap_or_default();
1640            let filename = result
1641                .document
1642                .source_path
1643                .strip_prefix(&self.source_dir)
1644                .unwrap_or(&result.document.source_path)
1645                .to_string_lossy()
1646                .into_owned();
1647            self.search_index.lock().unwrap().feed(
1648                &result.docname,
1649                &filename,
1650                &title,
1651                &doctree,
1652                env,
1653            );
1654
1655            self.resolved
1656                .lock()
1657                .unwrap()
1658                .insert(result.docname.clone(), doctree.root.pformat());
1659        }
1660
1661        if unresolvable_domain_refs > 0 {
1662            // References into domains this build has no resolver for —
1663            // `refdomain` outside `{"", "std", "py"}` (`c:`, `cpp:`, `js:`,
1664            // ...) — counted by the resolver rather than warned about.
1665            info!(
1666                "{unresolvable_domain_refs} cross-domain reference(s) not validated \
1667                 (domain not implemented until M5)"
1668            );
1669        }
1670    }
1671
1672    /// Section and figure numbering (`TocTreeCollector.get_updated_docs`,
1673    /// `collectors/toctree.py:194`), run in that order: figure numbers are
1674    /// scoped by the section numbers the first pass assigns.
1675    ///
1676    /// The doctree loader hands the walks whatever this build already has in
1677    /// memory — every read result carries its doctree, including the ones a
1678    /// warm cache hit loaded from disk — and falls back to the persisted
1679    /// doctree for anything else.
1680    ///
1681    /// The returned docnames (Sphinx's `rewrite_needed`) are the documents
1682    /// whose numbering moved, which Sphinx adds to its write set. They are
1683    /// logged rather than consumed here because this builder's write set is
1684    /// already every found document (see [`Self::write_phase`]) — a
1685    /// superset — so there is nothing left for them to widen.
1686    fn number_phase(&self, env: &mut BuildEnvironment, results: &[ReadResult]) {
1687        let in_memory: HashMap<&str, &Doctree> = results
1688            .iter()
1689            .map(|result| (result.docname.as_str(), &result.doctree))
1690            .collect();
1691        let load_doctree = |docname: &str| -> Option<std::borrow::Cow<'_, Doctree>> {
1692            match in_memory.get(docname) {
1693                Some(doctree) => Some(std::borrow::Cow::Borrowed(*doctree)),
1694                None => self.load_doctree(docname).map(std::borrow::Cow::Owned),
1695            }
1696        };
1697
1698        let sections = env_numbers::assign_section_numbers(env, &load_doctree);
1699        for warning in sections.warnings {
1700            self.report_numbering_warning(&warning, results);
1701        }
1702        let figures = env_numbers::assign_figure_numbers(
1703            env,
1704            self.config.numfig,
1705            self.config.numfig_secnum_depth,
1706            &load_doctree,
1707        );
1708        debug!(
1709            "Numbering: {} document(s) with changed section numbers, {} with changed figure numbers",
1710            sections.changed.len(),
1711            figures.len()
1712        );
1713    }
1714
1715    /// Surface one numbering diagnostic at the location Sphinx logs it —
1716    /// the `(source, line)` of the `toctree` node it names
1717    /// (`location=toctreenode`), which the parse record for that
1718    /// document's Nth toctree carries; a toctree spliced in from an
1719    /// included file names that file through the doctree's source table.
1720    fn report_numbering_warning(
1721        &self,
1722        warning: &env_numbers::NumberingWarning,
1723        results: &[ReadResult],
1724    ) {
1725        let result = results
1726            .iter()
1727            .find(|result| result.docname == warning.docname);
1728        let document = result.map(|result| &result.document);
1729        let toctree = document.and_then(|document| document.toctrees.get(warning.toctree_index));
1730        let source = result
1731            .and_then(|result| {
1732                let toctree = toctree?;
1733                result
1734                    .doctree
1735                    .sources
1736                    .get(toctree.source as usize)
1737                    .map(PathBuf::from)
1738            })
1739            .or_else(|| document.map(|document| document.source_path.clone()))
1740            .unwrap_or_else(|| PathBuf::from(&warning.docname));
1741        let line = toctree.map(|toctree| toctree.line as usize);
1742        self.warnings.lock().unwrap().push(
1743            BuildWarning::new(
1744                source,
1745                line,
1746                warning.message.clone(),
1747                WarningType::MissingToctreeRef,
1748            )
1749            .with_category(warning.category.clone()),
1750        );
1751    }
1752
1753    /// Write phase: emit every document's rendered output, in parallel and
1754    /// after resolution, so that a page can be written with whole-project
1755    /// knowledge (numbering, relations) once those exist.
1756    ///
1757    /// **Every found document is written, not just the ones this build
1758    /// read.** Sphinx writes the read set plus the toctree containers of
1759    /// what changed plus the documents whose numbering moved
1760    /// (`builders/__init__.py:717-736`), because its HTML builder also
1761    /// compares each output file against its sources and can tell that the
1762    /// rest are already on disk and current. This builder cannot do that
1763    /// yet, so it writes the superset — which is also what makes a cache
1764    /// hit still produce a page, rather than leaving a hole in the output
1765    /// tree where an unchanged document should be.
1766    ///
1767    /// One page failing to write must not abort the build (the same rule the
1768    /// read phase follows): it becomes a `BuildErrorReport`, and a non-zero
1769    /// exit, while the remaining pages are still written.
1770    fn write_phase(&self, documents: &[Document]) {
1771        documents.par_iter().for_each(|document| {
1772            if let Err(e) = self.write_one(document) {
1773                self.errors.lock().unwrap().push(BuildErrorReport::new(
1774                    document.source_path.clone(),
1775                    None,
1776                    format!("{e:#}"),
1777                    ErrorType::Other,
1778                ));
1779            }
1780        });
1781    }
1782
1783    fn write_one(&self, document: &Document) -> Result<()> {
1784        let output_path = self.get_output_path(&document.source_path)?;
1785        if let Some(parent) = output_path.parent() {
1786            std::fs::create_dir_all(parent)?;
1787        }
1788        std::fs::write(&output_path, &document.html)?;
1789        Ok(())
1790    }
1791
1792    /// `<cache_dir>/doctrees/<blake3(docname)>.doctree`.
1793    fn doctree_path(&self, docname: &str) -> PathBuf {
1794        let hash = blake3::hash(docname.as_bytes());
1795        self.cache
1796            .cache_dir()
1797            .join(DOCTREE_SUBDIR)
1798            .join(format!("{}.doctree", hash.to_hex()))
1799    }
1800
1801    /// Persist one document's doctree, behind the
1802    /// [`DOCTREE_MAGIC`]/[`DOCTREE_FORMAT_VERSION`] header that lets a later
1803    /// build tell whether the bytes are still readable *by meaning*, not
1804    /// just by bincode.
1805    fn store_doctree(&self, docname: &str, doctree: &Doctree) -> Result<()> {
1806        let path = self.doctree_path(docname);
1807        if let Some(parent) = path.parent() {
1808            std::fs::create_dir_all(parent)?;
1809        }
1810        let blob = crate::doctree::to_bincode(doctree);
1811        let mut bytes = Vec::with_capacity(DOCTREE_HEADER_LEN + blob.len());
1812        bytes.extend_from_slice(DOCTREE_MAGIC);
1813        bytes.extend_from_slice(&DOCTREE_FORMAT_VERSION.to_le_bytes());
1814        bytes.extend_from_slice(&blob);
1815        std::fs::write(path, bytes)?;
1816        Ok(())
1817    }
1818
1819    /// The persisted doctree for `docname`, or `None` if it is missing,
1820    /// carries another format version (including none at all — a file
1821    /// written before the header existed), or cannot be decoded (a
1822    /// truncated write). Every `None` means the same thing to callers: this
1823    /// document has to be read again.
1824    fn load_doctree(&self, docname: &str) -> Option<Doctree> {
1825        let path = self.doctree_path(docname);
1826        let bytes = std::fs::read(&path).ok()?;
1827        let Some(blob) = current_format_doctree(&bytes) else {
1828            debug!(
1829                "Ignoring doctree {} written in another format (re-reading {docname})",
1830                path.display()
1831            );
1832            return None;
1833        };
1834        match crate::doctree::from_bincode(blob) {
1835            Ok(doctree) => Some(doctree),
1836            Err(e) => {
1837                debug!(
1838                    "Ignoring unreadable doctree {}: {e:#} (re-reading {docname})",
1839                    path.display()
1840                );
1841                None
1842            }
1843        }
1844    }
1845
1846    fn get_output_path(&self, source_path: &Path) -> Result<PathBuf> {
1847        let relative_path = source_path.strip_prefix(&self.source_dir)?;
1848        let mut output_path = self.output_dir.join(relative_path);
1849
1850        // Change extension to .html
1851        output_path.set_extension("html");
1852
1853        Ok(output_path)
1854    }
1855
1856    async fn generate_indices(&self, _documents: &[Document]) -> Result<()> {
1857        info!("Generating indices and cross-references");
1858
1859        // The same `get_objects()` rows the search index lists, through the
1860        // same writer the direct `HTMLBuilder` uses.
1861        let domains = env::objects::domain_objects(&self.env);
1862        InventoryFile::dump(
1863            self.output_dir.join("objects.inv"),
1864            &self.config.project,
1865            self.config.version.as_deref().unwrap_or(""),
1866            &domains,
1867            // `StandaloneHTMLBuilder.get_target_uri`: `quote(docname) + link_suffix`.
1868            |docname| format!("{}.html", utils::quote_url_path(docname)),
1869        )
1870        .await
1871    }
1872
1873    async fn copy_static_assets(&self) -> Result<()> {
1874        info!("Copying static assets");
1875
1876        // Create _static directory
1877        let static_output_dir = self.output_dir.join("_static");
1878        tokio::fs::create_dir_all(&static_output_dir).await?;
1879
1880        // Copy built-in static assets - use relative path from binary location
1881        let exe_dir = std::env::current_exe()?
1882            .parent()
1883            .ok_or_else(|| anyhow::anyhow!("Could not determine executable directory"))?
1884            .to_path_buf();
1885
1886        // Try multiple possible locations for static assets
1887        let possible_static_dirs = [
1888            exe_dir.join("../static"),                      // Release build
1889            exe_dir.join("../../static"),                   // Debug build
1890            exe_dir.join("../../../static"),                // Deep build
1891            Path::new("rust-builder/static").to_path_buf(), // Local development
1892        ];
1893
1894        let mut static_assets_copied = false;
1895        for builtin_static_dir in &possible_static_dirs {
1896            if builtin_static_dir.exists() {
1897                debug!("Found static assets at: {:?}", builtin_static_dir);
1898                for entry in std::fs::read_dir(builtin_static_dir)? {
1899                    let entry = entry?;
1900                    let file_path = entry.path();
1901                    if file_path.is_file() {
1902                        let file_name = file_path.file_name().unwrap();
1903                        let dest_path = static_output_dir.join(file_name);
1904                        tokio::fs::copy(&file_path, &dest_path).await?;
1905                        debug!("Copied static asset: {:?}", file_name);
1906                    }
1907                }
1908                static_assets_copied = true;
1909                break;
1910            }
1911        }
1912
1913        if !static_assets_copied {
1914            debug!("No built-in static assets found, creating basic ones");
1915            // Create minimal CSS files if not found
1916            self.create_default_static_assets(&static_output_dir)
1917                .await?;
1918        }
1919
1920        // Copy project-specific static assets
1921        let static_dirs = [
1922            self.source_dir.join("_static"),
1923            self.source_dir.join("_templates"),
1924        ];
1925
1926        for static_dir in &static_dirs {
1927            if static_dir.exists() {
1928                let dest = self.output_dir.join(static_dir.file_name().unwrap());
1929                utils::copy_dir_recursive(static_dir, &dest).await?;
1930                debug!("Copied static directory: {:?}", static_dir);
1931            }
1932        }
1933
1934        Ok(())
1935    }
1936
1937    async fn create_default_static_assets(&self, static_dir: &Path) -> Result<()> {
1938        // Create basic pygments.css
1939        let pygments_css = include_str!("../static/pygments.css");
1940        tokio::fs::write(static_dir.join("pygments.css"), pygments_css).await?;
1941
1942        // Create basic theme.css
1943        let theme_css = include_str!("../static/theme.css");
1944        tokio::fs::write(static_dir.join("theme.css"), theme_css).await?;
1945
1946        // Create basic JavaScript files
1947        let jquery_js = include_str!("../static/jquery.js");
1948        tokio::fs::write(static_dir.join("jquery.js"), jquery_js).await?;
1949
1950        let doctools_js = include_str!("../static/doctools.js");
1951        tokio::fs::write(static_dir.join("doctools.js"), doctools_js).await?;
1952
1953        let sphinx_highlight_js = include_str!("../static/sphinx_highlight.js");
1954        tokio::fs::write(static_dir.join("sphinx_highlight.js"), sphinx_highlight_js).await?;
1955
1956        debug!("Created default static assets");
1957        Ok(())
1958    }
1959
1960    /// Root-relative docname (no extension, forward slashes) for a source
1961    /// path.
1962    fn docname_of_path(&self, path: &Path) -> String {
1963        let relative = path.strip_prefix(&self.source_dir).unwrap_or(path);
1964        relative
1965            .with_extension("")
1966            .to_string_lossy()
1967            .replace('\\', "/")
1968    }
1969
1970    /// Run the directive/role validation system over every RST document.
1971    ///
1972    /// Findings surface as build *warnings* (so `-W`/`-w` govern promotion);
1973    /// `Unknown` results stay silent — the built-in validators cover a
1974    /// fraction of real Sphinx, and reporting the rest would drown every
1975    /// real project in noise.
1976    fn validate_directives_and_roles(&self, processed_docs: &[Document], doctrees: &[Doctree]) {
1977        use crate::directives::validation::{
1978            DirectiveValidationResult, DirectiveValidationSystem, ParsedDirective, ParsedRole,
1979            RoleValidationResult, SourceLocation,
1980        };
1981        use crate::document::DocumentContent;
1982
1983        debug_assert_eq!(processed_docs.len(), doctrees.len());
1984        let results: Vec<(Vec<BuildWarning>, usize)> = processed_docs
1985            .par_iter()
1986            .zip(doctrees.par_iter())
1987            .filter_map(|(doc, doctree)| {
1988                if !matches!(&doc.content, DocumentContent::RestructuredText(_)) {
1989                    return None;
1990                }
1991
1992                let mut warnings = Vec::new();
1993                let mut unknown = 0usize;
1994                // Statistics make validate_* take &mut self, so each document
1995                // gets its own (cheap) system instance for the parallel pass.
1996                let mut system = DirectiveValidationSystem::new();
1997                // Since wave 3 the feed comes from the parse-time records
1998                // (M1-scanner-compatible tuples), not a raw re-scan. Each
1999                // record's `line` is numbered within its own source, so the
2000                // path comes from the doctree's source table: a directive
2001                // inside an included file is reported against that file.
2002                let file_of = |source: u16| -> String {
2003                    doctree
2004                        .sources
2005                        .get(source as usize)
2006                        .cloned()
2007                        .unwrap_or_else(|| doc.source_path.display().to_string())
2008                };
2009                let directives: Vec<ParsedDirective> = doc
2010                    .directive_records
2011                    .iter()
2012                    .map(|r| ParsedDirective {
2013                        name: r.name.clone(),
2014                        arguments: r.arguments.clone(),
2015                        options: r.options.iter().cloned().collect(),
2016                        content: r.content.clone(),
2017                        location: SourceLocation {
2018                            file: file_of(r.source),
2019                            line: r.line as usize,
2020                            column: 0,
2021                        },
2022                    })
2023                    .collect();
2024                let roles: Vec<ParsedRole> = doc
2025                    .role_records
2026                    .iter()
2027                    .map(|r| ParsedRole {
2028                        name: r.name.clone(),
2029                        target: r.target.clone(),
2030                        display_text: r.display.clone(),
2031                        location: SourceLocation {
2032                            file: file_of(r.source),
2033                            line: r.line as usize,
2034                            column: 0,
2035                        },
2036                    })
2037                    .collect();
2038
2039                for directive in &directives {
2040                    match system.validate_directive(directive) {
2041                        DirectiveValidationResult::Valid => {}
2042                        DirectiveValidationResult::Unknown => unknown += 1,
2043                        DirectiveValidationResult::Warning(msg)
2044                        | DirectiveValidationResult::Error(msg) => {
2045                            warnings.push(BuildWarning::new(
2046                                PathBuf::from(&directive.location.file),
2047                                Some(directive.location.line),
2048                                msg,
2049                                crate::error::WarningType::Other,
2050                            ));
2051                        }
2052                    }
2053                }
2054
2055                for role in &roles {
2056                    match system.validate_role(role) {
2057                        RoleValidationResult::Valid => {}
2058                        RoleValidationResult::Unknown => unknown += 1,
2059                        RoleValidationResult::Warning(msg) | RoleValidationResult::Error(msg) => {
2060                            warnings.push(BuildWarning::new(
2061                                PathBuf::from(&role.location.file),
2062                                Some(role.location.line),
2063                                msg,
2064                                crate::error::WarningType::Other,
2065                            ));
2066                        }
2067                    }
2068                }
2069
2070                Some((warnings, unknown))
2071            })
2072            .collect();
2073
2074        let mut unknown_total = 0usize;
2075        for (warnings, unknown) in results {
2076            unknown_total += unknown;
2077            for warning in warnings {
2078                self.add_warning(warning);
2079            }
2080        }
2081        if unknown_total > 0 {
2082            debug!(
2083                "{} directive/role occurrence(s) had no validator and were not checked",
2084                unknown_total
2085            );
2086        }
2087    }
2088
2089    async fn generate_search_index(&self) -> Result<()> {
2090        info!("Generating search index");
2091        let value = self.search_index.lock().unwrap().freeze(&self.env);
2092        let contents = crate::search::dumps_sphinx_index(&value)?;
2093        tokio::fs::write(self.output_dir.join("searchindex.js"), contents).await?;
2094        Ok(())
2095    }
2096
2097    /// The environment this build produced (empty before [`Self::build`]).
2098    pub fn env(&self) -> &BuildEnvironment {
2099        &self.env
2100    }
2101
2102    /// [`BuildEnvironment::snapshot`] of this build's environment — the
2103    /// shape the `env_differential` oracle compares against — plus the
2104    /// `resolved_pformat` of every document this build resolved, which is
2105    /// build output rather than environment state and so has no home inside
2106    /// [`BuildEnvironment`].
2107    pub fn snapshot_env(&self) -> serde_json::Value {
2108        let mut snapshot = self.env.snapshot();
2109        let resolved: serde_json::Map<String, serde_json::Value> = self
2110            .resolved
2111            .lock()
2112            .unwrap()
2113            .iter()
2114            .map(|(docname, pformat)| (docname.clone(), serde_json::Value::String(pformat.clone())))
2115            .collect();
2116        if let Some(object) = snapshot.as_object_mut() {
2117            object.insert(
2118                "resolved_pformat".to_string(),
2119                serde_json::Value::Object(resolved),
2120            );
2121            object.insert(
2122                "genindex".to_string(),
2123                env_genindex::snapshot(&self.genindex.lock().unwrap()),
2124            );
2125            object.insert(
2126                "py_modindex".to_string(),
2127                env_py_domain::modindex_snapshot(&self.py_modindex.lock().unwrap()),
2128            );
2129        }
2130        snapshot
2131    }
2132}
2133
2134/// The bincode blob inside a persisted doctree file, or `None` if the file
2135/// does not start with this build's [`DOCTREE_MAGIC`] +
2136/// [`DOCTREE_FORMAT_VERSION`] header.
2137fn current_format_doctree(bytes: &[u8]) -> Option<&[u8]> {
2138    let (header, blob) = bytes.split_at_checked(DOCTREE_HEADER_LEN)?;
2139    if &header[..DOCTREE_MAGIC.len()] != DOCTREE_MAGIC {
2140        return None;
2141    }
2142    let version = u32::from_le_bytes(header[DOCTREE_MAGIC.len()..].try_into().ok()?);
2143    (version == DOCTREE_FORMAT_VERSION).then_some(blob)
2144}
2145
2146/// A file's modification time in microseconds since the Unix epoch — the
2147/// unit `env.all_docs` read times are in, so the two are directly
2148/// comparable (Sphinx's `_StrPath._last_modified_time`).
2149///
2150/// `None` when the file cannot be stat-ed, which is Sphinx's `OSError`
2151/// path: the caller treats it as "this document is outdated".
2152fn modified_us(path: &Path) -> Option<u64> {
2153    let modified = std::fs::metadata(path).ok()?.modified().ok()?;
2154    Some(
2155        modified
2156            .duration_since(UNIX_EPOCH)
2157            .map(|since| since.as_micros() as u64)
2158            .unwrap_or(0),
2159    )
2160}
2161
2162/// Wall-clock microseconds since the Unix epoch, the unit Sphinx stores in
2163/// `env.all_docs` (`time.time_ns() // 1_000`). A pre-epoch clock yields 0
2164/// rather than wrapping.
2165fn now_micros() -> u64 {
2166    SystemTime::now()
2167        .duration_since(UNIX_EPOCH)
2168        .map(|d| d.as_micros() as u64)
2169        .unwrap_or(0)
2170}
2171
2172#[cfg(test)]
2173mod tests {
2174    use super::*;
2175    use tempfile::TempDir;
2176
2177    fn write_project(source_dir: &Path) {
2178        std::fs::create_dir_all(source_dir).unwrap();
2179        std::fs::write(
2180            source_dir.join("index.rst"),
2181            "Index\n=====\n\n.. toctree::\n\n   a\n",
2182        )
2183        .unwrap();
2184        std::fs::write(source_dir.join("a.rst"), "A\n=\n\nBody.\n").unwrap();
2185    }
2186
2187    fn build_incrementally(source_dir: &Path, output_dir: &Path) -> (BuildStats, SphinxBuilder) {
2188        let mut builder = SphinxBuilder::new(
2189            BuildConfig::default(),
2190            source_dir.to_path_buf(),
2191            output_dir.to_path_buf(),
2192        )
2193        .unwrap();
2194        builder.enable_incremental();
2195        let stats = tokio::runtime::Builder::new_current_thread()
2196            .enable_all()
2197            .build()
2198            .unwrap()
2199            .block_on(builder.build())
2200            .unwrap();
2201        (stats, builder)
2202    }
2203
2204    /// `-W` and `-n` steer diagnostics, not content: toggling either must
2205    /// leave the fingerprint — and therefore the whole cache directory —
2206    /// alone. Sphinx cannot invalidate on either (`config.py:272` gives
2207    /// `nitpicky` rebuild class `''`; `warningiserror` is not a `Config`
2208    /// value), so neither may invalidate here.
2209    #[test]
2210    fn operational_flags_are_excluded_from_the_fingerprint() {
2211        let base = BuildConfig::default();
2212        let baseline = config_fingerprint(&base).unwrap();
2213
2214        let mut warned = base.clone();
2215        warned.fail_on_warning = true;
2216        assert_eq!(config_fingerprint(&warned).unwrap(), baseline, "-W");
2217
2218        let mut nitpicky = base.clone();
2219        nitpicky.nitpicky = true;
2220        assert_eq!(config_fingerprint(&nitpicky).unwrap(), baseline, "-n");
2221
2222        let mut both = base.clone();
2223        both.fail_on_warning = true;
2224        both.nitpicky = true;
2225        assert_eq!(config_fingerprint(&both).unwrap(), baseline, "-W -n");
2226    }
2227
2228    /// The filter is exactly two keys wide. Everything else — `tags`
2229    /// included, because tags select `only::` branches and so change parse
2230    /// output — still invalidates.
2231    #[test]
2232    fn content_bearing_config_still_changes_the_fingerprint() {
2233        let base = BuildConfig::default();
2234        let baseline = config_fingerprint(&base).unwrap();
2235
2236        let mut tagged = base.clone();
2237        tagged.tags = vec!["draft".to_string()];
2238        assert_ne!(config_fingerprint(&tagged).unwrap(), baseline, "tags");
2239
2240        // `source_encoding` is rebuild class `'env'` (`config.py:244`): a
2241        // change must re-read every document.
2242        let mut encoded = base.clone();
2243        encoded.source_encoding = "latin-1".to_string();
2244        assert_ne!(
2245            config_fingerprint(&encoded).unwrap(),
2246            baseline,
2247            "source_encoding"
2248        );
2249
2250        // ...while the config-inited diagnostic record is not configuration
2251        // at all, and must not invalidate anything.
2252        let mut mismatched = base.clone();
2253        mismatched.note_confval_type_mismatch("maximum_signature_line_length", "str");
2254        assert_eq!(
2255            config_fingerprint(&mismatched).unwrap(),
2256            baseline,
2257            "confval_type_mismatches"
2258        );
2259
2260        let mut nitpick_ignore = base.clone();
2261        nitpick_ignore.nitpick_ignore = vec![("ref".to_string(), "x".to_string())];
2262        assert_ne!(
2263            config_fingerprint(&nitpick_ignore).unwrap(),
2264            baseline,
2265            "nitpick_ignore is data, not an operational flag"
2266        );
2267
2268        let mut numfig = base.clone();
2269        numfig.numfig = true;
2270        assert_ne!(config_fingerprint(&numfig).unwrap(), baseline, "numfig");
2271    }
2272
2273    /// Serializing through `serde_json::Value` sorts every map, so a
2274    /// multi-key `html_context` hashes the same on every call — the
2275    /// property that keeps the cache from being wiped on every build.
2276    #[test]
2277    fn multi_key_html_context_fingerprints_stably() {
2278        let mut config = BuildConfig::default();
2279        for key in ["a", "b", "c", "d", "e", "f", "g", "h"] {
2280            config.html_context.insert(
2281                key.to_string(),
2282                serde_json::Value::String(key.to_uppercase()),
2283            );
2284        }
2285
2286        let first = config_fingerprint(&config).unwrap();
2287        for _ in 0..8 {
2288            assert_eq!(config_fingerprint(&config).unwrap(), first);
2289        }
2290
2291        // ...and it is still sensitive to the contents.
2292        let mut changed = config.clone();
2293        changed
2294            .html_context
2295            .insert("a".to_string(), serde_json::Value::String("Z".to_string()));
2296        assert_ne!(config_fingerprint(&changed).unwrap(), first);
2297    }
2298
2299    /// Row 9 of the include checklist: the parse-time records replay
2300    /// into `env.included`/`env.dependencies` on merge, and the orphan
2301    /// warning consults `env.included` (src/env/toctree.rs check) — a doc
2302    /// reachable only through an `include` stays silent while a genuinely
2303    /// unlinked one still warns.
2304    #[test]
2305    fn include_records_replay_into_the_environment_and_suppress_the_orphan() {
2306        let tmp = TempDir::new().unwrap();
2307        let source_dir = tmp.path().join("source");
2308        let output_dir = tmp.path().join("build");
2309        std::fs::create_dir_all(&source_dir).unwrap();
2310        std::fs::write(
2311            source_dir.join("index.rst"),
2312            "Index\n=====\n\n.. toctree::\n\n   a\n",
2313        )
2314        .unwrap();
2315        std::fs::write(
2316            source_dir.join("a.rst"),
2317            "A\n=\n\n.. include:: part.rst\n\n.. include:: snippet.txt\n",
2318        )
2319        .unwrap();
2320        std::fs::write(source_dir.join("part.rst"), "part para\n").unwrap();
2321        std::fs::write(source_dir.join("snippet.txt"), "plain snippet\n").unwrap();
2322        std::fs::write(
2323            source_dir.join("not_linked.rst"),
2324            "Not Linked\n==========\n\nOrphan candidate.\n",
2325        )
2326        .unwrap();
2327
2328        let (stats, builder) = build_incrementally(&source_dir, &output_dir);
2329
2330        // Canonicalized like the builder's own source_dir.
2331        let src = crate::utils::canonicalize_simplified(&source_dir).unwrap();
2332        assert_eq!(
2333            builder.env.included.get("a"),
2334            Some(&std::collections::BTreeSet::from(["part".to_string()])),
2335            "only the docname-mapping include registers (snippet.txt maps to no docname)"
2336        );
2337        assert_eq!(
2338            builder
2339                .env
2340                .dependencies
2341                .get("a")
2342                .map(|set| set.iter().cloned().collect::<Vec<_>>()),
2343            Some(vec![src.join("part.rst"), src.join("snippet.txt")]),
2344            "every opened include target is a dependency, the non-doc file too"
2345        );
2346
2347        let orphan_warnings: Vec<String> = stats
2348            .warning_details
2349            .iter()
2350            .filter(|w| w.message.contains("isn't included in any toctree"))
2351            .map(|w| w.file.display().to_string())
2352            .collect();
2353        assert_eq!(
2354            orphan_warnings.len(),
2355            1,
2356            "exactly the genuinely unlinked doc warns: {orphan_warnings:?}"
2357        );
2358        assert!(
2359            orphan_warnings[0].ends_with("not_linked.rst"),
2360            "{orphan_warnings:?}"
2361        );
2362    }
2363
2364    #[test]
2365    fn every_read_document_persists_its_doctree() {
2366        let tmp = TempDir::new().unwrap();
2367        let source_dir = tmp.path().join("source");
2368        let output_dir = tmp.path().join("build");
2369        write_project(&source_dir);
2370
2371        let (_stats, builder) = build_incrementally(&source_dir, &output_dir);
2372
2373        for docname in ["index", "a"] {
2374            let path = builder.doctree_path(docname);
2375            assert!(
2376                path.is_file(),
2377                "{docname}: no doctree at {}",
2378                path.display()
2379            );
2380            let doctree = builder.load_doctree(docname).expect("doctree decodes");
2381            assert_eq!(doctree.root.kind, crate::doctree::kinds::DOCUMENT);
2382        }
2383        assert!(
2384            builder.cache.cache_dir().join("env.bin").is_file(),
2385            "the resolve phase must persist the environment"
2386        );
2387    }
2388
2389    #[test]
2390    fn cache_hit_whose_doctree_is_missing_is_treated_as_a_miss() {
2391        let tmp = TempDir::new().unwrap();
2392        let source_dir = tmp.path().join("source");
2393        let output_dir = tmp.path().join("build");
2394        write_project(&source_dir);
2395
2396        let (first, builder) = build_incrementally(&source_dir, &output_dir);
2397        assert_eq!(first.cache_hits, 0, "cold build cannot hit the cache");
2398
2399        // Warm: both documents come from the cache.
2400        let (warm, _) = build_incrementally(&source_dir, &output_dir);
2401        assert_eq!(warm.cache_hits, 2);
2402
2403        // Delete one document's doctree. Its cache entry is still valid, but
2404        // unusable — the build must re-read the file rather than pretend.
2405        std::fs::remove_file(builder.doctree_path("a")).unwrap();
2406        let (degraded, rebuilt) = build_incrementally(&source_dir, &output_dir);
2407        assert_eq!(
2408            degraded.cache_hits, 1,
2409            "a document whose doctree is gone is a cache miss, not a hit"
2410        );
2411        assert!(
2412            rebuilt.doctree_path("a").is_file(),
2413            "the re-read must persist the doctree it just produced"
2414        );
2415        assert_eq!(degraded.errors, 0);
2416
2417        // And the environment is complete either way.
2418        let env = rebuilt.env();
2419        assert_eq!(env.all_docs.len(), 2);
2420        assert!(env.tocs.contains_key("a"));
2421    }
2422
2423    #[test]
2424    fn persisted_doctrees_carry_the_format_version_header() {
2425        let tmp = TempDir::new().unwrap();
2426        let source_dir = tmp.path().join("source");
2427        let output_dir = tmp.path().join("build");
2428        write_project(&source_dir);
2429
2430        let (_stats, builder) = build_incrementally(&source_dir, &output_dir);
2431
2432        let bytes = std::fs::read(builder.doctree_path("index")).unwrap();
2433        assert_eq!(
2434            &bytes[..DOCTREE_MAGIC.len()],
2435            DOCTREE_MAGIC,
2436            "a persisted doctree must be self-identifying"
2437        );
2438        let version = u32::from_le_bytes(
2439            bytes[DOCTREE_MAGIC.len()..DOCTREE_MAGIC.len() + 4]
2440                .try_into()
2441                .unwrap(),
2442        );
2443        assert_eq!(version, DOCTREE_FORMAT_VERSION);
2444        assert_eq!(
2445            builder.load_doctree("index").unwrap().root.kind,
2446            crate::doctree::kinds::DOCUMENT,
2447            "the header must not disturb the round trip"
2448        );
2449    }
2450
2451    #[test]
2452    fn a_doctree_written_in_the_unversioned_format_is_treated_as_a_miss() {
2453        let tmp = TempDir::new().unwrap();
2454        let source_dir = tmp.path().join("source");
2455        let output_dir = tmp.path().join("build");
2456        write_project(&source_dir);
2457
2458        let (_cold, builder) = build_incrementally(&source_dir, &output_dir);
2459
2460        // Exactly what the pre-versioning builder wrote: a bare bincode
2461        // blob. It still *decodes* — which is the trap: an old blob whose
2462        // attribute shapes have since changed decodes into a plausible
2463        // doctree and then mis-harvests. The version word is what makes it
2464        // a miss instead.
2465        let doctree = builder.load_doctree("index").expect("doctree decodes");
2466        std::fs::write(
2467            builder.doctree_path("index"),
2468            crate::doctree::to_bincode(&doctree),
2469        )
2470        .unwrap();
2471        assert!(
2472            builder.load_doctree("index").is_none(),
2473            "an unversioned blob must not be trusted"
2474        );
2475
2476        let (stats, rebuilt) = build_incrementally(&source_dir, &output_dir);
2477        assert_eq!(
2478            stats.cache_hits, 1,
2479            "the document whose doctree is stale must be re-read"
2480        );
2481        assert!(rebuilt.load_doctree("index").is_some());
2482    }
2483
2484    #[test]
2485    fn a_doctree_from_a_future_format_version_is_treated_as_a_miss() {
2486        let tmp = TempDir::new().unwrap();
2487        let source_dir = tmp.path().join("source");
2488        let output_dir = tmp.path().join("build");
2489        write_project(&source_dir);
2490
2491        let (_cold, builder) = build_incrementally(&source_dir, &output_dir);
2492        let doctree = builder.load_doctree("index").expect("doctree decodes");
2493
2494        let mut bytes = Vec::from(DOCTREE_MAGIC);
2495        bytes.extend_from_slice(&(DOCTREE_FORMAT_VERSION + 1).to_le_bytes());
2496        bytes.extend_from_slice(&crate::doctree::to_bincode(&doctree));
2497        std::fs::write(builder.doctree_path("index"), bytes).unwrap();
2498
2499        assert!(builder.load_doctree("index").is_none());
2500    }
2501
2502    #[test]
2503    fn corrupt_doctree_file_is_treated_as_a_miss() {
2504        let tmp = TempDir::new().unwrap();
2505        let source_dir = tmp.path().join("source");
2506        let output_dir = tmp.path().join("build");
2507        write_project(&source_dir);
2508
2509        let (_cold, builder) = build_incrementally(&source_dir, &output_dir);
2510        std::fs::write(builder.doctree_path("index"), b"not a doctree").unwrap();
2511
2512        let (stats, rebuilt) = build_incrementally(&source_dir, &output_dir);
2513        assert_eq!(stats.cache_hits, 1);
2514        assert_eq!(stats.errors, 0);
2515        assert!(rebuilt.load_doctree("index").is_some());
2516    }
2517
2518    /// The cache directory is optional infrastructure: a build whose
2519    /// environment cannot be written still produced valid output, and
2520    /// saying "build failed" over it would be a lie. (A directory where
2521    /// `env.bin` belongs is the portable way to make exactly that one write
2522    /// fail while every other cache write succeeds.)
2523    #[test]
2524    fn a_build_whose_environment_cannot_be_saved_still_writes_its_output() {
2525        let tmp = TempDir::new().unwrap();
2526        let source_dir = tmp.path().join("source");
2527        let output_dir = tmp.path().join("build");
2528        write_project(&source_dir);
2529
2530        let (_cold, builder) = build_incrementally(&source_dir, &output_dir);
2531        let env_file = builder.cache.cache_dir().join("env.bin");
2532        std::fs::remove_file(&env_file).unwrap();
2533        std::fs::create_dir(&env_file).unwrap();
2534
2535        let (stats, rebuilt) = build_incrementally(&source_dir, &output_dir);
2536
2537        assert_eq!(stats.errors, 0, "an unsaveable environment is not an error");
2538        assert!(
2539            output_dir.join("index.html").is_file() && output_dir.join("a.html").is_file(),
2540            "the pages this build produced are still written"
2541        );
2542        assert_eq!(
2543            rebuilt.env().all_docs.len(),
2544            2,
2545            "the in-memory environment is complete; only its persistence failed"
2546        );
2547    }
2548
2549    /// `-E`: the persisted environment is not to be trusted, and neither is
2550    /// the half of it already sitting in memory.
2551    #[test]
2552    fn fresh_env_discards_the_loaded_environment_and_re_reads_everything() {
2553        let tmp = TempDir::new().unwrap();
2554        let source_dir = tmp.path().join("source");
2555        let output_dir = tmp.path().join("build");
2556        write_project(&source_dir);
2557
2558        build_incrementally(&source_dir, &output_dir);
2559
2560        let mut builder = SphinxBuilder::new(
2561            BuildConfig::default(),
2562            source_dir.clone(),
2563            output_dir.clone(),
2564        )
2565        .unwrap();
2566        builder.enable_incremental();
2567        assert_eq!(
2568            builder.env().all_docs.len(),
2569            2,
2570            "the builder loads the saved environment"
2571        );
2572
2573        builder.fresh_env().unwrap();
2574        assert!(
2575            builder.env().all_docs.is_empty(),
2576            "-E starts from an empty environment, not the loaded one"
2577        );
2578
2579        let stats = tokio::runtime::Builder::new_current_thread()
2580            .enable_all()
2581            .build()
2582            .unwrap()
2583            .block_on(builder.build())
2584            .unwrap();
2585        assert_eq!(stats.cache_hits, 0, "every document is new again");
2586        assert_eq!(stats.files_skipped, 0);
2587        assert_eq!(builder.env().all_docs.len(), 2);
2588    }
2589
2590    /// A build with the document cache off reads every document — which is
2591    /// what `sphinx-build -a` maps to here.
2592    #[test]
2593    fn a_non_incremental_build_reads_every_document() {
2594        let tmp = TempDir::new().unwrap();
2595        let source_dir = tmp.path().join("source");
2596        let output_dir = tmp.path().join("build");
2597        write_project(&source_dir);
2598
2599        build_incrementally(&source_dir, &output_dir);
2600
2601        let mut builder = SphinxBuilder::new(
2602            BuildConfig::default(),
2603            source_dir.clone(),
2604            output_dir.clone(),
2605        )
2606        .unwrap();
2607        let stats = tokio::runtime::Builder::new_current_thread()
2608            .enable_all()
2609            .build()
2610            .unwrap()
2611            .block_on(builder.build())
2612            .unwrap();
2613
2614        assert_eq!(stats.cache_hits, 0);
2615        assert_eq!(stats.files_skipped, 0, "nothing was skipped: all was read");
2616        assert!(output_dir.join("index.html").is_file() && output_dir.join("a.html").is_file());
2617    }
2618
2619    #[test]
2620    fn cache_hit_still_writes_output_and_fills_the_environment() {
2621        let tmp = TempDir::new().unwrap();
2622        let source_dir = tmp.path().join("source");
2623        let output_dir = tmp.path().join("build");
2624        write_project(&source_dir);
2625
2626        let (_cold, _) = build_incrementally(&source_dir, &output_dir);
2627        std::fs::remove_file(output_dir.join("index.html")).unwrap();
2628        std::fs::remove_file(output_dir.join("a.html")).unwrap();
2629
2630        let (warm, builder) = build_incrementally(&source_dir, &output_dir);
2631
2632        assert_eq!(warm.cache_hits, 2);
2633        assert!(output_dir.join("index.html").is_file());
2634        assert!(output_dir.join("a.html").is_file());
2635        assert_eq!(
2636            builder.env().toctree_includes.get("index"),
2637            Some(&vec!["a".to_string()]),
2638            "a fully cached build still rebuilds the environment from the \
2639             persisted doctrees"
2640        );
2641    }
2642
2643    /// The `config:` hash real `sphinx-build` 9.1.0 writes to `.buildinfo`
2644    /// for one-setting `conf.py` files, pinned here so the html-category
2645    /// table cannot drift from Sphinx's defaults or from the literal-vs-
2646    /// default rule. The `sphinx_rtd_theme` value is from
2647    /// `BuildInfo(app.config, app.tags, {'html'})` over the same config,
2648    /// because Sphinx cannot load that theme in the oracle environment.
2649    #[test]
2650    fn buildinfo_config_hash_matches_sphinx_for_configured_values() {
2651        let cases = [
2652            ("project = 'Fixture'\n", "bc9d5af5c9b6d45a25e31a5ff598283a"),
2653            (
2654                "project = 'Fixture'\nhtml_theme = 'alabaster'\n",
2655                "bc9d5af5c9b6d45a25e31a5ff598283a",
2656            ),
2657            (
2658                "project = 'Fixture'\nhtml_theme = 'classic'\n",
2659                "58dd99f3a17211a74d9b5a57b3e09abd",
2660            ),
2661            (
2662                "project = 'Fixture'\nhtml_theme = 'sphinx_rtd_theme'\n",
2663                "dbf973fabe50a41cd3aba49a63addfb0",
2664            ),
2665            (
2666                "project = 'Fixture'\nhtml_css_files = ['a.css']\n",
2667                "05a38004493ea6ed2731ff1f46a9dbea",
2668            ),
2669            (
2670                "project = 'Fixture'\nhtml_css_files = [('a.css', {'media': 'print'})]\n",
2671                "2baebae26e35eb22fb7aa2e2d1aa9597",
2672            ),
2673            (
2674                "project = 'Fixture'\nhtml_js_files = ['b.js']\n",
2675                "0e188cc551c4d963314c65091a73a387",
2676            ),
2677            (
2678                "project = 'Fixture'\nhtml_static_path = ['_static']\n",
2679                "199d55036ccc652911c8d7544403296b",
2680            ),
2681            (
2682                "project = 'Fixture'\ntemplates_path = ['_templates']\n",
2683                "dd8472423e919d99edb63ef88589b4d1",
2684            ),
2685            (
2686                "project = 'Fixture'\nhtml_theme_options = {'nosidebar': True}\n",
2687                "a2a03cb33b0b36892e5a3836146f8e60",
2688            ),
2689            (
2690                "project = 'Fixture'\nhtml_last_updated_fmt = '%Y'\n",
2691                "7106224b6b475e01e49bd6989dc2fa29",
2692            ),
2693            (
2694                "project = 'Fixture'\nhtml_use_opensearch = 'https://example.org'\n",
2695                "0acadda62c6d2f4f946d7a0fa145cd53",
2696            ),
2697            (
2698                "project = 'My.Project v2'\n",
2699                "ed09fc06b53dbc75b8ac7d282178aaab",
2700            ),
2701            (
2702                "project = 'Fixture'\nmodindex_common_prefix = ['probe.']\n",
2703                "0652c60ff2bdad1e6cafdc981c25594b",
2704            ),
2705            (
2706                "project = 'Fixture'\nhtml_logo = 'logo.png'\nhtml_favicon = 'fav.ico'\n\
2707                 html_title = 'Custom Title'\nhtml_short_title = 'Short'\n\
2708                 html_show_sphinx = False\nhtml_copy_source = False\n\
2709                 copyright = '2026, Me'\nversion = '1.2'\nrelease = '1.2.3'\n",
2710                "5b84a1185cee8c88c692fc47d88b91dc",
2711            ),
2712        ];
2713        let dir = tempfile::tempdir().unwrap();
2714        for (conf, expected) in cases {
2715            std::fs::write(dir.path().join("conf.py"), conf).unwrap();
2716            let parsed = crate::python_config::PythonConfigParser::new()
2717                .unwrap()
2718                .parse_conf_py(dir.path().join("conf.py"))
2719                .unwrap();
2720            let mut config = parsed.to_build_config().unwrap();
2721            // The driver's `config-inited` step: the probe directory holds
2722            // no logo or favicon file, so both settings fall back to `None`
2723            // exactly as they did in the oracle run.
2724            config.validate_html_logo_and_favicon(dir.path());
2725            let contents = sphinx_build_info_contents(&config, &config.tags).unwrap();
2726            let line = contents
2727                .lines()
2728                .find(|line| line.starts_with("config: "))
2729                .unwrap();
2730            assert_eq!(&line["config: ".len()..], expected, "{conf}");
2731        }
2732    }
2733
2734    #[test]
2735    fn make_filename_strips_like_sphinx() {
2736        assert_eq!(make_filename("My.Project v2"), "MyProjectv2");
2737        assert_eq!(make_filename("snake_case-ok"), "snake_case-ok");
2738        assert_eq!(make_filename("¡!"), "sphinx");
2739    }
2740}