soma_application/graduation/
recovery.rs

1use std::{fs, path::Path};
2
3use super::*;
4
5/// Restore the exact pre-operation provider files and graduation state.
6pub fn recover(workspace: &Path, provider_root: &Path) -> anyhow::Result<()> {
7    recover_before(
8        workspace,
9        provider_root,
10        std::time::Instant::now() + std::time::Duration::from_secs(30),
11    )
12}
13
14fn recover_before(
15    workspace: &Path,
16    provider_root: &Path,
17    deadline: std::time::Instant,
18) -> anyhow::Result<()> {
19    let _lock = WorkspaceLock::acquire_before(workspace, deadline)?;
20    recover_transaction(workspace, provider_root)
21}
22
23/// Recover interrupted transactions beneath an operator-owned graduation
24/// root before provider discovery observes partially promoted files.
25pub fn recover_all(root: &Path, provider_root: &Path) -> anyhow::Result<usize> {
26    if !root.exists() {
27        return Ok(0);
28    }
29    let root = root.canonicalize()?;
30    let mut recovered = 0;
31    let mut visited = 0;
32    let mut entries = 0;
33    let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30);
34    let mut pending = vec![(root.clone(), 0usize)];
35    while let Some((path, depth)) = pending.pop() {
36        if std::time::Instant::now() >= deadline {
37            anyhow::bail!("graduation recovery exceeded its global deadline");
38        }
39        visited += 1;
40        if visited > MAX_RECOVERY_DIRECTORIES {
41            anyhow::bail!(
42                "graduation recovery exceeds {MAX_RECOVERY_DIRECTORIES} directories beneath {}",
43                root.display()
44            );
45        }
46        let transaction = path.join(TRANSACTION_DIR);
47        match fs::symlink_metadata(&transaction) {
48            Ok(metadata) if metadata.file_type().is_symlink() => anyhow::bail!(
49                "graduation transaction directory must not be a symlink: {}",
50                transaction.display()
51            ),
52            Ok(metadata) if metadata.file_type().is_dir() => {
53                recover_before(&path, provider_root, deadline)?;
54                recovered += 1;
55            }
56            Ok(_) => anyhow::bail!(
57                "graduation transaction marker is not a directory: {}",
58                transaction.display()
59            ),
60            Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
61            Err(error) => return Err(error.into()),
62        }
63        if depth >= MAX_RECOVERY_DEPTH {
64            ensure_leaf(&path, &root, &mut entries)?;
65            continue;
66        }
67        for entry in fs::read_dir(&path)? {
68            let entry = entry?;
69            count_entry(&root, &mut entries)?;
70            if entry.file_name() == TRANSACTION_DIR {
71                continue;
72            }
73            if entry
74                .file_name()
75                .to_string_lossy()
76                .starts_with(".graduation-transaction-complete-")
77            {
78                let metadata = fs::symlink_metadata(entry.path())?;
79                if metadata.file_type().is_symlink() || !metadata.file_type().is_dir() {
80                    anyhow::bail!(
81                        "graduation transaction tombstone is invalid: {}",
82                        entry.path().display()
83                    );
84                }
85                let removed = remove_committed_tombstone(&path, provider_root, &entry.path())?;
86                entries = entries.saturating_add(removed);
87                if entries > MAX_RECOVERY_ENTRIES {
88                    anyhow::bail!(
89                        "graduation recovery exceeds {MAX_RECOVERY_ENTRIES} entries beneath {}",
90                        root.display()
91                    );
92                }
93                continue;
94            }
95            let metadata = fs::symlink_metadata(entry.path())?;
96            if !metadata.file_type().is_dir() || metadata.file_type().is_symlink() {
97                continue;
98            }
99            let child = entry.path().canonicalize()?;
100            if !child.starts_with(&root) {
101                anyhow::bail!(
102                    "graduation recovery directory escapes configured root: {}",
103                    child.display()
104                );
105            }
106            pending.push((child, depth + 1));
107        }
108    }
109    Ok(recovered)
110}
111
112fn ensure_leaf(path: &Path, root: &Path, entries: &mut usize) -> anyhow::Result<()> {
113    for entry in fs::read_dir(path)? {
114        let entry = entry?;
115        count_entry(root, entries)?;
116        if entry.file_name() == TRANSACTION_DIR {
117            continue;
118        }
119        let metadata = fs::symlink_metadata(entry.path())?;
120        if metadata.file_type().is_dir() && !metadata.file_type().is_symlink() {
121            anyhow::bail!(
122                "graduation recovery exceeds depth {MAX_RECOVERY_DEPTH} beneath {}",
123                root.display()
124            );
125        }
126    }
127    Ok(())
128}
129
130fn count_entry(root: &Path, entries: &mut usize) -> anyhow::Result<()> {
131    *entries += 1;
132    if *entries > MAX_RECOVERY_ENTRIES {
133        anyhow::bail!(
134            "graduation recovery exceeds {MAX_RECOVERY_ENTRIES} entries beneath {}",
135            root.display()
136        );
137    }
138    Ok(())
139}
soma_application/graduation/recovery.rs

soma_application/graduation/
recovery.rs
