soma_provider_adapters/python/
cache.rs

1//! Read-only inventory for content-addressed Python environment caches.
2//!
3//! Inventory never executes Python, invokes uv, follows symlinks, or mutates a
4//! cache entry. It classifies every visible entry so later prune and repair
5//! operations can make decisions from one shared model.
6
7use std::{
8    fs, io,
9    path::{Path, PathBuf},
10    time::UNIX_EPOCH,
11};
12
13use serde::Serialize;
14use sha2::{Digest, Sha256};
15use thiserror::Error;
16
17use super::{
18    environment::{PythonRuntimeFingerprint, PythonWheelTag},
19    materializer::{READY_FILE, READY_SCHEMA_VERSION, ReadyMarker},
20};
21
22#[path = "cache_prune.rs"]
23mod prune;
24pub use prune::{
25    PythonEnvironmentPruneCandidate, PythonEnvironmentPruneError, PythonEnvironmentPruneOutcome,
26    PythonEnvironmentPrunePlan, PythonEnvironmentPrunePolicy, PythonEnvironmentPruneReport,
27};
28
29#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
30#[serde(rename_all = "snake_case")]
31pub enum PythonEnvironmentCacheState {
32    Ready,
33    Incomplete,
34    Invalid,
35    Staging,
36}
37
38#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
39pub struct PythonEnvironmentCacheMetadata {
40    pub schema_version: u32,
41    pub environment_key: String,
42    pub plan_version: u32,
43    pub dependency_count: usize,
44    pub runtime: PythonRuntimeFingerprint,
45    pub sdk_wheel_tag: PythonWheelTag,
46    pub sdk_wheel_sha256: String,
47    pub uv_version: String,
48    pub lock_sha256: String,
49    pub provider_source_sha256: Option<String>,
50    pub input_plan_key: Option<String>,
51}
52
53impl From<ReadyMarker> for PythonEnvironmentCacheMetadata {
54    fn from(marker: ReadyMarker) -> Self {
55        Self {
56            schema_version: marker.schema_version,
57            environment_key: marker.environment_key,
58            plan_version: marker.plan_version,
59            dependency_count: marker.dependency_count,
60            runtime: marker.runtime,
61            sdk_wheel_tag: marker.sdk_wheel_tag,
62            sdk_wheel_sha256: marker.sdk_wheel_sha256,
63            uv_version: marker.uv_version,
64            lock_sha256: marker.lock_sha256,
65            provider_source_sha256: marker.provider_source_sha256,
66            input_plan_key: marker.input_plan_key,
67        }
68    }
69}
70
71#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
72pub struct PythonEnvironmentCacheEntry {
73    pub directory: PathBuf,
74    pub key: Option<String>,
75    pub plan_directory_version: Option<u32>,
76    pub state: PythonEnvironmentCacheState,
77    pub size_bytes: u64,
78    pub file_count: u64,
79    pub modified_unix_seconds: Option<u64>,
80    pub metadata: Option<PythonEnvironmentCacheMetadata>,
81    pub issue: Option<String>,
82}
83
84#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize)]
85pub struct PythonEnvironmentCacheSummary {
86    pub ready: usize,
87    pub incomplete: usize,
88    pub invalid: usize,
89    pub staging: usize,
90    pub total_size_bytes: u64,
91    pub total_file_count: u64,
92}
93
94#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
95pub struct PythonEnvironmentCacheInventory {
96    pub root: PathBuf,
97    pub entries: Vec<PythonEnvironmentCacheEntry>,
98    pub summary: PythonEnvironmentCacheSummary,
99}
100
101#[derive(Debug, Error)]
102pub enum PythonEnvironmentCacheError {
103    #[error("Python environment cache I/O failed at {}: {source}", path.display())]
104    Io {
105        path: PathBuf,
106        #[source]
107        source: io::Error,
108    },
109    #[error("Python environment cache root is not a real directory: {}", path.display())]
110    UnsafeRoot { path: PathBuf },
111}
112
113#[derive(Debug, Clone)]
114pub struct PythonEnvironmentCache {
115    root: PathBuf,
116}
117
118impl PythonEnvironmentCache {
119    pub fn new(cache_root: impl Into<PathBuf>) -> Self {
120        Self {
121            root: cache_root.into().join("python"),
122        }
123    }
124
125    pub fn root(&self) -> &Path {
126        &self.root
127    }
128
129    pub fn inventory(
130        &self,
131    ) -> Result<PythonEnvironmentCacheInventory, PythonEnvironmentCacheError> {
132        let root_metadata = match fs::symlink_metadata(&self.root) {
133            Ok(metadata) => metadata,
134            Err(error) if error.kind() == io::ErrorKind::NotFound => {
135                return Ok(PythonEnvironmentCacheInventory {
136                    root: self.root.clone(),
137                    entries: Vec::new(),
138                    summary: PythonEnvironmentCacheSummary::default(),
139                });
140            }
141            Err(source) => {
142                return Err(PythonEnvironmentCacheError::Io {
143                    path: self.root.clone(),
144                    source,
145                });
146            }
147        };
148        if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() {
149            return Err(PythonEnvironmentCacheError::UnsafeRoot {
150                path: self.root.clone(),
151            });
152        }
153
154        let mut version_directories = read_paths(&self.root)?;
155        version_directories.sort();
156        let mut entries = Vec::new();
157        for version_directory in version_directories {
158            let version_metadata = symlink_metadata(&version_directory)?;
159            if version_metadata.file_type().is_symlink() || !version_metadata.is_dir() {
160                entries.push(classify_non_directory(
161                    &version_directory,
162                    None,
163                    version_metadata,
164                ));
165                continue;
166            }
167            let plan_directory_version = version_directory
168                .file_name()
169                .and_then(|name| name.to_str())
170                .and_then(|name| name.strip_prefix('v'))
171                .and_then(|version| version.parse::<u32>().ok());
172            let mut children = read_paths(&version_directory)?;
173            children.sort();
174            for child in children {
175                entries.push(inspect_entry(&child, plan_directory_version)?);
176            }
177        }
178        entries.sort_by(|left, right| left.directory.cmp(&right.directory));
179        let summary = summarize(&entries);
180        Ok(PythonEnvironmentCacheInventory {
181            root: self.root.clone(),
182            entries,
183            summary,
184        })
185    }
186}
187
188fn inspect_entry(
189    directory: &Path,
190    plan_directory_version: Option<u32>,
191) -> Result<PythonEnvironmentCacheEntry, PythonEnvironmentCacheError> {
192    let metadata = symlink_metadata(directory)?;
193    if metadata.file_type().is_symlink() || !metadata.is_dir() {
194        return Ok(classify_non_directory(
195            directory,
196            plan_directory_version,
197            metadata,
198        ));
199    }
200    let stats = tree_stats(directory)?;
201    let name = directory
202        .file_name()
203        .and_then(|name| name.to_str())
204        .unwrap_or_default();
205    if name.starts_with('.')
206        && (name.contains(".tmp-")
207            || name.contains(".prune-")
208            || name.contains(".repair-")
209            || name.contains(".update-"))
210    {
211        let issue = if name.contains(".prune-") {
212            "temporary prune quarantine"
213        } else if name.contains(".repair-") {
214            "temporary repair quarantine"
215        } else if name.contains(".update-") {
216            "temporary update candidate"
217        } else {
218            "temporary materialization directory"
219        };
220        return Ok(entry(
221            directory,
222            None,
223            plan_directory_version,
224            PythonEnvironmentCacheState::Staging,
225            stats,
226            None,
227            Some(issue.to_owned()),
228        ));
229    }
230
231    let key = Some(name.to_owned());
232    let marker_path = directory.join(READY_FILE);
233    let marker_metadata = match fs::symlink_metadata(&marker_path) {
234        Ok(metadata) => metadata,
235        Err(error) if error.kind() == io::ErrorKind::NotFound => {
236            return Ok(entry(
237                directory,
238                key,
239                plan_directory_version,
240                PythonEnvironmentCacheState::Incomplete,
241                stats,
242                None,
243                Some("readiness marker is missing".to_owned()),
244            ));
245        }
246        Err(source) => {
247            return Err(PythonEnvironmentCacheError::Io {
248                path: marker_path,
249                source,
250            });
251        }
252    };
253    if marker_metadata.file_type().is_symlink() || !marker_metadata.is_file() {
254        return Ok(entry(
255            directory,
256            key,
257            plan_directory_version,
258            PythonEnvironmentCacheState::Invalid,
259            stats,
260            None,
261            Some("readiness marker is not a regular file".to_owned()),
262        ));
263    }
264    let marker_bytes =
265        fs::read(&marker_path).map_err(|source| PythonEnvironmentCacheError::Io {
266            path: marker_path,
267            source,
268        })?;
269    let marker: ReadyMarker = match serde_json::from_slice(&marker_bytes) {
270        Ok(marker) => marker,
271        Err(error) => {
272            return Ok(entry(
273                directory,
274                key,
275                plan_directory_version,
276                PythonEnvironmentCacheState::Invalid,
277                stats,
278                None,
279                Some(format!("readiness marker is invalid: {error}")),
280            ));
281        }
282    };
283    let cache_metadata = PythonEnvironmentCacheMetadata::from(marker);
284    let issue = validate_ready_entry(directory, name, plan_directory_version, &cache_metadata)?;
285    let state = if issue.is_some() {
286        PythonEnvironmentCacheState::Invalid
287    } else {
288        PythonEnvironmentCacheState::Ready
289    };
290    Ok(entry(
291        directory,
292        key,
293        plan_directory_version,
294        state,
295        stats,
296        Some(cache_metadata),
297        issue,
298    ))
299}
300
301fn validate_ready_entry(
302    directory: &Path,
303    directory_key: &str,
304    plan_directory_version: Option<u32>,
305    metadata: &PythonEnvironmentCacheMetadata,
306) -> Result<Option<String>, PythonEnvironmentCacheError> {
307    if metadata.schema_version != READY_SCHEMA_VERSION {
308        return Ok(Some(format!(
309            "unsupported readiness schema version {}; expected {READY_SCHEMA_VERSION}",
310            metadata.schema_version
311        )));
312    }
313    if metadata.environment_key != directory_key {
314        return Ok(Some(
315            "readiness marker key does not match directory name".to_owned(),
316        ));
317    }
318    if plan_directory_version != Some(metadata.plan_version) {
319        return Ok(Some(
320            "readiness marker plan version does not match version directory".to_owned(),
321        ));
322    }
323    let python = environment_python_path(directory);
324    let python_metadata = match fs::metadata(&python) {
325        Ok(metadata) => metadata,
326        Err(error) if error.kind() == io::ErrorKind::NotFound => {
327            return Ok(Some("prepared Python interpreter is missing".to_owned()));
328        }
329        Err(source) => {
330            return Err(PythonEnvironmentCacheError::Io {
331                path: python,
332                source,
333            });
334        }
335    };
336    if !python_metadata.is_file() {
337        return Ok(Some(
338            "prepared Python interpreter is not a regular file".to_owned(),
339        ));
340    }
341    let lockfile = directory.join("uv.lock");
342    let lock_metadata = match fs::symlink_metadata(&lockfile) {
343        Ok(metadata) => metadata,
344        Err(error) if error.kind() == io::ErrorKind::NotFound => {
345            return Ok(Some("uv.lock is missing".to_owned()));
346        }
347        Err(source) => {
348            return Err(PythonEnvironmentCacheError::Io {
349                path: lockfile,
350                source,
351            });
352        }
353    };
354    if lock_metadata.file_type().is_symlink() || !lock_metadata.is_file() {
355        return Ok(Some("uv.lock is not a regular file".to_owned()));
356    }
357    let lock = fs::read(&lockfile).map_err(|source| PythonEnvironmentCacheError::Io {
358        path: lockfile,
359        source,
360    })?;
361    if sha256_hex(&lock) != metadata.lock_sha256 {
362        return Ok(Some(
363            "uv.lock digest does not match readiness marker".to_owned(),
364        ));
365    }
366    Ok(None)
367}
368
369fn classify_non_directory(
370    path: &Path,
371    plan_directory_version: Option<u32>,
372    metadata: fs::Metadata,
373) -> PythonEnvironmentCacheEntry {
374    let file_type = if metadata.file_type().is_symlink() {
375        "symbolic link"
376    } else {
377        "non-directory entry"
378    };
379    entry(
380        path,
381        path.file_name()
382            .and_then(|name| name.to_str())
383            .map(str::to_owned),
384        plan_directory_version,
385        PythonEnvironmentCacheState::Invalid,
386        EntryStats::from_metadata(&metadata),
387        None,
388        Some(format!("cache entry is a {file_type}")),
389    )
390}
391
392fn entry(
393    directory: &Path,
394    key: Option<String>,
395    plan_directory_version: Option<u32>,
396    state: PythonEnvironmentCacheState,
397    stats: EntryStats,
398    metadata: Option<PythonEnvironmentCacheMetadata>,
399    issue: Option<String>,
400) -> PythonEnvironmentCacheEntry {
401    PythonEnvironmentCacheEntry {
402        directory: directory.to_path_buf(),
403        key,
404        plan_directory_version,
405        state,
406        size_bytes: stats.size_bytes,
407        file_count: stats.file_count,
408        modified_unix_seconds: stats.modified_unix_seconds,
409        metadata,
410        issue,
411    }
412}
413
414fn summarize(entries: &[PythonEnvironmentCacheEntry]) -> PythonEnvironmentCacheSummary {
415    let mut summary = PythonEnvironmentCacheSummary::default();
416    for entry in entries {
417        match entry.state {
418            PythonEnvironmentCacheState::Ready => summary.ready += 1,
419            PythonEnvironmentCacheState::Incomplete => summary.incomplete += 1,
420            PythonEnvironmentCacheState::Invalid => summary.invalid += 1,
421            PythonEnvironmentCacheState::Staging => summary.staging += 1,
422        }
423        summary.total_size_bytes = summary.total_size_bytes.saturating_add(entry.size_bytes);
424        summary.total_file_count = summary.total_file_count.saturating_add(entry.file_count);
425    }
426    summary
427}
428
429#[derive(Debug, Clone, Copy, Default)]
430struct EntryStats {
431    size_bytes: u64,
432    file_count: u64,
433    modified_unix_seconds: Option<u64>,
434}
435
436impl EntryStats {
437    fn from_metadata(metadata: &fs::Metadata) -> Self {
438        Self {
439            size_bytes: metadata.len(),
440            file_count: u64::from(metadata.is_file()),
441            modified_unix_seconds: modified_unix_seconds(metadata),
442        }
443    }
444
445    fn merge(&mut self, other: Self) {
446        self.size_bytes = self.size_bytes.saturating_add(other.size_bytes);
447        self.file_count = self.file_count.saturating_add(other.file_count);
448        self.modified_unix_seconds = self.modified_unix_seconds.max(other.modified_unix_seconds);
449    }
450}
451
452fn tree_stats(path: &Path) -> Result<EntryStats, PythonEnvironmentCacheError> {
453    let mut stats = EntryStats::default();
454    let mut pending = vec![path.to_path_buf()];
455    while let Some(current) = pending.pop() {
456        let metadata = symlink_metadata(&current)?;
457        stats.merge(EntryStats::from_metadata(&metadata));
458        if metadata.file_type().is_symlink() || !metadata.is_dir() {
459            continue;
460        }
461        pending.extend(read_paths(&current)?);
462    }
463    Ok(stats)
464}
465
466fn read_paths(path: &Path) -> Result<Vec<PathBuf>, PythonEnvironmentCacheError> {
467    fs::read_dir(path)
468        .map_err(|source| PythonEnvironmentCacheError::Io {
469            path: path.to_path_buf(),
470            source,
471        })?
472        .map(|entry| {
473            entry
474                .map(|entry| entry.path())
475                .map_err(|source| PythonEnvironmentCacheError::Io {
476                    path: path.to_path_buf(),
477                    source,
478                })
479        })
480        .collect()
481}
482
483fn symlink_metadata(path: &Path) -> Result<fs::Metadata, PythonEnvironmentCacheError> {
484    fs::symlink_metadata(path).map_err(|source| PythonEnvironmentCacheError::Io {
485        path: path.to_path_buf(),
486        source,
487    })
488}
489
490fn modified_unix_seconds(metadata: &fs::Metadata) -> Option<u64> {
491    metadata
492        .modified()
493        .ok()?
494        .duration_since(UNIX_EPOCH)
495        .ok()
496        .map(|duration| duration.as_secs())
497}
498
499fn environment_python_path(directory: &Path) -> PathBuf {
500    let unix = directory.join(".venv/bin/python");
501    if fs::symlink_metadata(&unix).is_ok() {
502        unix
503    } else {
504        directory.join(".venv/Scripts/python.exe")
505    }
506}
507
508fn sha256_hex(bytes: &[u8]) -> String {
509    const HEX: &[u8; 16] = b"0123456789abcdef";
510    let digest = Sha256::digest(bytes);
511    let mut encoded = String::with_capacity(digest.len() * 2);
512    for byte in digest {
513        encoded.push(HEX[(byte >> 4) as usize] as char);
514        encoded.push(HEX[(byte & 0x0f) as usize] as char);
515    }
516    encoded
517}
518
519#[cfg(test)]
520#[path = "cache_tests.rs"]
521mod tests;
soma_provider_adapters/python/cache.rs

soma_provider_adapters/python/
cache.rs
