use std::fs; use std::io::Read; use std::path::{Path, PathBuf}; use std::process::Command; use clusterflux_core::Digest; use sha2::{Digest as _, Sha256}; const MAX_SNAPSHOT_FILES: usize = 20_000; const MAX_SNAPSHOT_FILE_BYTES: u64 = 256 * 1024 * 1024; const MAX_SNAPSHOT_TOTAL_BYTES: u64 = 1024 * 1024 * 1024; #[derive(Clone, Debug, PartialEq, Eq)] pub(crate) struct SourceSnapshotInventory { pub(crate) digest: Digest, pub(crate) provider: &'static str, pub(crate) file_count: usize, pub(crate) total_bytes: u64, } #[derive(Clone, Debug, PartialEq, Eq)] struct SnapshotFile { path: String, kind: &'static str, executable: bool, digest: Digest, size_bytes: u64, } pub(crate) fn snapshot_project(project_root: &Path) -> Result { let project_root = project_root .canonicalize() .map_err(|error| format!("resolve source checkout: {error}"))?; if !project_root.is_dir() { return Err("source checkout root is not a directory".to_owned()); } if let Some(repository_root) = git_repository_root(&project_root)? { snapshot_git_project(&repository_root, &project_root) } else { snapshot_filesystem_project(&project_root) } } fn snapshot_git_project( repository_root: &Path, project_root: &Path, ) -> Result { let project_prefix = project_root .strip_prefix(repository_root) .map_err(|_| "source checkout is outside its discovered Git repository".to_owned())?; let project_prefix = if project_prefix.as_os_str().is_empty() { ".".to_owned() } else { slash_path(project_prefix)? }; let mut command = git_command(repository_root); command .args([ "ls-files", "-z", "--cached", "--others", "--exclude-standard", "--", ]) .arg(&project_prefix); let output = command .output() .map_err(|error| format!("enumerate Git source snapshot: {error}"))?; if !output.status.success() { return Err(format!( "enumerate Git source snapshot failed: {}", String::from_utf8_lossy(&output.stderr).trim() )); } let mut files = Vec::new(); for raw in output .stdout .split(|byte| *byte == 0) .filter(|raw| !raw.is_empty()) { let repository_relative = std::str::from_utf8(raw).map_err(|_| "Git source path is not UTF-8".to_owned())?; let absolute = repository_root.join(repository_relative); let relative = absolute .strip_prefix(project_root) .map_err(|_| "Git enumerated source outside the selected project".to_owned())?; files.push(snapshot_file(&absolute, &slash_path(relative)?)?); } files.sort_by(|left, right| left.path.cmp(&right.path)); files.dedup_by(|left, right| left.path == right.path); validate_snapshot_bounds(&files)?; let head = git_text(repository_root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unborn".to_owned()); let roots = git_text(repository_root, &["rev-list", "--max-parents=0", "HEAD"]) .unwrap_or_else(|| "unborn".to_owned()); let remote = git_text(repository_root, &["config", "--get", "remote.origin.url"]) .unwrap_or_else(|| "no-origin".to_owned()); let submodules = git_text( repository_root, &["submodule", "status", "--recursive", "--", &project_prefix], ) .unwrap_or_else(|| "no-submodules".to_owned()); finish_snapshot( "git", [ b"node-source-snapshot:git:v1".to_vec(), head.into_bytes(), roots.into_bytes(), remote.into_bytes(), submodules.into_bytes(), project_prefix.into_bytes(), ], files, ) } fn snapshot_filesystem_project(project_root: &Path) -> Result { let mut paths = Vec::new(); collect_filesystem_paths(project_root, project_root, &mut paths)?; paths.sort(); if paths.len() > MAX_SNAPSHOT_FILES { return Err(format!( "source snapshot has {} files; limit is {MAX_SNAPSHOT_FILES}", paths.len() )); } let mut files = Vec::with_capacity(paths.len()); for absolute in paths { let relative = absolute .strip_prefix(project_root) .map_err(|_| "filesystem source escaped its project root".to_owned())?; files.push(snapshot_file(&absolute, &slash_path(relative)?)?); } validate_snapshot_bounds(&files)?; finish_snapshot( "filesystem", [b"node-source-snapshot:filesystem:v1".to_vec()], files, ) } fn finish_snapshot( provider: &'static str, identity_parts: [Vec; N], files: Vec, ) -> Result { let mut parts = identity_parts.into_iter().collect::>(); let mut total_bytes = 0_u64; for file in &files { total_bytes = total_bytes .checked_add(file.size_bytes) .ok_or_else(|| "source snapshot byte accounting overflowed".to_owned())?; parts.push(file.path.as_bytes().to_vec()); parts.push(file.kind.as_bytes().to_vec()); parts.push(if file.executable { b"x" } else { b"-" }.to_vec()); parts.push(file.digest.as_str().as_bytes().to_vec()); parts.push(file.size_bytes.to_string().into_bytes()); } Ok(SourceSnapshotInventory { digest: Digest::from_parts(parts), provider, file_count: files.len(), total_bytes, }) } fn snapshot_file(absolute: &Path, relative: &str) -> Result { validate_relative_source_path(relative)?; let metadata = match fs::symlink_metadata(absolute) { Ok(metadata) => metadata, Err(error) if error.kind() == std::io::ErrorKind::NotFound => { return Ok(SnapshotFile { path: relative.to_owned(), kind: "deleted", executable: false, digest: Digest::sha256("deleted"), size_bytes: 0, }) } Err(error) => return Err(error.to_string()), }; if metadata.file_type().is_symlink() { let target = fs::read_link(absolute).map_err(|error| error.to_string())?; let target = target .to_str() .ok_or_else(|| "source symlink target is not UTF-8".to_owned())?; return Ok(SnapshotFile { path: relative.to_owned(), kind: "symlink", executable: false, digest: Digest::from_parts([b"source-symlink:v1".as_slice(), target.as_bytes()]), size_bytes: target.len() as u64, }); } if !metadata.is_file() { return Err(format!("source snapshot entry `{relative}` is not a file")); } if metadata.len() > MAX_SNAPSHOT_FILE_BYTES { return Err(format!( "source file `{relative}` is {} bytes; per-file limit is {MAX_SNAPSHOT_FILE_BYTES}", metadata.len() )); } let mut file = fs::File::open(absolute).map_err(|error| error.to_string())?; let mut hasher = Sha256::new(); let mut size_bytes = 0_u64; let mut buffer = [0_u8; 64 * 1024]; loop { let read = file.read(&mut buffer).map_err(|error| error.to_string())?; if read == 0 { break; } size_bytes = size_bytes .checked_add(read as u64) .ok_or_else(|| "source file size overflowed".to_owned())?; if size_bytes > MAX_SNAPSHOT_FILE_BYTES { return Err(format!( "source file `{relative}` grew beyond {MAX_SNAPSHOT_FILE_BYTES} bytes while hashing" )); } hasher.update(&buffer[..read]); } let digest = Digest::from_sha256_hex(format!("{:x}", hasher.finalize()))?; #[cfg(unix)] let executable = { use std::os::unix::fs::PermissionsExt; metadata.permissions().mode() & 0o111 != 0 }; #[cfg(not(unix))] let executable = false; Ok(SnapshotFile { path: relative.to_owned(), kind: "file", executable, digest, size_bytes, }) } fn validate_snapshot_bounds(files: &[SnapshotFile]) -> Result<(), String> { if files.len() > MAX_SNAPSHOT_FILES { return Err(format!( "source snapshot has {} files; limit is {MAX_SNAPSHOT_FILES}", files.len() )); } let total = files .iter() .try_fold(0_u64, |total, file| total.checked_add(file.size_bytes)) .ok_or_else(|| "source snapshot byte accounting overflowed".to_owned())?; if total > MAX_SNAPSHOT_TOTAL_BYTES { return Err(format!( "source snapshot is {total} bytes; limit is {MAX_SNAPSHOT_TOTAL_BYTES}" )); } Ok(()) } fn collect_filesystem_paths( root: &Path, directory: &Path, paths: &mut Vec, ) -> Result<(), String> { for entry in fs::read_dir(directory).map_err(|error| error.to_string())? { let entry = entry.map_err(|error| error.to_string())?; let name = entry .file_name() .into_string() .map_err(|_| "source path is not UTF-8".to_owned())?; if directory == root && matches!(name.as_str(), ".git" | ".clusterflux" | "target") { continue; } let file_type = entry.file_type().map_err(|error| error.to_string())?; if file_type.is_dir() { collect_filesystem_paths(root, &entry.path(), paths)?; } else if file_type.is_file() || file_type.is_symlink() { paths.push(entry.path()); if paths.len() > MAX_SNAPSHOT_FILES { return Err(format!( "source snapshot exceeds file-count limit of {MAX_SNAPSHOT_FILES}" )); } } } Ok(()) } fn git_repository_root(project_root: &Path) -> Result, String> { let output = git_command(project_root) .args(["rev-parse", "--show-toplevel"]) .output(); let Ok(output) = output else { return Ok(None); }; if !output.status.success() { return Ok(None); } let root = String::from_utf8(output.stdout) .map_err(|_| "Git repository root is not UTF-8".to_owned())?; let root = PathBuf::from(root.trim()) .canonicalize() .map_err(|error| error.to_string())?; Ok(Some(root)) } fn git_text(repository_root: &Path, args: &[&str]) -> Option { let output = git_command(repository_root).args(args).output().ok()?; if !output.status.success() { return None; } String::from_utf8(output.stdout) .ok() .map(|value| value.trim().to_owned()) } fn git_command(repository_root: &Path) -> Command { let mut command = Command::new("git"); command .arg("-C") .arg(repository_root) .env("GIT_OPTIONAL_LOCKS", "0") .env("GIT_TERMINAL_PROMPT", "0"); command } fn slash_path(path: &Path) -> Result { let mut result = String::new(); for component in path.components() { let component = component .as_os_str() .to_str() .ok_or_else(|| "source path is not UTF-8".to_owned())?; if !result.is_empty() { result.push('/'); } result.push_str(component); } Ok(result) } fn validate_relative_source_path(path: &str) -> Result<(), String> { if path.is_empty() || path.starts_with('/') || path.split('/').any(|component| { component.is_empty() || component == "." || component == ".." || component.contains('\0') }) { return Err("source snapshot path must be a safe relative path".to_owned()); } Ok(()) } #[cfg(test)] mod tests { use super::*; fn git(root: &Path, args: &[&str]) { let output = Command::new("git") .arg("-C") .arg(root) .args(args) .env("GIT_AUTHOR_NAME", "Clusterflux Test") .env("GIT_AUTHOR_EMAIL", "test@clusterflux.invalid") .env("GIT_COMMITTER_NAME", "Clusterflux Test") .env("GIT_COMMITTER_EMAIL", "test@clusterflux.invalid") .output() .unwrap(); assert!( output.status.success(), "git {args:?} failed: {}", String::from_utf8_lossy(&output.stderr) ); } fn initialize_repository(root: &Path, message: &str) { git(root, &["init", "--quiet"]); git(root, &["add", "."]); git(root, &["commit", "--quiet", "-m", message]); } #[test] fn dirty_and_untracked_content_changes_snapshot_identity() { let temp = tempfile::tempdir().unwrap(); fs::create_dir_all(temp.path().join("src")).unwrap(); fs::write(temp.path().join("src/lib.rs"), "pub fn value() -> u8 { 1 }").unwrap(); initialize_repository(temp.path(), "initial source"); let first = snapshot_project(temp.path()).unwrap(); fs::write(temp.path().join("src/lib.rs"), "pub fn value() -> u8 { 2 }").unwrap(); let dirty = snapshot_project(temp.path()).unwrap(); fs::write( temp.path().join("fixture.c"), "int main(void) { return 0; }", ) .unwrap(); let untracked = snapshot_project(temp.path()).unwrap(); assert_ne!(first.digest, dirty.digest); assert_ne!(dirty.digest, untracked.digest); assert_eq!(untracked.provider, "git"); assert_eq!(untracked.file_count, 2); } #[test] fn different_repositories_at_the_same_path_are_not_identified_by_path() { let temp = tempfile::tempdir().unwrap(); fs::write(temp.path().join("source.txt"), "identical source content").unwrap(); initialize_repository(temp.path(), "first repository identity"); let first = snapshot_project(temp.path()).unwrap(); fs::remove_dir_all(temp.path().join(".git")).unwrap(); initialize_repository(temp.path(), "second repository identity"); let second = snapshot_project(temp.path()).unwrap(); assert_ne!(first.digest, second.digest); assert_eq!(first.provider, "git"); assert_eq!(second.provider, "git"); } }