clusterflux-public/crates/clusterflux-node/src/source_snapshot.rs

use std::fs;
use std::io::Read;
use std::path::{Path, PathBuf};
use std::process::Command;

use clusterflux_core::Digest;
use sha2::{Digest as _, Sha256};

const MAX_SNAPSHOT_FILES: usize = 20_000;
const MAX_SNAPSHOT_FILE_BYTES: u64 = 256 * 1024 * 1024;
const MAX_SNAPSHOT_TOTAL_BYTES: u64 = 1024 * 1024 * 1024;

#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct SourceSnapshotInventory {
    pub(crate) digest: Digest,
    pub(crate) provider: &'static str,
    pub(crate) file_count: usize,
    pub(crate) total_bytes: u64,
}

#[derive(Clone, Debug, PartialEq, Eq)]
struct SnapshotFile {
    path: String,
    kind: &'static str,
    executable: bool,
    digest: Digest,
    size_bytes: u64,
}

pub(crate) fn snapshot_project(project_root: &Path) -> Result<SourceSnapshotInventory, String> {
    let project_root = project_root
        .canonicalize()
        .map_err(|error| format!("resolve source checkout: {error}"))?;
    if !project_root.is_dir() {
        return Err("source checkout root is not a directory".to_owned());
    }
    if let Some(repository_root) = git_repository_root(&project_root)? {
        snapshot_git_project(&repository_root, &project_root)
    } else {
        snapshot_filesystem_project(&project_root)
    }
}

fn snapshot_git_project(
    repository_root: &Path,
    project_root: &Path,
) -> Result<SourceSnapshotInventory, String> {
    let project_prefix = project_root
        .strip_prefix(repository_root)
        .map_err(|_| "source checkout is outside its discovered Git repository".to_owned())?;
    let project_prefix = if project_prefix.as_os_str().is_empty() {
        ".".to_owned()
    } else {
        slash_path(project_prefix)?
    };
    let mut command = git_command(repository_root);
    command
        .args([
            "ls-files",
            "-z",
            "--cached",
            "--others",
            "--exclude-standard",
            "--",
        ])
        .arg(&project_prefix);
    let output = command
        .output()
        .map_err(|error| format!("enumerate Git source snapshot: {error}"))?;
    if !output.status.success() {
        return Err(format!(
            "enumerate Git source snapshot failed: {}",
            String::from_utf8_lossy(&output.stderr).trim()
        ));
    }
    let mut files = Vec::new();
    for raw in output
        .stdout
        .split(|byte| *byte == 0)
        .filter(|raw| !raw.is_empty())
    {
        let repository_relative =
            std::str::from_utf8(raw).map_err(|_| "Git source path is not UTF-8".to_owned())?;
        let absolute = repository_root.join(repository_relative);
        let relative = absolute
            .strip_prefix(project_root)
            .map_err(|_| "Git enumerated source outside the selected project".to_owned())?;
        files.push(snapshot_file(&absolute, &slash_path(relative)?)?);
    }
    files.sort_by(|left, right| left.path.cmp(&right.path));
    files.dedup_by(|left, right| left.path == right.path);
    validate_snapshot_bounds(&files)?;

    let head =
        git_text(repository_root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unborn".to_owned());
    let roots = git_text(repository_root, &["rev-list", "--max-parents=0", "HEAD"])
        .unwrap_or_else(|| "unborn".to_owned());
    let remote = git_text(repository_root, &["config", "--get", "remote.origin.url"])
        .unwrap_or_else(|| "no-origin".to_owned());
    let submodules = git_text(
        repository_root,
        &["submodule", "status", "--recursive", "--", &project_prefix],
    )
    .unwrap_or_else(|| "no-submodules".to_owned());
    finish_snapshot(
        "git",
        [
            b"node-source-snapshot:git:v1".to_vec(),
            head.into_bytes(),
            roots.into_bytes(),
            remote.into_bytes(),
            submodules.into_bytes(),
            project_prefix.into_bytes(),
        ],
        files,
    )
}

fn snapshot_filesystem_project(project_root: &Path) -> Result<SourceSnapshotInventory, String> {
    let mut paths = Vec::new();
    collect_filesystem_paths(project_root, project_root, &mut paths)?;
    paths.sort();
    if paths.len() > MAX_SNAPSHOT_FILES {
        return Err(format!(
            "source snapshot has {} files; limit is {MAX_SNAPSHOT_FILES}",
            paths.len()
        ));
    }
    let mut files = Vec::with_capacity(paths.len());
    for absolute in paths {
        let relative = absolute
            .strip_prefix(project_root)
            .map_err(|_| "filesystem source escaped its project root".to_owned())?;
        files.push(snapshot_file(&absolute, &slash_path(relative)?)?);
    }
    validate_snapshot_bounds(&files)?;
    finish_snapshot(
        "filesystem",
        [b"node-source-snapshot:filesystem:v1".to_vec()],
        files,
    )
}

fn finish_snapshot<const N: usize>(
    provider: &'static str,
    identity_parts: [Vec<u8>; N],
    files: Vec<SnapshotFile>,
) -> Result<SourceSnapshotInventory, String> {
    let mut parts = identity_parts.into_iter().collect::<Vec<_>>();
    let mut total_bytes = 0_u64;
    for file in &files {
        total_bytes = total_bytes
            .checked_add(file.size_bytes)
            .ok_or_else(|| "source snapshot byte accounting overflowed".to_owned())?;
        parts.push(file.path.as_bytes().to_vec());
        parts.push(file.kind.as_bytes().to_vec());
        parts.push(if file.executable { b"x" } else { b"-" }.to_vec());
        parts.push(file.digest.as_str().as_bytes().to_vec());
        parts.push(file.size_bytes.to_string().into_bytes());
    }
    Ok(SourceSnapshotInventory {
        digest: Digest::from_parts(parts),
        provider,
        file_count: files.len(),
        total_bytes,
    })
}

fn snapshot_file(absolute: &Path, relative: &str) -> Result<SnapshotFile, String> {
    validate_relative_source_path(relative)?;
    let metadata = match fs::symlink_metadata(absolute) {
        Ok(metadata) => metadata,
        Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
            return Ok(SnapshotFile {
                path: relative.to_owned(),
                kind: "deleted",
                executable: false,
                digest: Digest::sha256("deleted"),
                size_bytes: 0,
            })
        }
        Err(error) => return Err(error.to_string()),
    };
    if metadata.file_type().is_symlink() {
        let target = fs::read_link(absolute).map_err(|error| error.to_string())?;
        let target = target
            .to_str()
            .ok_or_else(|| "source symlink target is not UTF-8".to_owned())?;
        return Ok(SnapshotFile {
            path: relative.to_owned(),
            kind: "symlink",
            executable: false,
            digest: Digest::from_parts([b"source-symlink:v1".as_slice(), target.as_bytes()]),
            size_bytes: target.len() as u64,
        });
    }
    if !metadata.is_file() {
        return Err(format!("source snapshot entry `{relative}` is not a file"));
    }
    if metadata.len() > MAX_SNAPSHOT_FILE_BYTES {
        return Err(format!(
            "source file `{relative}` is {} bytes; per-file limit is {MAX_SNAPSHOT_FILE_BYTES}",
            metadata.len()
        ));
    }
    let mut file = fs::File::open(absolute).map_err(|error| error.to_string())?;
    let mut hasher = Sha256::new();
    let mut size_bytes = 0_u64;
    let mut buffer = [0_u8; 64 * 1024];
    loop {
        let read = file.read(&mut buffer).map_err(|error| error.to_string())?;
        if read == 0 {
            break;
        }
        size_bytes = size_bytes
            .checked_add(read as u64)
            .ok_or_else(|| "source file size overflowed".to_owned())?;
        if size_bytes > MAX_SNAPSHOT_FILE_BYTES {
            return Err(format!(
                "source file `{relative}` grew beyond {MAX_SNAPSHOT_FILE_BYTES} bytes while hashing"
            ));
        }
        hasher.update(&buffer[..read]);
    }
    let digest = Digest::from_sha256_hex(format!("{:x}", hasher.finalize()))?;
    #[cfg(unix)]
    let executable = {
        use std::os::unix::fs::PermissionsExt;
        metadata.permissions().mode() & 0o111 != 0
    };
    #[cfg(not(unix))]
    let executable = false;
    Ok(SnapshotFile {
        path: relative.to_owned(),
        kind: "file",
        executable,
        digest,
        size_bytes,
    })
}

fn validate_snapshot_bounds(files: &[SnapshotFile]) -> Result<(), String> {
    if files.len() > MAX_SNAPSHOT_FILES {
        return Err(format!(
            "source snapshot has {} files; limit is {MAX_SNAPSHOT_FILES}",
            files.len()
        ));
    }
    let total = files
        .iter()
        .try_fold(0_u64, |total, file| total.checked_add(file.size_bytes))
        .ok_or_else(|| "source snapshot byte accounting overflowed".to_owned())?;
    if total > MAX_SNAPSHOT_TOTAL_BYTES {
        return Err(format!(
            "source snapshot is {total} bytes; limit is {MAX_SNAPSHOT_TOTAL_BYTES}"
        ));
    }
    Ok(())
}

fn collect_filesystem_paths(
    root: &Path,
    directory: &Path,
    paths: &mut Vec<PathBuf>,
) -> Result<(), String> {
    for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
        let entry = entry.map_err(|error| error.to_string())?;
        let name = entry
            .file_name()
            .into_string()
            .map_err(|_| "source path is not UTF-8".to_owned())?;
        if directory == root && matches!(name.as_str(), ".git" | ".clusterflux" | "target") {
            continue;
        }
        let file_type = entry.file_type().map_err(|error| error.to_string())?;
        if file_type.is_dir() {
            collect_filesystem_paths(root, &entry.path(), paths)?;
        } else if file_type.is_file() || file_type.is_symlink() {
            paths.push(entry.path());
            if paths.len() > MAX_SNAPSHOT_FILES {
                return Err(format!(
                    "source snapshot exceeds file-count limit of {MAX_SNAPSHOT_FILES}"
                ));
            }
        }
    }
    Ok(())
}

fn git_repository_root(project_root: &Path) -> Result<Option<PathBuf>, String> {
    let output = git_command(project_root)
        .args(["rev-parse", "--show-toplevel"])
        .output();
    let Ok(output) = output else {
        return Ok(None);
    };
    if !output.status.success() {
        return Ok(None);
    }
    let root = String::from_utf8(output.stdout)
        .map_err(|_| "Git repository root is not UTF-8".to_owned())?;
    let root = PathBuf::from(root.trim())
        .canonicalize()
        .map_err(|error| error.to_string())?;
    Ok(Some(root))
}

fn git_text(repository_root: &Path, args: &[&str]) -> Option<String> {
    let output = git_command(repository_root).args(args).output().ok()?;
    if !output.status.success() {
        return None;
    }
    String::from_utf8(output.stdout)
        .ok()
        .map(|value| value.trim().to_owned())
}

fn git_command(repository_root: &Path) -> Command {
    let mut command = Command::new("git");
    command
        .arg("-C")
        .arg(repository_root)
        .env("GIT_OPTIONAL_LOCKS", "0")
        .env("GIT_TERMINAL_PROMPT", "0");
    command
}

fn slash_path(path: &Path) -> Result<String, String> {
    let mut result = String::new();
    for component in path.components() {
        let component = component
            .as_os_str()
            .to_str()
            .ok_or_else(|| "source path is not UTF-8".to_owned())?;
        if !result.is_empty() {
            result.push('/');
        }
        result.push_str(component);
    }
    Ok(result)
}

fn validate_relative_source_path(path: &str) -> Result<(), String> {
    if path.is_empty()
        || path.starts_with('/')
        || path.split('/').any(|component| {
            component.is_empty()
                || component == "."
                || component == ".."
                || component.contains('\0')
        })
    {
        return Err("source snapshot path must be a safe relative path".to_owned());
    }
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;

    fn git(root: &Path, args: &[&str]) {
        let output = Command::new("git")
            .arg("-C")
            .arg(root)
            .args(args)
            .env("GIT_AUTHOR_NAME", "Clusterflux Test")
            .env("GIT_AUTHOR_EMAIL", "test@clusterflux.invalid")
            .env("GIT_COMMITTER_NAME", "Clusterflux Test")
            .env("GIT_COMMITTER_EMAIL", "test@clusterflux.invalid")
            .output()
            .unwrap();
        assert!(
            output.status.success(),
            "git {args:?} failed: {}",
            String::from_utf8_lossy(&output.stderr)
        );
    }

    fn initialize_repository(root: &Path, message: &str) {
        git(root, &["init", "--quiet"]);
        git(root, &["add", "."]);
        git(root, &["commit", "--quiet", "-m", message]);
    }

    #[test]
    fn dirty_and_untracked_content_changes_snapshot_identity() {
        let temp = tempfile::tempdir().unwrap();
        fs::create_dir_all(temp.path().join("src")).unwrap();
        fs::write(temp.path().join("src/lib.rs"), "pub fn value() -> u8 { 1 }").unwrap();
        initialize_repository(temp.path(), "initial source");
        let first = snapshot_project(temp.path()).unwrap();

        fs::write(temp.path().join("src/lib.rs"), "pub fn value() -> u8 { 2 }").unwrap();
        let dirty = snapshot_project(temp.path()).unwrap();
        fs::write(
            temp.path().join("fixture.c"),
            "int main(void) { return 0; }",
        )
        .unwrap();
        let untracked = snapshot_project(temp.path()).unwrap();

        assert_ne!(first.digest, dirty.digest);
        assert_ne!(dirty.digest, untracked.digest);
        assert_eq!(untracked.provider, "git");
        assert_eq!(untracked.file_count, 2);
    }

    #[test]
    fn different_repositories_at_the_same_path_are_not_identified_by_path() {
        let temp = tempfile::tempdir().unwrap();
        fs::write(temp.path().join("source.txt"), "identical source content").unwrap();
        initialize_repository(temp.path(), "first repository identity");
        let first = snapshot_project(temp.path()).unwrap();

        fs::remove_dir_all(temp.path().join(".git")).unwrap();
        initialize_repository(temp.path(), "second repository identity");
        let second = snapshot_project(temp.path()).unwrap();

        assert_ne!(first.digest, second.digest);
        assert_eq!(first.provider, "git");
        assert_eq!(second.provider, "git");
    }
}