Public release release-ff7f847f143d
Source commit: ff7f847f143dc67864bed68a86cf259114384bd5 Public tree identity: sha256:bb7dcd2ac78bdbad2f4eba0f49be649d446d67f96f4eb2796941c026412fc032
This commit is contained in:
Clusterflux release
commit
70319cde15
210 changed files with 78958 additions and 0 deletions
424
crates/clusterflux-node/src/source_snapshot.rs
Normal file
424
crates/clusterflux-node/src/source_snapshot.rs
Normal file
|
|
@ -0,0 +1,424 @@
|
|||
use std::fs;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command;
|
||||
|
||||
use clusterflux_core::Digest;
|
||||
use sha2::{Digest as _, Sha256};
|
||||
|
||||
const MAX_SNAPSHOT_FILES: usize = 20_000;
|
||||
const MAX_SNAPSHOT_FILE_BYTES: u64 = 256 * 1024 * 1024;
|
||||
const MAX_SNAPSHOT_TOTAL_BYTES: u64 = 1024 * 1024 * 1024;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct SourceSnapshotInventory {
|
||||
pub(crate) digest: Digest,
|
||||
pub(crate) provider: &'static str,
|
||||
pub(crate) file_count: usize,
|
||||
pub(crate) total_bytes: u64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
struct SnapshotFile {
|
||||
path: String,
|
||||
kind: &'static str,
|
||||
executable: bool,
|
||||
digest: Digest,
|
||||
size_bytes: u64,
|
||||
}
|
||||
|
||||
pub(crate) fn snapshot_project(project_root: &Path) -> Result<SourceSnapshotInventory, String> {
|
||||
let project_root = project_root
|
||||
.canonicalize()
|
||||
.map_err(|error| format!("resolve source checkout: {error}"))?;
|
||||
if !project_root.is_dir() {
|
||||
return Err("source checkout root is not a directory".to_owned());
|
||||
}
|
||||
if let Some(repository_root) = git_repository_root(&project_root)? {
|
||||
snapshot_git_project(&repository_root, &project_root)
|
||||
} else {
|
||||
snapshot_filesystem_project(&project_root)
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot_git_project(
|
||||
repository_root: &Path,
|
||||
project_root: &Path,
|
||||
) -> Result<SourceSnapshotInventory, String> {
|
||||
let project_prefix = project_root
|
||||
.strip_prefix(repository_root)
|
||||
.map_err(|_| "source checkout is outside its discovered Git repository".to_owned())?;
|
||||
let project_prefix = if project_prefix.as_os_str().is_empty() {
|
||||
".".to_owned()
|
||||
} else {
|
||||
slash_path(project_prefix)?
|
||||
};
|
||||
let mut command = git_command(repository_root);
|
||||
command
|
||||
.args([
|
||||
"ls-files",
|
||||
"-z",
|
||||
"--cached",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"--",
|
||||
])
|
||||
.arg(&project_prefix);
|
||||
let output = command
|
||||
.output()
|
||||
.map_err(|error| format!("enumerate Git source snapshot: {error}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(format!(
|
||||
"enumerate Git source snapshot failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr).trim()
|
||||
));
|
||||
}
|
||||
let mut files = Vec::new();
|
||||
for raw in output
|
||||
.stdout
|
||||
.split(|byte| *byte == 0)
|
||||
.filter(|raw| !raw.is_empty())
|
||||
{
|
||||
let repository_relative =
|
||||
std::str::from_utf8(raw).map_err(|_| "Git source path is not UTF-8".to_owned())?;
|
||||
let absolute = repository_root.join(repository_relative);
|
||||
let relative = absolute
|
||||
.strip_prefix(project_root)
|
||||
.map_err(|_| "Git enumerated source outside the selected project".to_owned())?;
|
||||
files.push(snapshot_file(&absolute, &slash_path(relative)?)?);
|
||||
}
|
||||
files.sort_by(|left, right| left.path.cmp(&right.path));
|
||||
files.dedup_by(|left, right| left.path == right.path);
|
||||
validate_snapshot_bounds(&files)?;
|
||||
|
||||
let head =
|
||||
git_text(repository_root, &["rev-parse", "HEAD"]).unwrap_or_else(|| "unborn".to_owned());
|
||||
let roots = git_text(repository_root, &["rev-list", "--max-parents=0", "HEAD"])
|
||||
.unwrap_or_else(|| "unborn".to_owned());
|
||||
let remote = git_text(repository_root, &["config", "--get", "remote.origin.url"])
|
||||
.unwrap_or_else(|| "no-origin".to_owned());
|
||||
let submodules = git_text(
|
||||
repository_root,
|
||||
&["submodule", "status", "--recursive", "--", &project_prefix],
|
||||
)
|
||||
.unwrap_or_else(|| "no-submodules".to_owned());
|
||||
finish_snapshot(
|
||||
"git",
|
||||
[
|
||||
b"node-source-snapshot:git:v1".to_vec(),
|
||||
head.into_bytes(),
|
||||
roots.into_bytes(),
|
||||
remote.into_bytes(),
|
||||
submodules.into_bytes(),
|
||||
project_prefix.into_bytes(),
|
||||
],
|
||||
files,
|
||||
)
|
||||
}
|
||||
|
||||
fn snapshot_filesystem_project(project_root: &Path) -> Result<SourceSnapshotInventory, String> {
|
||||
let mut paths = Vec::new();
|
||||
collect_filesystem_paths(project_root, project_root, &mut paths)?;
|
||||
paths.sort();
|
||||
if paths.len() > MAX_SNAPSHOT_FILES {
|
||||
return Err(format!(
|
||||
"source snapshot has {} files; limit is {MAX_SNAPSHOT_FILES}",
|
||||
paths.len()
|
||||
));
|
||||
}
|
||||
let mut files = Vec::with_capacity(paths.len());
|
||||
for absolute in paths {
|
||||
let relative = absolute
|
||||
.strip_prefix(project_root)
|
||||
.map_err(|_| "filesystem source escaped its project root".to_owned())?;
|
||||
files.push(snapshot_file(&absolute, &slash_path(relative)?)?);
|
||||
}
|
||||
validate_snapshot_bounds(&files)?;
|
||||
finish_snapshot(
|
||||
"filesystem",
|
||||
[b"node-source-snapshot:filesystem:v1".to_vec()],
|
||||
files,
|
||||
)
|
||||
}
|
||||
|
||||
fn finish_snapshot<const N: usize>(
|
||||
provider: &'static str,
|
||||
identity_parts: [Vec<u8>; N],
|
||||
files: Vec<SnapshotFile>,
|
||||
) -> Result<SourceSnapshotInventory, String> {
|
||||
let mut parts = identity_parts.into_iter().collect::<Vec<_>>();
|
||||
let mut total_bytes = 0_u64;
|
||||
for file in &files {
|
||||
total_bytes = total_bytes
|
||||
.checked_add(file.size_bytes)
|
||||
.ok_or_else(|| "source snapshot byte accounting overflowed".to_owned())?;
|
||||
parts.push(file.path.as_bytes().to_vec());
|
||||
parts.push(file.kind.as_bytes().to_vec());
|
||||
parts.push(if file.executable { b"x" } else { b"-" }.to_vec());
|
||||
parts.push(file.digest.as_str().as_bytes().to_vec());
|
||||
parts.push(file.size_bytes.to_string().into_bytes());
|
||||
}
|
||||
Ok(SourceSnapshotInventory {
|
||||
digest: Digest::from_parts(parts),
|
||||
provider,
|
||||
file_count: files.len(),
|
||||
total_bytes,
|
||||
})
|
||||
}
|
||||
|
||||
fn snapshot_file(absolute: &Path, relative: &str) -> Result<SnapshotFile, String> {
|
||||
validate_relative_source_path(relative)?;
|
||||
let metadata = match fs::symlink_metadata(absolute) {
|
||||
Ok(metadata) => metadata,
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Ok(SnapshotFile {
|
||||
path: relative.to_owned(),
|
||||
kind: "deleted",
|
||||
executable: false,
|
||||
digest: Digest::sha256("deleted"),
|
||||
size_bytes: 0,
|
||||
})
|
||||
}
|
||||
Err(error) => return Err(error.to_string()),
|
||||
};
|
||||
if metadata.file_type().is_symlink() {
|
||||
let target = fs::read_link(absolute).map_err(|error| error.to_string())?;
|
||||
let target = target
|
||||
.to_str()
|
||||
.ok_or_else(|| "source symlink target is not UTF-8".to_owned())?;
|
||||
return Ok(SnapshotFile {
|
||||
path: relative.to_owned(),
|
||||
kind: "symlink",
|
||||
executable: false,
|
||||
digest: Digest::from_parts([b"source-symlink:v1".as_slice(), target.as_bytes()]),
|
||||
size_bytes: target.len() as u64,
|
||||
});
|
||||
}
|
||||
if !metadata.is_file() {
|
||||
return Err(format!("source snapshot entry `{relative}` is not a file"));
|
||||
}
|
||||
if metadata.len() > MAX_SNAPSHOT_FILE_BYTES {
|
||||
return Err(format!(
|
||||
"source file `{relative}` is {} bytes; per-file limit is {MAX_SNAPSHOT_FILE_BYTES}",
|
||||
metadata.len()
|
||||
));
|
||||
}
|
||||
let mut file = fs::File::open(absolute).map_err(|error| error.to_string())?;
|
||||
let mut hasher = Sha256::new();
|
||||
let mut size_bytes = 0_u64;
|
||||
let mut buffer = [0_u8; 64 * 1024];
|
||||
loop {
|
||||
let read = file.read(&mut buffer).map_err(|error| error.to_string())?;
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
size_bytes = size_bytes
|
||||
.checked_add(read as u64)
|
||||
.ok_or_else(|| "source file size overflowed".to_owned())?;
|
||||
if size_bytes > MAX_SNAPSHOT_FILE_BYTES {
|
||||
return Err(format!(
|
||||
"source file `{relative}` grew beyond {MAX_SNAPSHOT_FILE_BYTES} bytes while hashing"
|
||||
));
|
||||
}
|
||||
hasher.update(&buffer[..read]);
|
||||
}
|
||||
let digest = Digest::from_sha256_hex(format!("{:x}", hasher.finalize()))?;
|
||||
#[cfg(unix)]
|
||||
let executable = {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
metadata.permissions().mode() & 0o111 != 0
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let executable = false;
|
||||
Ok(SnapshotFile {
|
||||
path: relative.to_owned(),
|
||||
kind: "file",
|
||||
executable,
|
||||
digest,
|
||||
size_bytes,
|
||||
})
|
||||
}
|
||||
|
||||
fn validate_snapshot_bounds(files: &[SnapshotFile]) -> Result<(), String> {
|
||||
if files.len() > MAX_SNAPSHOT_FILES {
|
||||
return Err(format!(
|
||||
"source snapshot has {} files; limit is {MAX_SNAPSHOT_FILES}",
|
||||
files.len()
|
||||
));
|
||||
}
|
||||
let total = files
|
||||
.iter()
|
||||
.try_fold(0_u64, |total, file| total.checked_add(file.size_bytes))
|
||||
.ok_or_else(|| "source snapshot byte accounting overflowed".to_owned())?;
|
||||
if total > MAX_SNAPSHOT_TOTAL_BYTES {
|
||||
return Err(format!(
|
||||
"source snapshot is {total} bytes; limit is {MAX_SNAPSHOT_TOTAL_BYTES}"
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn collect_filesystem_paths(
|
||||
root: &Path,
|
||||
directory: &Path,
|
||||
paths: &mut Vec<PathBuf>,
|
||||
) -> Result<(), String> {
|
||||
for entry in fs::read_dir(directory).map_err(|error| error.to_string())? {
|
||||
let entry = entry.map_err(|error| error.to_string())?;
|
||||
let name = entry
|
||||
.file_name()
|
||||
.into_string()
|
||||
.map_err(|_| "source path is not UTF-8".to_owned())?;
|
||||
if directory == root && matches!(name.as_str(), ".git" | ".clusterflux" | "target") {
|
||||
continue;
|
||||
}
|
||||
let file_type = entry.file_type().map_err(|error| error.to_string())?;
|
||||
if file_type.is_dir() {
|
||||
collect_filesystem_paths(root, &entry.path(), paths)?;
|
||||
} else if file_type.is_file() || file_type.is_symlink() {
|
||||
paths.push(entry.path());
|
||||
if paths.len() > MAX_SNAPSHOT_FILES {
|
||||
return Err(format!(
|
||||
"source snapshot exceeds file-count limit of {MAX_SNAPSHOT_FILES}"
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn git_repository_root(project_root: &Path) -> Result<Option<PathBuf>, String> {
|
||||
let output = git_command(project_root)
|
||||
.args(["rev-parse", "--show-toplevel"])
|
||||
.output();
|
||||
let Ok(output) = output else {
|
||||
return Ok(None);
|
||||
};
|
||||
if !output.status.success() {
|
||||
return Ok(None);
|
||||
}
|
||||
let root = String::from_utf8(output.stdout)
|
||||
.map_err(|_| "Git repository root is not UTF-8".to_owned())?;
|
||||
let root = PathBuf::from(root.trim())
|
||||
.canonicalize()
|
||||
.map_err(|error| error.to_string())?;
|
||||
Ok(Some(root))
|
||||
}
|
||||
|
||||
fn git_text(repository_root: &Path, args: &[&str]) -> Option<String> {
|
||||
let output = git_command(repository_root).args(args).output().ok()?;
|
||||
if !output.status.success() {
|
||||
return None;
|
||||
}
|
||||
String::from_utf8(output.stdout)
|
||||
.ok()
|
||||
.map(|value| value.trim().to_owned())
|
||||
}
|
||||
|
||||
fn git_command(repository_root: &Path) -> Command {
|
||||
let mut command = Command::new("git");
|
||||
command
|
||||
.arg("-C")
|
||||
.arg(repository_root)
|
||||
.env("GIT_OPTIONAL_LOCKS", "0")
|
||||
.env("GIT_TERMINAL_PROMPT", "0");
|
||||
command
|
||||
}
|
||||
|
||||
fn slash_path(path: &Path) -> Result<String, String> {
|
||||
let mut result = String::new();
|
||||
for component in path.components() {
|
||||
let component = component
|
||||
.as_os_str()
|
||||
.to_str()
|
||||
.ok_or_else(|| "source path is not UTF-8".to_owned())?;
|
||||
if !result.is_empty() {
|
||||
result.push('/');
|
||||
}
|
||||
result.push_str(component);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn validate_relative_source_path(path: &str) -> Result<(), String> {
|
||||
if path.is_empty()
|
||||
|| path.starts_with('/')
|
||||
|| path.split('/').any(|component| {
|
||||
component.is_empty()
|
||||
|| component == "."
|
||||
|| component == ".."
|
||||
|| component.contains('\0')
|
||||
})
|
||||
{
|
||||
return Err("source snapshot path must be a safe relative path".to_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn git(root: &Path, args: &[&str]) {
|
||||
let output = Command::new("git")
|
||||
.arg("-C")
|
||||
.arg(root)
|
||||
.args(args)
|
||||
.env("GIT_AUTHOR_NAME", "Clusterflux Test")
|
||||
.env("GIT_AUTHOR_EMAIL", "test@clusterflux.invalid")
|
||||
.env("GIT_COMMITTER_NAME", "Clusterflux Test")
|
||||
.env("GIT_COMMITTER_EMAIL", "test@clusterflux.invalid")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"git {args:?} failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
}
|
||||
|
||||
fn initialize_repository(root: &Path, message: &str) {
|
||||
git(root, &["init", "--quiet"]);
|
||||
git(root, &["add", "."]);
|
||||
git(root, &["commit", "--quiet", "-m", message]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dirty_and_untracked_content_changes_snapshot_identity() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
fs::create_dir_all(temp.path().join("src")).unwrap();
|
||||
fs::write(temp.path().join("src/lib.rs"), "pub fn value() -> u8 { 1 }").unwrap();
|
||||
initialize_repository(temp.path(), "initial source");
|
||||
let first = snapshot_project(temp.path()).unwrap();
|
||||
|
||||
fs::write(temp.path().join("src/lib.rs"), "pub fn value() -> u8 { 2 }").unwrap();
|
||||
let dirty = snapshot_project(temp.path()).unwrap();
|
||||
fs::write(
|
||||
temp.path().join("fixture.c"),
|
||||
"int main(void) { return 0; }",
|
||||
)
|
||||
.unwrap();
|
||||
let untracked = snapshot_project(temp.path()).unwrap();
|
||||
|
||||
assert_ne!(first.digest, dirty.digest);
|
||||
assert_ne!(dirty.digest, untracked.digest);
|
||||
assert_eq!(untracked.provider, "git");
|
||||
assert_eq!(untracked.file_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn different_repositories_at_the_same_path_are_not_identified_by_path() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
fs::write(temp.path().join("source.txt"), "identical source content").unwrap();
|
||||
initialize_repository(temp.path(), "first repository identity");
|
||||
let first = snapshot_project(temp.path()).unwrap();
|
||||
|
||||
fs::remove_dir_all(temp.path().join(".git")).unwrap();
|
||||
initialize_repository(temp.path(), "second repository identity");
|
||||
let second = snapshot_project(temp.path()).unwrap();
|
||||
|
||||
assert_ne!(first.digest, second.digest);
|
||||
assert_eq!(first.provider, "git");
|
||||
assert_eq!(second.provider, "git");
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue