311 lines
14 KiB
JavaScript
311 lines
14 KiB
JavaScript
#!/usr/bin/env node
|
|
|
|
const assert = require("assert");
|
|
const fs = require("fs");
|
|
const path = require("path");
|
|
|
|
const repo = path.resolve(__dirname, "..");
|
|
|
|
function read(relativePath) {
|
|
const fullPath = path.join(repo, relativePath);
|
|
if (!fs.existsSync(fullPath)) {
|
|
if (fs.existsSync(path.join(repo, "DISASMER_PUBLIC_TREE.json"))) {
|
|
console.log(
|
|
`CLI-first contract smoke skipped: ${relativePath} is filtered from this public tree`
|
|
);
|
|
process.exit(0);
|
|
}
|
|
throw new Error(`${relativePath} is missing`);
|
|
}
|
|
return fs.readFileSync(fullPath, "utf8");
|
|
}
|
|
|
|
function criterionLines(source) {
|
|
return source
|
|
.split(/\r?\n/)
|
|
.filter((line) => /^- \[[ x]\] \*\*/.test(line));
|
|
}
|
|
|
|
function expect(source, name, pattern) {
|
|
assert.match(source, pattern, `missing CLI-first contract evidence: ${name}`);
|
|
}
|
|
|
|
const criteria = read("cli_acceptance_criteria.md");
|
|
const cli = read("crates/disasmer-cli/src/main.rs");
|
|
const coordinator = read("crates/disasmer-coordinator/src/service.rs");
|
|
const outputModeSmoke = read("scripts/cli-output-mode-smoke.js");
|
|
const errorExitSmoke = read("scripts/cli-error-exit-smoke.js");
|
|
|
|
expect(criteria, "header", /^# Disasmer CLI-First MVP Acceptance Criteria/m);
|
|
expect(
|
|
criteria,
|
|
"addendum status",
|
|
/\*\*Status:\*\* CLI-first addendum to `acceptance_criteria\.md` and `acceptance_criteria_phase2\.md`/
|
|
);
|
|
expect(
|
|
criteria,
|
|
"website exception",
|
|
/hosted account creation as the only intentional private website exception/
|
|
);
|
|
expect(
|
|
criteria,
|
|
"no duplicate work note",
|
|
/does not automatically mean new product code, a new feature, or even actual implementation work is required/
|
|
);
|
|
expect(
|
|
criteria,
|
|
"future hosted business non-goal",
|
|
/billing, paid-plan checkout, team\/org management, provider setup wizards, secret-manager UI, full support tooling, broad moderation consoles, and durable account\/business-process management are intentionally outside this CLI-first MVP slice/
|
|
);
|
|
|
|
const lines = criterionLines(criteria);
|
|
assert(lines.length > 0, "CLI-first criteria must contain criteria lines");
|
|
for (const line of lines) {
|
|
assert.match(
|
|
line,
|
|
/^- \[[ x]\] \*\*(Passed|Partial|Open|Postponed)(?: \([^)]+\))?:\*\*/,
|
|
`CLI-first criterion lacks an explicit status prefix: ${line}`
|
|
);
|
|
}
|
|
|
|
const openCriteria = lines.filter((line) => /\*\*Open(?::| \()/.test(line));
|
|
assert.deepStrictEqual(
|
|
openCriteria,
|
|
[
|
|
"- [ ] **Open:** Every criterion in this document is validated end-to-end or with integration-style coverage, preferably against a realistic NixOS deployment that can pretend to be live hosted infrastructure.",
|
|
],
|
|
"only the full CLI-first verification gate should remain Open while command/API facts are Partial"
|
|
);
|
|
|
|
for (const [name, pattern] of [
|
|
["top-level version metadata", /#\[command\(name = "disasmer", version, arg_required_else_help = true\)\]/],
|
|
["human report renderer", /fn human_report\(value: &Value\) -> String/],
|
|
["shared report emitter", /fn emit_report<T: Serialize>\(report: &T, json_output: bool\) -> Result<\(\)>/],
|
|
["doctor command", /Doctor\(DoctorArgs\)/],
|
|
["auth status command", /enum AuthCommands[\s\S]*Status\(AuthStatusArgs\)/],
|
|
["auth logout command", /enum AuthCommands[\s\S]*Logout\(AuthLogoutArgs\)/],
|
|
["key lifecycle commands", /enum KeyCommands[\s\S]*Add\(KeyAddArgs\)[\s\S]*List\(KeyListArgs\)[\s\S]*Revoke\(KeyRevokeArgs\)/],
|
|
["project commands", /enum ProjectCommands[\s\S]*Init\(ProjectInitArgs\)[\s\S]*Status\(ProjectStatusArgs\)[\s\S]*List\(ProjectListArgs\)[\s\S]*Select\(ProjectSelectArgs\)/],
|
|
["inspect command", /Inspect\(BundleInspectArgs\)/],
|
|
["build command", /Build\(BuildArgs\)/],
|
|
["node lifecycle commands", /enum NodeCommands[\s\S]*Attach\(AttachArgs\)[\s\S]*Enroll\(NodeEnrollArgs\)[\s\S]*List\(NodeListArgs\)[\s\S]*Status\(NodeStatusArgs\)[\s\S]*Revoke\(NodeRevokeArgs\)/],
|
|
["process commands", /enum ProcessCommands[\s\S]*Status\(ProcessStatusArgs\)[\s\S]*Restart\(ProcessRestartArgs\)[\s\S]*Cancel\(ProcessCancelArgs\)/],
|
|
["task lifecycle commands", /enum TaskCommands[\s\S]*List\(TaskListArgs\)[\s\S]*Restart\(TaskRestartArgs\)/],
|
|
["logs command", /Logs\(LogsArgs\)/],
|
|
["artifact commands", /enum ArtifactCommands[\s\S]*List\(ArtifactListArgs\)[\s\S]*Download\(ArtifactDownloadArgs\)[\s\S]*Export\(ArtifactExportArgs\)/],
|
|
["DAP command", /Dap\(DapArgs\)/],
|
|
["debug attach command", /enum DebugCommands[\s\S]*Attach\(DebugAttachArgs\)/],
|
|
["quota command", /enum QuotaCommands[\s\S]*Status\(QuotaStatusArgs\)/],
|
|
["admin commands", /enum AdminCommands[\s\S]*Status\(AdminStatusArgs\)[\s\S]*Bootstrap\(AdminBootstrapArgs\)[\s\S]*RevokeNode\(NodeRevokeArgs\)[\s\S]*StopProcess\(ProcessCancelArgs\)[\s\S]*SuspendTenant\(AdminSuspendTenantArgs\)/],
|
|
]) {
|
|
expect(cli, name, pattern);
|
|
}
|
|
|
|
for (const [name, pattern] of [
|
|
["CLI parse coverage", /fn cli_first_mvp_command_surface_parses\(\)/],
|
|
["CLI version coverage", /fn top_level_version_is_available\(\)/],
|
|
["CLI JSON parse coverage", /fn cli_first_json_mode_parses_for_primary_commands\(\)/],
|
|
["CLI human output coverage", /fn human_report_is_text_not_json\(\)/],
|
|
["CLI key lifecycle coverage", /fn key_lifecycle_reports_project_scoped_agent_credentials\(\)/],
|
|
["CLI agent workflow actor coverage", /fn run_with_agent_public_key_sends_attributable_workflow_actor\(\)/],
|
|
["CLI node revoke coverage", /fn node_revoke_reports_scoped_credential_revocation\(\)/],
|
|
["CLI admin public API coverage", /fn admin_status_and_suspend_use_public_coordinator_api\(\)/],
|
|
["CLI debug attach coverage", /fn debug_attach_reports_public_authorization\(\)/],
|
|
["doctor unchecked reachability coverage", /fn doctor_reports_unchecked_coordinator_reachability_without_config\(\)/],
|
|
["doctor ping reachability coverage", /fn doctor_pings_configured_coordinator\(\)/],
|
|
["project local config coverage", /fn project_init_select_and_status_use_local_project_config\(\)/],
|
|
["project coordinator status coverage", /fn project_status_queries_public_coordinator_state\(\)/],
|
|
["run coordinator active-process coverage", /fn run_contacts_configured_coordinator_and_reports_active_process_conflicts\(\)/],
|
|
["CLI error classifier coverage", /fn cli_error_classifier_distinguishes_mvp_failure_categories\(\)/],
|
|
["CLI command exit-code coverage", /fn command_report_exit_code_marks_command_failures_only\(\)/],
|
|
["CLI run rejection category coverage", /fn run_rejection_reports_machine_readable_error_category\(\)/],
|
|
["node attach grant disclosure coverage", /fn node_attach_discloses_dangerous_capability_grants\(\)/],
|
|
["quota local status coverage", /fn quota_status_uses_project_config_and_generic_public_limits\(\)/],
|
|
["quota coordinator usage coverage", /fn quota_status_queries_public_coordinator_usage\(\)/],
|
|
["task event summary coverage", /fn process_task_log_and_artifact_reports_summarize_task_events\(\)/],
|
|
["artifact download/export report coverage", /fn artifact_download_and_export_reports_expose_safe_session_boundaries\(\)/],
|
|
["process control report coverage", /fn process_restart_and_cancel_reports_expose_control_boundaries\(\)/],
|
|
["task restart report coverage", /fn task_restart_reports_clean_boundary_requirements\(\)/],
|
|
["build no full repo upload coverage", /fn build_command_reuses_bundle_inspection_without_full_repo_upload\(\)/],
|
|
["inspect missing environment coverage", /fn bundle_inspect_reports_missing_environment_references_before_schedule\(\)/],
|
|
["inspect source provider override coverage", /fn bundle_inspect_reports_source_provider_overrides_before_schedule\(\)/],
|
|
["build missing environment gate coverage", /fn build_blocks_before_schedule_on_missing_environment_reference\(\)/],
|
|
["safe coordinator-required plans", /fn node_enroll_and_process_commands_have_safe_plan_without_coordinator\(\)/],
|
|
]) {
|
|
expect(cli, name, pattern);
|
|
}
|
|
|
|
expect(
|
|
coordinator,
|
|
"coordinator whole-process cancellation coverage",
|
|
/fn service_cancels_whole_process_and_blocks_new_task_launches\(\)/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator single active process coverage",
|
|
/fn service_rejects_second_active_process_unless_restarting_same_process\(\)/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator agent key lifecycle coverage",
|
|
/fn service_manages_project_scoped_agent_public_keys\(\)/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator agent workflow dispatch coverage",
|
|
/fn service_runs_agent_workflows_with_scoped_key_attribution\(\)/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator agent workflow authorization",
|
|
/authorize_agent_project_run\([\s\S]*project:run/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator agent workflow actor fields",
|
|
/pub struct WorkflowActor[\s\S]*agent: Option<AgentId>[\s\S]*public_key_fingerprint: Option<Digest>[\s\S]*authenticated_without_browser/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator node revoke coverage",
|
|
/fn service_revokes_node_credentials_and_live_descriptors\(\)/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator public admin suspension coverage",
|
|
/fn service_reports_and_enforces_public_admin_tenant_suspension\(\)/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator debug attach coverage",
|
|
/fn service_authorizes_debug_attach_through_public_api\(\)/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator task restart boundary coverage",
|
|
/fn service_reports_task_restart_boundary_through_public_api\(\)/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"coordinator task events retain placement reasons",
|
|
/pub struct TaskCompletionEvent[\s\S]*placement: Option<Placement>[\s\S]*task_placements[\s\S]*event\.placement = self\.task_placements\.remove/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"public debug operation audit event",
|
|
/pub struct DebugAuditEvent[\s\S]*charged_debug_read_bytes[\s\S]*used_debug_read_bytes/
|
|
);
|
|
expect(
|
|
coordinator,
|
|
"public debug operation metering",
|
|
/record_debug_audit_event\([\s\S]*LimitKind::DebugReadBytes[\s\S]*DEBUG_CONTROL_READ_BYTES/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI surfaces debug audit and quota fields",
|
|
/debug_reads_quota_limited[\s\S]*charged_debug_read_bytes[\s\S]*used_debug_read_bytes/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI sends agent workflow actor fields",
|
|
/fn add_workflow_actor_fields[\s\S]*actor_agent[\s\S]*agent_public_key_fingerprint/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI exposes node attach grant disclosures",
|
|
/struct CapabilityGrantDisclosure[\s\S]*coordinator_policy_limited[\s\S]*fn capability_grant_disclosures/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI exposes task placement reasons",
|
|
/fn task_summaries[\s\S]*node_placement[\s\S]*reasons[\s\S]*explanation_available/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI renders task placement reasons",
|
|
/fn push_task_placement_reasons[\s\S]*placement \{task_name\}: \{node\}/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI classifies machine-readable error categories",
|
|
/fn classify_cli_error_message[\s\S]*"authentication"[\s\S]*"authorization"[\s\S]*"quota"[\s\S]*"policy"[\s\S]*"capability"[\s\S]*"connectivity"[\s\S]*"environment"[\s\S]*"program"/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI reports stable error-code contract",
|
|
/fn cli_error_exit_code[\s\S]*"authentication" => 20[\s\S]*"authorization" => 21[\s\S]*"quota" => 22[\s\S]*"policy" => 23[\s\S]*"capability" => 24[\s\S]*"connectivity" => 25[\s\S]*"environment" => 26[\s\S]*"program" => 27/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI attaches machine errors to run and task reports",
|
|
/run_start_summary[\s\S]*"machine_error": machine_error/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI attaches machine errors to task failures",
|
|
/task_failure_machine_error[\s\S]*cli_error_summary_with_default/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI applies process exit code after printing command failure report",
|
|
/fn emit_report<T: Serialize>[\s\S]*apply_command_report_exit_code[\s\S]*std::process::exit\(exit_code\)/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI wraps fallible main with classified exit",
|
|
/fn main\(\)[\s\S]*cli_error_summary\(&message\)[\s\S]*std::process::exit\(exit_code\)/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI parses dangerous capability overrides",
|
|
/"host-filesystem"[\s\S]*Capability::HostFilesystem[\s\S]*"network"[\s\S]*Capability::Network[\s\S]*"secrets"[\s\S]*Capability::Secrets/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI exposes source provider diagnostics",
|
|
/source_provider_statuses[\s\S]*source_provider_selection[\s\S]*source_provider_unsupported/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI exposes environment pre-schedule diagnostics",
|
|
/environment_diagnostics_for_inputs[\s\S]*diagnose_environment_references[\s\S]*missing_environment/
|
|
);
|
|
expect(
|
|
cli,
|
|
"CLI blocks build before scheduling unsafe inputs",
|
|
/blocked_before_schedule[\s\S]*scheduled_work[\s\S]*false/
|
|
);
|
|
|
|
for (const [name, pattern] of [
|
|
["agent --json flag", /struct AgentEnrollArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
|
["bundle inspect --json flag", /struct BundleInspectArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
|
["build --json flag", /struct BuildArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
|
["run --json flag", /struct RunArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
|
["node attach --json flag", /struct AttachArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
|
["DAP plan --json flag", /struct DapArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
|
["shared scope --json flag", /struct CliScopeArgs[\s\S]*#\[arg\(long\)\]\s*json: bool/],
|
|
]) {
|
|
expect(cli, name, pattern);
|
|
}
|
|
|
|
for (const [name, pattern] of [
|
|
["human default assertion", /default output should be human-readable text, not JSON/],
|
|
["login JSON mode", /\["login", "--coordinator", "https:\/\/coord\.example\.test", "--json"\]/],
|
|
["doctor human mode", /\["doctor"\]/],
|
|
["doctor reachability JSON mode", /doctorJson\.coordinator_reachability\.status/],
|
|
["bundle inspect JSON mode", /\["bundle", "inspect", "--project", project, "--json"\]/],
|
|
["auth expiry posture", /token_expiry_posture[\s\S]*expires_at/],
|
|
]) {
|
|
expect(outputModeSmoke, name, pattern);
|
|
}
|
|
|
|
for (const [name, pattern] of [
|
|
["fake coordinator quota rejection", /quota unavailable: resource limit exceeded for api_calls/],
|
|
["run uses JSON mode", /"run"[\s\S]*"--json"/],
|
|
["actual quota exit code", /assert\.strictEqual\(result\.code, 22/],
|
|
["exit-code application in JSON", /process_exit_code_applied[\s\S]*true/],
|
|
]) {
|
|
expect(errorExitSmoke, name, pattern);
|
|
}
|
|
|
|
console.log("CLI-first contract smoke passed");
|