steel-dev__cli/tests/cli_blackbox.rs

//! Black-box integration tests for the `steel` CLI binary.
//!
//! These tests exercise the compiled binary via `std::process::Command`,
//! verifying exit codes, stdout/stderr patterns, and flag parsing.
//! They do NOT import any library code — they treat the CLI as an opaque
//! executable, ensuring the user-facing contract is preserved.

use std::process::{Command, Output};

/// Build a `Command` targeting the `steel` binary with a sanitized environment.
///
/// - `STEEL_CONFIG_DIR` is set to a per-test temp directory so we never
///   touch the real user config.
/// - `STEEL_API_KEY` is explicitly removed to avoid accidental auth.
/// - `HOME` is preserved so the binary can still resolve paths if needed.
fn steel_cmd() -> (Command, tempfile::TempDir) {
    let tmp = tempfile::tempdir().expect("failed to create temp dir");
    let mut cmd = Command::new(env!("CARGO_BIN_EXE_steel"));
    cmd.env("STEEL_CONFIG_DIR", tmp.path());
    cmd.env_remove("STEEL_API_KEY");
    // Suppress update checks so they don't interfere with output
    cmd.arg("--no-update-check");
    (cmd, tmp)
}

/// Convenience: run steel with the given args and return the Output.
fn run(args: &[&str]) -> Output {
    let (mut cmd, _tmp) = steel_cmd();
    cmd.args(args);
    cmd.output().expect("failed to execute steel binary")
}

/// Convert Output stdout to a String (lossy).
fn stdout(output: &Output) -> String {
    String::from_utf8_lossy(&output.stdout).to_string()
}

/// Convert Output stderr to a String (lossy).
fn stderr(output: &Output) -> String {
    String::from_utf8_lossy(&output.stderr).to_string()
}

// ─── Version & top-level help ───────────────────────────────────────

#[test]
fn version_exits_zero_and_prints_version() {
    // --version is handled by clap before --no-update-check is parsed,
    // so we call the binary directly without the helper.
    let mut cmd = Command::new(env!("CARGO_BIN_EXE_steel"));
    cmd.arg("--version");
    let output = cmd.output().expect("failed to execute steel binary");
    assert!(output.status.success(), "steel --version should exit 0");
    let out = String::from_utf8_lossy(&output.stdout);
    assert!(
        out.contains("steel-cli") || out.contains("steel"),
        "stdout should contain the binary name, got: {out}"
    );
    // Version string from Cargo.toml
    let version = env!("CARGO_PKG_VERSION");
    assert!(
        out.contains(version),
        "stdout should contain version {version}, got: {out}"
    );
}

#[test]
fn help_exits_zero_and_mentions_steel_cli() {
    let output = run(&["--help"]);
    assert!(output.status.success(), "steel --help should exit 0");
    let out = stdout(&output);
    assert!(
        out.contains("Steel CLI"),
        "help output should contain 'Steel CLI', got: {out}"
    );
}

// ─── Subcommand help availability ───────────────────────────────────

#[test]
fn browser_help_lists_subcommands() {
    let output = run(&["browser", "--help"]);
    assert!(
        output.status.success(),
        "steel browser --help should exit 0"
    );
    let out = stdout(&output);
    for sub in &["start", "stop", "sessions", "live"] {
        assert!(
            out.contains(sub),
            "browser help should list '{sub}', got: {out}"
        );
    }
}

#[test]
fn scrape_help_shows_expected_flags() {
    let output = run(&["scrape", "--help"]);
    assert!(output.status.success(), "steel scrape --help should exit 0");
    let out = stdout(&output);
    for flag in &["url", "--format", "--delay"] {
        assert!(
            out.contains(flag),
            "scrape help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn screenshot_help_shows_expected_flags() {
    let output = run(&["screenshot", "--help"]);
    assert!(
        output.status.success(),
        "steel screenshot --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["url", "--delay", "--full-page"] {
        assert!(
            out.contains(flag),
            "screenshot help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn pdf_help_shows_expected_flags() {
    let output = run(&["pdf", "--help"]);
    assert!(output.status.success(), "steel pdf --help should exit 0");
    let out = stdout(&output);
    for flag in &["url", "--delay"] {
        assert!(
            out.contains(flag),
            "pdf help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn dev_help_lists_subcommands() {
    let output = run(&["dev", "--help"]);
    assert!(output.status.success(), "steel dev --help should exit 0");
    let out = stdout(&output);
    for sub in &["install", "start", "stop"] {
        assert!(
            out.contains(sub),
            "dev help should list '{sub}', got: {out}"
        );
    }
}

#[test]
fn credentials_help_lists_subcommands() {
    let output = run(&["credentials", "--help"]);
    assert!(
        output.status.success(),
        "steel credentials --help should exit 0"
    );
    let out = stdout(&output);
    for sub in &["create", "update", "delete", "list"] {
        assert!(
            out.contains(sub),
            "credentials help should list '{sub}', got: {out}"
        );
    }
}

#[test]
fn skills_help_lists_subcommands() {
    let output = run(&["skills", "--help"]);
    assert!(output.status.success(), "steel skills --help should exit 0");
    let out = stdout(&output);
    for sub in &["list", "install", "update", "doctor", "open", "paths"] {
        assert!(
            out.contains(sub),
            "skills help should list '{sub}', got: {out}"
        );
    }
}

#[test]
fn skills_list_offline_shows_launch_catalog() {
    let output = run(&["skills", "list", "--offline"]);
    assert!(
        output.status.success(),
        "steel skills list --offline should exit 0; stderr: {}",
        stderr(&output)
    );
    let out = stdout(&output);
    for skill in &[
        "steel-browser",
        "steel-developer",
        "steel-session-debugging",
        "steel-reliability",
        "steel-skill-creator",
    ] {
        assert!(
            out.contains(skill),
            "skills list should include '{skill}', got: {out}"
        );
    }
}

#[test]
fn profile_help_lists_subcommands() {
    let output = run(&["profile", "--help"]);
    assert!(
        output.status.success(),
        "steel profile --help should exit 0"
    );
    let out = stdout(&output);
    for sub in &["import", "sync", "list", "delete"] {
        assert!(
            out.contains(sub),
            "profile help should list '{sub}', got: {out}"
        );
    }
}

#[test]
fn sessions_help_lists_subcommands() {
    let output = run(&["sessions", "--help"]);
    assert!(
        output.status.success(),
        "steel sessions --help should exit 0"
    );
    let out = stdout(&output);
    for sub in &["list", "get", "release", "logs", "agent-logs", "traces"] {
        assert!(
            out.contains(sub),
            "sessions help should list '{sub}', got: {out}"
        );
    }
}

#[test]
fn sessions_logs_help_shows_expected_flags() {
    let output = run(&["sessions", "logs", "--help"]);
    assert!(
        output.status.success(),
        "steel sessions logs --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &[
        "--session",
        "--follow",
        "--since",
        "--event-type",
        "--limit",
        "--offset",
    ] {
        assert!(
            out.contains(flag),
            "sessions logs help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn sessions_traces_help_shows_expected_flags() {
    let output = run(&["sessions", "traces", "--help"]);
    assert!(
        output.status.success(),
        "steel sessions traces --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--session", "--follow", "--since", "--event-type"] {
        assert!(
            out.contains(flag),
            "sessions traces help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn login_help_exits_zero() {
    let output = run(&["login", "--help"]);
    assert!(output.status.success(), "steel login --help should exit 0");
}

#[test]
fn auth_alias_help_exits_zero() {
    let output = run(&["auth", "--help"]);
    assert!(
        output.status.success(),
        "steel auth --help should exit 0 (alias for login)"
    );
}

// ─── Error handling contracts ───────────────────────────────────────

#[test]
fn scrape_without_url_fails() {
    let output = run(&["scrape"]);
    assert!(
        !output.status.success(),
        "steel scrape (no URL) should exit non-zero"
    );
    let err = stderr(&output);
    let out = stdout(&output);
    let combined = format!("{err}{out}");
    // The error should mention that a URL is missing (from resolve_tool_url)
    // or be a usage error from clap. Either way, it should not silently succeed.
    assert!(
        combined.to_lowercase().contains("url")
            || combined.to_lowercase().contains("missing")
            || combined.to_lowercase().contains("argument")
            || combined.to_lowercase().contains("error"),
        "error output should mention URL or missing argument, got: {combined}"
    );
}

#[test]
fn screenshot_without_url_fails() {
    let output = run(&["screenshot"]);
    assert!(
        !output.status.success(),
        "steel screenshot (no URL) should exit non-zero"
    );
    let err = stderr(&output);
    let out = stdout(&output);
    let combined = format!("{err}{out}");
    assert!(
        combined.to_lowercase().contains("url")
            || combined.to_lowercase().contains("missing")
            || combined.to_lowercase().contains("argument")
            || combined.to_lowercase().contains("error"),
        "error output should mention URL or missing argument, got: {combined}"
    );
}

#[test]
fn pdf_without_url_fails() {
    let output = run(&["pdf"]);
    assert!(
        !output.status.success(),
        "steel pdf (no URL) should exit non-zero"
    );
}

#[test]
fn nonexistent_command_fails() {
    let output = run(&["nonexistent-command"]);
    assert!(
        !output.status.success(),
        "steel nonexistent-command should exit non-zero"
    );
    let err = stderr(&output);
    // clap should report an unrecognized subcommand
    assert!(
        !err.is_empty(),
        "stderr should contain an error message for unknown command"
    );
}

#[test]
fn browser_start_without_auth_or_local_fails() {
    // With no API key and no --local flag, `browser start` should fail
    // with the actual daemon error (missing API key), not a generic timeout.
    let start = std::time::Instant::now();
    let output = run(&["browser", "start"]);
    let elapsed = start.elapsed();

    assert!(
        !output.status.success(),
        "steel browser start without auth should exit non-zero"
    );

    let err = stderr(&output);
    let out = stdout(&output);
    let combined = format!("{err}{out}");
    let lower = combined.to_lowercase();

    // The error should surface the actual daemon failure reason (missing API key),
    // not a generic "failed to start within 30s" timeout message.
    assert!(
        lower.contains("api key") || lower.contains("api_key") || lower.contains("login"),
        "output should mention the actual auth error (API key / login), got: {combined}"
    );

    // The daemon exits almost immediately on missing auth. With early exit detection,
    // this should complete in well under 10 seconds (not the 30s timeout).
    assert!(
        elapsed.as_secs() < 15,
        "should detect daemon failure quickly, but took {elapsed:?}"
    );
}

#[test]
fn browser_start_without_auth_json_mode_surfaces_error() {
    // In --json mode, the actual daemon error should also appear in the output.
    let output = run(&["--json", "browser", "start"]);
    assert!(!output.status.success());

    let out = stdout(&output);
    let parsed: serde_json::Value = serde_json::from_str(out.trim())
        .unwrap_or_else(|_| panic!("expected valid JSON, got: {out}"));
    assert_eq!(parsed["success"], false);

    let error_msg = parsed["error"].as_str().unwrap_or("");
    let lower = error_msg.to_lowercase();
    assert!(
        lower.contains("api key") || lower.contains("api_key") || lower.contains("login"),
        "JSON error should mention the actual auth error, got: {error_msg}"
    );
}

// ─── Environment variable contracts ─────────────────────────────────

#[test]
fn steel_config_dir_is_respected() {
    let tmp = tempfile::tempdir().expect("failed to create temp dir");
    let config_dir = tmp.path().join("custom-steel-config");

    let mut cmd = Command::new(env!("CARGO_BIN_EXE_steel"));
    cmd.env("STEEL_CONFIG_DIR", &config_dir);
    cmd.env_remove("STEEL_API_KEY");
    cmd.args(["--no-update-check", "config"]);

    let output = cmd.output().expect("failed to execute steel binary");

    // The command should not fail just because the config dir is custom;
    // it either creates the dir or reports the config location.
    // We verify that no writes happen to the default ~/.config/steel
    // by checking the custom dir was used (if any files were created).
    if config_dir.exists() {
        // If files were written, they should be in our custom dir.
        let entries: Vec<_> = std::fs::read_dir(&config_dir).unwrap().collect();
        // This is a positive signal that the custom dir was used.
        // (The dir might not exist if config command only reads.)
        let _ = entries;
    }

    // Additionally, check that the output (if any) references the custom path
    // or that the command at least ran without panicking.
    let combined = format!("{}{}", stdout(&output), stderr(&output));
    // The config command should either succeed or fail gracefully
    assert!(
        output.status.success() || !combined.is_empty(),
        "steel config with custom STEEL_CONFIG_DIR should run without panic"
    );
}

// ─── Subcommand-specific help flags ─────────────────────────────────

#[test]
fn browser_start_help_shows_expected_flags() {
    let output = run(&["browser", "start", "--help"]);
    assert!(
        output.status.success(),
        "steel browser start --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--session", "--local", "--stealth", "--proxy"] {
        assert!(
            out.contains(flag),
            "browser start help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn browser_stop_help_shows_expected_flags() {
    let output = run(&["browser", "stop", "--help"]);
    assert!(
        output.status.success(),
        "steel browser stop --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--session", "--all", "--local"] {
        assert!(
            out.contains(flag),
            "browser stop help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn browser_sessions_help_shows_expected_flags() {
    let output = run(&["browser", "sessions", "--help"]);
    assert!(
        output.status.success(),
        "steel browser sessions --help should exit 0"
    );
    let out = stdout(&output);
    {
        let flag = &"--local";
        assert!(
            out.contains(flag),
            "browser sessions help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn browser_live_help_shows_expected_flags() {
    let output = run(&["browser", "live", "--help"]);
    assert!(
        output.status.success(),
        "steel browser live --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--session", "--local"] {
        assert!(
            out.contains(flag),
            "browser live help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn credentials_create_help_shows_expected_flags() {
    let output = run(&["credentials", "create", "--help"]);
    assert!(
        output.status.success(),
        "steel credentials create --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--origin", "--username", "--password", "--namespace"] {
        assert!(
            out.contains(flag),
            "credentials create help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn credentials_delete_help_shows_expected_flags() {
    let output = run(&["credentials", "delete", "--help"]);
    assert!(
        output.status.success(),
        "steel credentials delete --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--origin", "--namespace", "--local"] {
        assert!(
            out.contains(flag),
            "credentials delete help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn dev_install_help_shows_expected_flags() {
    let output = run(&["dev", "install", "--help"]);
    assert!(
        output.status.success(),
        "steel dev install --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--repo-url", "--verbose"] {
        assert!(
            out.contains(flag),
            "dev install help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn dev_start_help_shows_expected_flags() {
    let output = run(&["dev", "start", "--help"]);
    assert!(
        output.status.success(),
        "steel dev start --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--port", "--verbose", "--docker-check"] {
        assert!(
            out.contains(flag),
            "dev start help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn profile_import_help_shows_expected_flags() {
    let output = run(&["profile", "import", "--help"]);
    assert!(
        output.status.success(),
        "steel profile import --help should exit 0"
    );
    let out = stdout(&output);
    for expected in &["[NAME]", "--from"] {
        assert!(
            out.contains(expected),
            "profile import help should mention '{expected}', got: {out}"
        );
    }
}

#[test]
fn profile_sync_help_shows_expected_flags() {
    let output = run(&["profile", "sync", "--help"]);
    assert!(
        output.status.success(),
        "steel profile sync --help should exit 0"
    );
    let out = stdout(&output);
    for expected in &["[NAME]", "--from"] {
        assert!(
            out.contains(expected),
            "profile sync help should mention '{expected}', got: {out}"
        );
    }
}

#[test]
fn profile_delete_help_shows_expected_flags() {
    let output = run(&["profile", "delete", "--help"]);
    assert!(
        output.status.success(),
        "steel profile delete --help should exit 0"
    );
    let out = stdout(&output);
    assert!(
        out.contains("[NAME]"),
        "profile delete help should mention '[NAME]', got: {out}"
    );
}

#[test]
fn profile_list_help_shows_expected_flags() {
    let output = run(&["profile", "list", "--help"]);
    assert!(
        output.status.success(),
        "steel profile list --help should exit 0"
    );
    let out = stdout(&output);
    assert!(
        out.contains("--json"),
        "profile list help should mention '--json', got: {out}"
    );
}

// ─── Flag parsing edge cases ────────────────────────────────────────

#[test]
fn scrape_unknown_flag_fails() {
    let output = run(&["scrape", "--nonexistent-flag"]);
    assert!(
        !output.status.success(),
        "steel scrape --nonexistent-flag should exit non-zero"
    );
    let err = stderr(&output);
    assert!(
        err.contains("unexpected")
            || err.contains("not expected")
            || err.contains("unrecognized")
            || err.contains("unknown")
            || err.contains("found argument"),
        "stderr should indicate an unknown flag, got: {err}"
    );
}

#[test]
fn browser_without_subcommand_fails() {
    let output = run(&["browser"]);
    assert!(
        !output.status.success(),
        "steel browser (no subcommand) should exit non-zero"
    );
    let err = stderr(&output);
    // clap should show an error about missing subcommand
    assert!(
        !err.is_empty(),
        "stderr should contain usage/error info for missing subcommand"
    );
}

#[test]
fn credentials_without_subcommand_fails() {
    let output = run(&["credentials"]);
    assert!(
        !output.status.success(),
        "steel credentials (no subcommand) should exit non-zero"
    );
}

#[test]
fn dev_without_subcommand_fails() {
    let output = run(&["dev"]);
    assert!(
        !output.status.success(),
        "steel dev (no subcommand) should exit non-zero"
    );
}

#[test]
fn profile_without_subcommand_fails() {
    let output = run(&["profile"]);
    assert!(
        !output.status.success(),
        "steel profile (no subcommand) should exit non-zero"
    );
}

// ─── Exit code specifics ────────────────────────────────────────────

#[test]
fn clap_usage_error_exits_with_code_2() {
    // clap exits with code 2 for usage errors (bad flags, missing subcommand).
    let output = run(&["--this-flag-does-not-exist"]);
    assert!(
        !output.status.success(),
        "invalid global flag should exit non-zero"
    );
    let code = output.status.code().expect("should have exit code");
    assert_eq!(
        code, 2,
        "clap usage error should exit with code 2, got: {code}"
    );
}

#[test]
fn help_flag_exits_with_code_0() {
    let output = run(&["--help"]);
    let code = output.status.code().expect("should have exit code");
    assert_eq!(code, 0, "--help should exit with code 0, got: {code}");
}

// ─── Computers ──────────────────────────────────────────────────────

fn run_without_computer_context(args: &[&str]) -> Output {
    let (mut cmd, _tmp) = steel_cmd();
    cmd.env_remove("STEEL_COMPUTER_ID");
    cmd.args(args);
    cmd.output().expect("failed to execute steel binary")
}

#[test]
fn computer_help_lists_subcommands() {
    let output = run(&["computer", "--help"]);
    assert!(
        output.status.success(),
        "steel computer --help should exit 0"
    );
    let out = stdout(&output);
    for sub in &[
        "create", "list", "get", "delete", "pause", "resume", "use", "exec", "ssh",
    ] {
        assert!(
            out.contains(sub),
            "computer help should list '{sub}', got: {out}"
        );
    }
}

#[test]
fn computer_exec_help_shows_expected_flags() {
    let output = run(&["computer", "exec", "--help"]);
    assert!(
        output.status.success(),
        "steel computer exec --help should exit 0"
    );
    let out = stdout(&output);
    for flag in &["--command", "--cwd", "--env", "--timeout", "ARGV"] {
        assert!(
            out.contains(flag),
            "computer exec help should mention '{flag}', got: {out}"
        );
    }
}

#[test]
fn computer_exec_without_a_computer_explains_how_to_pick_one() {
    let output = run_without_computer_context(&["computer", "exec", "--", "ls"]);
    assert!(
        !output.status.success(),
        "exec without a computer should fail"
    );
    let text = format!("{}{}", stdout(&output), stderr(&output));
    assert!(
        text.contains("steel computer use"),
        "the error should point at `steel computer use`, got: {text}"
    );
}

#[test]
fn computer_use_without_an_id_fails() {
    let output = run_without_computer_context(&["computer", "use"]);
    assert!(!output.status.success(), "use without an id should fail");
}