diff --git a/.github/workflows/nightly-e2e.yaml b/.github/workflows/nightly-e2e.yaml index 489cfbb0def..e9d33008399 100644 --- a/.github/workflows/nightly-e2e.yaml +++ b/.github/workflows/nightly-e2e.yaml @@ -75,7 +75,10 @@ jobs: if-no-files-found: ignore cloud-experimental-e2e: - if: github.repository == 'NVIDIA/NemoClaw' + # DISABLED: CLI/docs command reference drift causes check-docs failures. + # Landlock fix landed in OpenShell#810 (v0.0.32+). Re-enable once + # docs-drift is resolved. + if: github.repository == 'NVIDIA/NemoClaw' && vars.CLOUD_EXPERIMENTAL_E2E_ENABLED == 'true' runs-on: ubuntu-latest # Main suite + check-docs + network-policy skip script can exceed 45m on cold runners. timeout-minutes: 90 @@ -430,6 +433,7 @@ jobs: NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }} NEMOCLAW_NON_INTERACTIVE: "1" NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1" + NEMOCLAW_SANDBOX_NAME: "e2e-deploy-svc" run: bash test/e2e/test-deployment-services.sh - name: Upload test log on failure @@ -663,7 +667,27 @@ jobs: notify-on-failure: runs-on: ubuntu-latest - needs: [cloud-e2e, cloud-experimental-e2e, messaging-providers-e2e, token-rotation-e2e, sandbox-survival-e2e, hermes-e2e, skip-permissions-e2e, sandbox-operations-e2e, inference-routing-e2e, network-policy-e2e, deployment-services-e2e, diagnostics-e2e, snapshot-commands-e2e, shields-config-e2e, rebuild-openclaw-e2e, upgrade-stale-sandbox-e2e, rebuild-hermes-e2e, gpu-e2e] + needs: + [ + cloud-e2e, + cloud-experimental-e2e, + messaging-providers-e2e, + token-rotation-e2e, + sandbox-survival-e2e, + hermes-e2e, + skip-permissions-e2e, + sandbox-operations-e2e, + inference-routing-e2e, + network-policy-e2e, + deployment-services-e2e, + diagnostics-e2e, + snapshot-commands-e2e, + shields-config-e2e, + rebuild-openclaw-e2e, + upgrade-stale-sandbox-e2e, + rebuild-hermes-e2e, + gpu-e2e, + ] if: ${{ always() && (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) }} permissions: issues: write diff --git a/test/e2e/test-deployment-services.sh b/test/e2e/test-deployment-services.sh index 4aa8459053e..391528cbbce 100755 --- a/test/e2e/test-deployment-services.sh +++ b/test/e2e/test-deployment-services.sh @@ -60,6 +60,10 @@ skip() { echo -e "${YELLOW} SKIP${NC} $1 — $2" | tee -a "$LOG_FILE" } +# ── Config ─────────────────────────────────────────────────────────────────── +SANDBOX_NAME="${NEMOCLAW_SANDBOX_NAME:-e2e-deploy-svc}" +LOG_FILE="test-deployment-services-$(date +%Y%m%d-%H%M%S).log" + # ── Resolve repo root ──────────────────────────────────────────────────────── REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" @@ -124,7 +128,8 @@ preflight() { return 0 ;; esac - if curl -fsSL "https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${arch}" -o /tmp/cloudflared \ + local cf_url="${CLOUDFLARED_DOWNLOAD_URL:-https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-${arch}}" + if curl -fsSL "$cf_url" -o /tmp/cloudflared \ && chmod +x /tmp/cloudflared \ && sudo mv /tmp/cloudflared /usr/local/bin/cloudflared 2>/dev/null; then log "cloudflared installed" diff --git a/test/e2e/test-network-policy.sh b/test/e2e/test-network-policy.sh index 51af135325e..65c30b20032 100755 --- a/test/e2e/test-network-policy.sh +++ b/test/e2e/test-network-policy.sh @@ -213,6 +213,7 @@ setup_sandbox() { NEMOCLAW_NON_INTERACTIVE=1 \ NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE=1 \ NEMOCLAW_POLICY_TIER="restricted" \ + NEMOCLAW_RECREATE_SANDBOX=1 \ run_with_timeout 600 nemoclaw onboard --non-interactive --yes-i-accept-third-party-software \ 2>&1 | tee -a "$LOG_FILE" || { log "FATAL: Onboard failed" diff --git a/test/e2e/test-snapshot-commands.sh b/test/e2e/test-snapshot-commands.sh index b11158bc295..b1568488d4a 100755 --- a/test/e2e/test-snapshot-commands.sh +++ b/test/e2e/test-snapshot-commands.sh @@ -41,16 +41,44 @@ YELLOW='\033[1;33m' NC='\033[0m' pass() { echo -e "${GREEN}[PASS]${NC} $1"; } + +# Shared diagnostics — called by fail() and Phase 2b. +# Intentionally non-reentrant (single-threaded bash). +dump_diagnostics() { + local _fd="${1:-2}" # default to stderr + echo -e "${YELLOW}[DIAG]${NC} --- Diagnostics ---" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} nemoclaw path: $(command -v nemoclaw 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} nemoclaw version: $(nemoclaw --version 2>&1 || echo 'failed')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} node version: $(node --version 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Sandboxes: $(openshell sandbox list 2>&1 || echo 'unavailable')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Backup dir: $(ls -la "$HOME/.nemoclaw/rebuild-backups/${SANDBOX_NAME}/" 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Registry: $(cat "$HOME/.nemoclaw/sandboxes.json" 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Registry lock: $(ls -la "$HOME/.nemoclaw/sandboxes.json.lock" 2>&1 || echo 'no lock')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Config dir: $(ls -la "$HOME/.nemoclaw/" 2>&1 || echo 'not found')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} Docker ps: $(docker ps --format '{{.Names}} {{.Status}}' 2>&1 || echo 'unavailable')" >&"$_fd" + echo -e "${YELLOW}[DIAG]${NC} --- End diagnostics ---" >&"$_fd" +} + fail() { echo -e "${RED}[FAIL]${NC} $1" >&2 - echo -e "${YELLOW}[DIAG]${NC} --- Failure diagnostics ---" >&2 - echo -e "${YELLOW}[DIAG]${NC} Sandboxes: $(openshell sandbox list 2>&1 || echo 'unavailable')" >&2 - echo -e "${YELLOW}[DIAG]${NC} Backup dir: $(ls -la "$HOME/.nemoclaw/rebuild-backups/${SANDBOX_NAME}/" 2>&1 || echo 'not found')" >&2 - echo -e "${YELLOW}[DIAG]${NC} --- End diagnostics ---" >&2 + dump_diagnostics 2 exit 1 } info() { echo -e "${YELLOW}[INFO]${NC} $1"; } +# Run a command, capture its output and exit code without set -e killing us. +# Usage: run_capture VAR_NAME command [args...] +# Sets $VAR_NAME to the combined stdout+stderr and $_CAPTURE_RC to the exit code. +_CAPTURE_RC=0 +run_capture() { + local _var_name="$1" + shift + _CAPTURE_RC=0 + local _output + _output=$("$@" 2>&1) || _CAPTURE_RC=$? + printf -v "$_var_name" '%s' "$_output" +} + # ── Preflight ─────────────────────────────────────────────────────── [ -n "${NVIDIA_API_KEY:-}" ] || fail "NVIDIA_API_KEY is required" [ "${NEMOCLAW_NON_INTERACTIVE:-}" = "1" ] || fail "NEMOCLAW_NON_INTERACTIVE=1 is required" @@ -103,12 +131,24 @@ VERIFY=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" pass "Marker file written" +# ── Phase 2b: Pre-snapshot diagnostics ───────────────────────────── +# Collect state that helps diagnose Phase 3 failures (see #2350). +info "Phase 2b: Pre-snapshot diagnostics..." +dump_diagnostics 1 # stdout — informational, not a failure + # ── Phase 3: snapshot create ──────────────────────────────────────── info "Phase 3: Creating snapshot..." -SNAPSHOT_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot create 2>&1) +# Use run_capture to prevent set -e from swallowing error output. +# Previously, $(nemoclaw ... 2>&1) would exit the script immediately on +# failure, hiding the actual error message. See #2350. +run_capture SNAPSHOT_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot create echo "$SNAPSHOT_OUTPUT" +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot create exited with code $_CAPTURE_RC: ${SNAPSHOT_OUTPUT}" +fi + # The success marker is `✓ Snapshot v created ( directories)` — the # version token between "Snapshot" and "created" broke the old literal grep # for "Snapshot created". Use a regex that tolerates the version field. @@ -125,9 +165,13 @@ info "Snapshot path: ${SNAPSHOT_PATH:-unknown}" # ── Phase 4: snapshot list ────────────────────────────────────────── info "Phase 4: Listing snapshots..." -LIST_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot list 2>&1) +run_capture LIST_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot list echo "$LIST_OUTPUT" +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot list exited with code $_CAPTURE_RC: ${LIST_OUTPUT}" +fi + if echo "$LIST_OUTPUT" | grep -q "snapshot(s)"; then pass "snapshot list shows snapshots" else @@ -150,7 +194,10 @@ openshell sandbox exec --name "${SANDBOX_NAME}" -- \ GONE=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || echo "GONE") [ "$GONE" = "GONE" ] || fail "First marker should be deleted but got: ${GONE}" -nemoclaw "${SANDBOX_NAME}" snapshot create >/dev/null 2>&1 || fail "Second snapshot create failed" +run_capture _SECOND_SNAP nemoclaw "${SANDBOX_NAME}" snapshot create +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "Second snapshot create failed (code $_CAPTURE_RC): ${_SECOND_SNAP}" +fi pass "State modified, second snapshot created" # Perturb workspace so restore has to do real work @@ -161,11 +208,15 @@ openshell sandbox exec --name "${SANDBOX_NAME}" -- \ # ── Phase 6: snapshot restore (latest) ────────────────────────────── info "Phase 6: Restoring latest snapshot..." -RESTORE_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot restore 2>&1) +run_capture RESTORE_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot restore echo "$RESTORE_OUTPUT" +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot restore exited with code $_CAPTURE_RC: ${RESTORE_OUTPUT}" +fi + if ! echo "$RESTORE_OUTPUT" | grep -q "Restored"; then - fail "snapshot restore failed: ${RESTORE_OUTPUT}" + fail "snapshot restore did not report success: ${RESTORE_OUTPUT}" fi SECOND_CHECK=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${SECOND_MARKER}" 2>/dev/null || echo "MISSING") @@ -175,11 +226,15 @@ pass "Latest snapshot restored expected state" # ── Phase 7: snapshot restore with timestamp (first snapshot) ─────── info "Phase 7: Restoring first snapshot by timestamp..." -TARGETED_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot restore "${SNAPSHOT_TIMESTAMP}" 2>&1) +run_capture TARGETED_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot restore "${SNAPSHOT_TIMESTAMP}" echo "$TARGETED_OUTPUT" +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "targeted snapshot restore exited with code $_CAPTURE_RC: ${TARGETED_OUTPUT}" +fi + if ! echo "$TARGETED_OUTPUT" | grep -q "Restored"; then - fail "Targeted snapshot restore failed: ${TARGETED_OUTPUT}" + fail "targeted snapshot restore did not report success: ${TARGETED_OUTPUT}" fi FIRST_CHECK=$(openshell sandbox exec --name "${SANDBOX_NAME}" -- cat "${MARKER_FILE}" 2>/dev/null || echo "MISSING") @@ -206,7 +261,10 @@ fi # ── Phase 9: snapshot help ────────────────────────────────────────── info "Phase 9: Verifying snapshot help output..." -HELP_OUTPUT=$(nemoclaw "${SANDBOX_NAME}" snapshot 2>&1) +run_capture HELP_OUTPUT nemoclaw "${SANDBOX_NAME}" snapshot +if [ "$_CAPTURE_RC" -ne 0 ]; then + fail "snapshot help exited with code $_CAPTURE_RC: ${HELP_OUTPUT}" +fi if echo "$HELP_OUTPUT" | grep -q "snapshot create" \ && echo "$HELP_OUTPUT" | grep -q "snapshot list" \ && echo "$HELP_OUTPUT" | grep -q "snapshot restore"; then