copilotkit__copilotkit/showcase/packages/langgraph-python/tests/e2e/agentic-chat-reasoning.spec.ts

import { test, expect } from "@playwright/test";

// QA reference: qa/agentic-chat-reasoning.md
// Demo source: src/app/demos/agentic-chat-reasoning/{page.tsx, reasoning-block.tsx}
//
// The demo mounts a custom `reasoningMessage` slot (`ReasoningBlock`) that
// renders an amber banner with `data-testid="reasoning-block"`. The label
// inside the banner reads "Thinking…" while the agent is streaming, then
// flips to "Agent reasoning" once streaming settles. The backend is built
// on `deepagents.create_deep_agent` with `gpt-4o-mini`.
//
// SCOPE NOTE: end-to-end reasoning assertions against Railway are currently
// unstable — the `reasoning_agent` graph on the deployed Railway image did
// NOT produce a reasoning-block OR an assistant text bubble within 60s on
// three consecutive attempts during test authoring (see W8 bug note in
// docs/superpowers/plans/langgraph-python-column-wave1-bugs-scratch.md).
// This spec intentionally stops at "UI mounts and accepts a message"; the
// richer streaming / label-flip / final-answer assertions from the QA
// checklist live here as `.skip` so the spec documents the intended
// coverage without being flaky.
//
// Selectors are testid / role / stable text only. No LLM-text assertions.

test.describe("Agentic Chat (Reasoning)", () => {
  // Reasoning demo on Railway can take a while to respond; give individual
  // tests a comfortable envelope without being unbounded.
  test.setTimeout(120_000);

  test.beforeEach(async ({ page }) => {
    await page.goto("/demos/agentic-chat-reasoning");
  });

  test("page loads with chat input", async ({ page }) => {
    await expect(page.getByPlaceholder("Type a message")).toBeVisible();
  });

  test("send button is visible alongside the input", async ({ page }) => {
    await expect(page.getByPlaceholder("Type a message")).toBeVisible();
    await expect(
      page.locator('[data-testid="copilot-send-button"]').first(),
    ).toBeVisible();
  });

  test("typing a prompt and submitting does not crash the UI", async ({
    page,
  }) => {
    const input = page.getByPlaceholder("Type a message");
    await input.fill("Say hello in one short sentence.");
    await page.locator('[data-testid="copilot-send-button"]').first().click();

    // User-side: the textarea is cleared on submit by CopilotChat. That
    // alone proves the send pipeline ran — no LLM round-trip required.
    await expect(input).toHaveValue("", { timeout: 10_000 });
  });

  // --- Reasoning-block streaming coverage ---------------------------------
  // These assert the actual reasoning UX described in qa/agentic-chat-reasoning.md.
  // Currently .skip pending Railway `reasoning_agent` responding within 60s;
  // un-skip once the graph streams reliably.
  test.skip("multi-step prompt renders a reasoning-block", async ({ page }) => {
    const input = page.getByPlaceholder("Type a message");
    await input.fill(
      "If a train leaves Boston at 3pm going 60mph and another leaves NY at 4pm going 80mph, when do they meet? Explain your approach.",
    );
    await page.locator('[data-testid="copilot-send-button"]').first().click();

    const reasoningBlock = page
      .locator('[data-testid="reasoning-block"]')
      .first();
    await expect(reasoningBlock).toBeVisible({ timeout: 60000 });
    await expect(
      reasoningBlock.getByText("Reasoning", { exact: true }),
    ).toBeVisible({ timeout: 10000 });
  });

  test.skip("reasoning-block label flips from Thinking to Agent reasoning", async ({
    page,
  }) => {
    const input = page.getByPlaceholder("Type a message");
    await input.fill(
      "Think step-by-step: what is 17 times 23? Show your work before giving the final number.",
    );
    await page.locator('[data-testid="copilot-send-button"]').first().click();

    const reasoningBlock = page
      .locator('[data-testid="reasoning-block"]')
      .first();
    await expect(reasoningBlock).toBeVisible({ timeout: 60000 });
    await expect(reasoningBlock.getByText("Agent reasoning")).toBeVisible({
      timeout: 90000,
    });
  });

  test.skip("reasoning-block accumulates italic reasoning content", async ({
    page,
  }) => {
    const input = page.getByPlaceholder("Type a message");
    await input.fill(
      "Think step-by-step about 12 plus 30, then give me only the final number.",
    );
    await page.locator('[data-testid="copilot-send-button"]').first().click();

    const reasoningBlock = page
      .locator('[data-testid="reasoning-block"]')
      .first();
    await expect(reasoningBlock).toBeVisible({ timeout: 60000 });
    // ReasoningBlock renders content inside a div with italic class when
    // `message.content` is non-empty (see reasoning-block.tsx).
    await expect(reasoningBlock.locator(".italic")).toBeVisible({
      timeout: 45000,
    });
  });
});