copilotkit__copilotkit/showcase/integrations/ms-agent-python/tests/e2e/beautiful-chat.spec.ts

import { test, expect } from "@playwright/test";

test.describe("Beautiful Chat", () => {
  test.beforeEach(async ({ page }) => {
    await page.goto("/demos/beautiful-chat");
    // Wait for a suggestion pill to render before dispatching clicks —
    // otherwise the click can race hydration and silently no-op. Picking
    // any visible pill as the readiness signal works because all 9 pills
    // mount in the same render pass.
    await expect(
      page.getByRole("button", { name: "Toggle Theme (Frontend Tools)" }),
    ).toBeVisible({ timeout: 15000 });
  });

  test("page loads with logo, mode toggle, and chat input", async ({
    page,
  }) => {
    // CopilotKit logo (top-left of the chat pane)
    await expect(page.locator('img[alt="CopilotKit"]')).toBeVisible();

    // Mode toggle (Chat / App pills, fixed top-right). Use role=button + exact
    // name to disambiguate from other occurrences of the word "Chat".
    await expect(
      page.getByRole("button", { name: "Chat", exact: true }),
    ).toBeVisible();
    await expect(
      page.getByRole("button", { name: "App", exact: true }),
    ).toBeVisible();

    // CopilotChat input is rendered. CopilotKit's default chat input uses a
    // textarea with placeholder "Type a message" across all v2 demos.
    await expect(page.getByPlaceholder("Type a message")).toBeVisible();
  });

  test("all 9 suggestion pills render with verbatim titles", async ({
    page,
  }) => {
    const expectedPills = [
      "Pie Chart (Controlled Generative UI)",
      "Bar Chart (Controlled Generative UI)",
      "Schedule Meeting (Human In The Loop)",
      "Search Flights (A2UI Fixed Schema)",
      "Sales Dashboard (A2UI Dynamic)",
      "Excalidraw Diagram (MCP App)",
      "Calculator App (Open Generative UI)",
      "Toggle Theme (Frontend Tools)",
      "Task Manager (Shared State)",
    ];

    for (const title of expectedPills) {
      // Suggestions render as buttons containing the verbatim title text.
      await expect(page.getByRole("button", { name: title })).toBeVisible({
        timeout: 15000,
      });
    }
  });

  test("Toggle Theme pill flips the html class and runs the toggleTheme tool", async ({
    page,
  }) => {
    // "Toggle Theme" is the fastest round-trip: a single frontend tool call,
    // no chart rendering. Its aimock fixture (userMessage keyword "toggle")
    // returns a toggleTheme tool call.
    const html = page.locator("html");
    const initialClass = (await html.getAttribute("class")) ?? "";
    const initiallyDark = initialClass.includes("dark");

    await page
      .getByRole("button", { name: "Toggle Theme (Frontend Tools)" })
      .click();

    // Round-trip signal: the html `dark` class flips — proves both that the
    // agent responded AND that the frontend tool fired. The beautiful-chat
    // demo does not emit `[data-testid="copilot-assistant-message"]` on its chat turns (tool
    // calls render in-transcript without a text bubble), so we assert on the
    // tool's observable side effect instead of a chat-bubble selector.
    await expect
      .poll(
        async () => {
          const cls = (await html.getAttribute("class")) ?? "";
          return cls.includes("dark");
        },
        { timeout: 30000 },
      )
      .toBe(!initiallyDark);
  });

  test("Pie Chart pill renders a donut SVG with slice circles", async ({
    page,
  }) => {
    await page
      .getByRole("button", { name: "Pie Chart (Controlled Generative UI)" })
      .click();

    // The PieChart component renders an inline <svg> with one background
    // <circle> plus one <circle> per data slice
    // (components/generative-ui/charts/pie-chart.tsx). The aimock fixture for
    // "revenue distribution by category" returns 4 slices, so wait for at
    // least 5 circles total (background + 4 slices).
    const circles = page.locator("svg circle");
    await expect
      .poll(async () => await circles.count(), { timeout: 45000 })
      .toBeGreaterThanOrEqual(3);

    // Legend rows include a percentage ending in "%".
    await expect(page.getByText(/\d+%/).first()).toBeVisible({ timeout: 5000 });
  });

  test("Bar Chart pill renders a recharts bar chart with rectangles", async ({
    page,
  }) => {
    await page
      .getByRole("button", { name: "Bar Chart (Controlled Generative UI)" })
      .click();

    // Recharts renders bars inside a ResponsiveContainer. The root class is
    // stable across recharts versions.
    const barChartRoot = page.locator(".recharts-responsive-container").first();
    await expect(barChartRoot).toBeVisible({ timeout: 45000 });

    // At least 2 bar rectangles should render.
    const bars = page.locator(".recharts-bar-rectangle");
    await expect
      .poll(async () => await bars.count(), { timeout: 15000 })
      .toBeGreaterThanOrEqual(2);
  });

  test("Search Flights pill renders FlightCard surface from A2UI fixed schema", async ({
    page,
  }) => {
    test.setTimeout(120_000);
    // Backend: search_flights tool emits an a2ui_operations container with one
    // FlightCard component per flight. The agent (`src/agents/beautiful_chat.py`
    // `_build_flight_components`) emits literal-children components rather than
    // the structural-children template form, because the binder's structural
    // expansion isn't reliably exercised by sibling demos. Aimock returns 2
    // flights — United at $349 and Delta at $289.
    //
    // Visual fingerprint: the airline names and prices are inlined into each
    // FlightCard so they appear as literal text.
    const pill = page.getByRole("button", {
      name: "Search Flights (A2UI Fixed Schema)",
    });
    await expect(pill).toBeVisible({ timeout: 15_000 });
    await pill.click();

    // 60s budget: tool call + a2ui_operations round-trip can be slow on cold
    // starts. Assertion targets are aimock-fixture text, not LLM output, so
    // they're stable across runs.
    await expect(page.getByText("United Airlines").first()).toBeVisible({
      timeout: 60_000,
    });
    await expect(page.getByText("Delta").first()).toBeVisible({
      timeout: 5_000,
    });
    await expect(page.getByText("$349").first()).toBeVisible({
      timeout: 5_000,
    });
    await expect(page.getByText("$289").first()).toBeVisible({
      timeout: 5_000,
    });
  });

  test("Sales Dashboard pill renders A2UI dashboard surface", async ({
    page,
  }) => {
    test.setTimeout(180_000);
    // Backend: generate_a2ui tool calls a secondary LLM bound to
    // `_design_a2ui_surface` (renamed from `render_a2ui` to avoid the A2UI
    // middleware's default tool-call intercept on `render_a2ui`);
    // both calls hit aimock fixtures
    // (showcase/aimock/feature-parity.json — userMessage + toolName matchers
    // differentiate primary vs secondary calls; a toolCallId match breaks the
    // post-tool loop). The render_a2ui fixture ships a 3-metric + 2-chart
    // dashboard tree against `copilotkit://app-dashboard-catalog`.
    //
    // Visual fingerprint: a Metric label "Total Revenue", plus a recharts
    // ResponsiveContainer (the Pie/BarChart custom renderers wrap their
    // recharts content in one).
    const pill = page.getByRole("button", {
      name: "Sales Dashboard (A2UI Dynamic)",
    });
    await expect(pill).toBeVisible({ timeout: 15_000 });
    await pill.click();

    // 90s budget: secondary-LLM stage inside generate_a2ui can stall on cold
    // starts. The fixture chain (feature-parity.json) returns both a
    // generate_a2ui tool call and final narration text mentioning "Total
    // Revenue". When the A2UI middleware is active AND the secondary LLM
    // fixture fires, the dashboard renders as an A2UI surface with recharts
    // charts. When running against aimock without the full A2UI pipeline
    // (e.g. the secondary-LLM fixture doesn't fire), only the narration
    // text renders. Assert on the narration text as the primary signal, and
    // treat recharts rendering as a bonus (soft assertion).
    await expect(page.getByText(/Total Revenue/i).first()).toBeVisible({
      timeout: 90_000,
    });

    // Soft assertion: if the full A2UI pipeline fires, recharts containers
    // should appear. When running against aimock-only (no secondary LLM),
    // only the text narration renders — so we don't hard-fail on missing
    // charts. The recharts check still catches regressions when the A2UI
    // pipeline IS active.
    await expect(page.getByText("$1.2M").first()).toBeVisible({
      timeout: 15_000,
    });

    const chartRoot = page.locator(".recharts-responsive-container").first();
    await expect(chartRoot).toBeVisible({ timeout: 15_000 });
    const chartsRendered = await chartRoot
      .isVisible({ timeout: 15_000 })
      .catch(() => false);

    if (chartsRendered) {
      // Regression guard (#4733 / #4734): the deployed Sales Dashboard used
      // to surface "A2UI render error: Catalog not found: ...". Assert the
      // error string is absent so any future revert trips this test.
      await expect(page.getByText(/Catalog not found/i)).toHaveCount(0);
      await expect(
        page.getByText(/Cannot create component .* without a type/i),
      ).toHaveCount(0);

      // Regression guard: only ONE dashboard surface should render.
      const allCharts = page.locator(".recharts-responsive-container");
      await expect
        .poll(async () => await allCharts.count(), { timeout: 5_000 })
        .toBeLessThanOrEqual(2); // 1 pie + 1 bar = 2 charts
    }
  });

  test("Task Manager pill streams 3 todos into the shared-state canvas", async ({
    page,
  }) => {
    test.setTimeout(120_000);

    await page
      .getByRole("button", { name: "Task Manager (Shared State)" })
      .click();

    const todoColumn = page.locator('section[aria-label="To Do column"]');
    await expect(todoColumn).toBeVisible({ timeout: 60_000 });

    await expect(page.getByText("Read the CopilotKit docs")).toBeVisible({
      timeout: 60_000,
    });
    await expect(page.getByText("Build a CopilotKit prototype")).toBeVisible({
      timeout: 5_000,
    });
    await expect(page.getByText("Explore shared agent state")).toBeVisible({
      timeout: 5_000,
    });
  });

  test("Sales Dashboard pill after another pill — A2UI surface still mounts (turnIndex regression)", async ({
    page,
  }) => {
    // Regression guard: the Sales Dashboard fixture chain used to gate the
    // leg-2 `generate_a2ui` call on `turnIndex: 1`. That fixture matcher
    // counts assistant messages in the WHOLE thread, so clicking ANY pill
    // before Sales Dashboard pushed the count past 1 → the matcher silently
    // missed → `generate_a2ui` never fired → no A2UI surface rendered (only
    // the toolCallId-keyed final-narration fixture's text appeared). Replaced
    // with a `userMessage` + `hasToolResult: true` + `toolName: query_data`
    // triple in feature-parity.json so the matcher is order-independent.
    // See showcase/RUNBOOK.md "Do not use `turnIndex` in new fixtures."
    test.setTimeout(180_000);

    // Click a cheap first pill to advance turnIndex past 1.
    await page
      .getByRole("button", { name: "Toggle Theme (Frontend Tools)" })
      .click();
    await expect(
      page.locator('[data-testid="copilot-assistant-message"]').first(),
    ).toBeVisible({ timeout: 30_000 });

    // Now click Sales Dashboard. With the turnIndex fix in place, the A2UI
    // surface must still mount. Without the fix, only the final narration
    // text appears and no surface element is in the DOM.
    await page
      .getByRole("button", { name: "Sales Dashboard (A2UI Dynamic)" })
      .click();

    // Surface mount signal: A2UI activity renderer drops a
    // `[data-surface-id]` (or sibling) attribute somewhere in the DOM tree
    // for any rendered surface.
    const surface = page
      .locator(
        '[data-surface-id], [data-a2ui-surface], [data-testid*="surface"]',
      )
      .first();
    await expect(surface).toBeVisible({ timeout: 120_000 });

    // Narration text should also appear (this used to pass on its own and
    // mask the broken surface — keep it asserted alongside the surface
    // check so the test catches both modes).
    await expect(page.getByText(/Total Revenue/i).first()).toBeVisible({
      timeout: 5_000,
    });
  });

  test("Pie Chart then Search Flights keeps the A2UI FlightCard surface working", async ({
    page,
  }) => {
    test.setTimeout(180_000);

    await page
      .getByRole("button", { name: "Pie Chart (Controlled Generative UI)" })
      .click();

    await expect(
      page.getByText(/Pie chart rendered above/i).first(),
    ).toBeVisible({ timeout: 60_000 });

    await page
      .getByRole("button", { name: "Search Flights (A2UI Fixed Schema)" })
      .click();

    await expect(page.getByText("United Airlines").first()).toBeVisible({
      timeout: 60_000,
    });
    await expect(page.getByText("Delta").first()).toBeVisible({
      timeout: 5_000,
    });
    await expect(page.getByText("$349").first()).toBeVisible({
      timeout: 5_000,
    });
    await expect(
      page.getByText(/Two flights shown above/i).first(),
    ).toBeVisible({ timeout: 15_000 });
  });

  test("Search Flights then Sales Dashboard renders the dashboard UI, not text only", async ({
    page,
  }) => {
    test.setTimeout(240_000);

    await page
      .getByRole("button", { name: "Search Flights (A2UI Fixed Schema)" })
      .click();

    await expect(page.getByText("United Airlines").first()).toBeVisible({
      timeout: 60_000,
    });
    await expect(
      page.getByText(/Two flights shown above/i).first(),
    ).toBeVisible({ timeout: 15_000 });

    await page
      .getByRole("button", { name: "Sales Dashboard (A2UI Dynamic)" })
      .click();

    await expect(page.getByText(/Total Revenue/i).first()).toBeVisible({
      timeout: 120_000,
    });
    await expect(page.getByText("$1.2M").first()).toBeVisible({
      timeout: 15_000,
    });
    await expect(page.getByText(/New Customers/i).first()).toBeVisible({
      timeout: 5_000,
    });

    await expect(page.getByText(/Catalog not found/i)).toHaveCount(0);
    await expect(
      page.getByText(/Cannot create component .* without a type/i),
    ).toHaveCount(0);

    await expect
      .poll(
        async () =>
          await page.locator(".recharts-responsive-container").count(),
        { timeout: 15_000 },
      )
      .toBeGreaterThanOrEqual(2);
  });
});