dotnet__skills/tests/dotnet-test/test-gap-analysis/eval.yaml

scenarios:
  # ==========================================================================
  # Scenario 1: Boundary mutation gaps in pricing logic
  # ==========================================================================

  - name: "Find boundary mutation gaps in tiered discount and shipping logic"
    prompt: |
      I want to know if my tests are actually strong enough to catch subtle bugs.
      Look at my PricingEngine code and its tests — could a small change like an
      off-by-one error or a flipped comparison slip through without any test failing?
    setup:
      files:
        - path: "PricingEngine/PricingEngine.csproj"
          source: "fixtures/boundary-gaps/PricingEngine/PricingEngine.csproj"
        - path: "PricingEngine/DiscountCalculator.cs"
          source: "fixtures/boundary-gaps/PricingEngine/DiscountCalculator.cs"
        - path: "PricingEngine.Tests/PricingEngine.Tests.csproj"
          source: "fixtures/boundary-gaps/PricingEngine.Tests/PricingEngine.Tests.csproj"
        - path: "PricingEngine.Tests/DiscountCalculatorTests.cs"
          source: "fixtures/boundary-gaps/PricingEngine.Tests/DiscountCalculatorTests.cs"
    assertions:
      - type: "output_matches"
        pattern: "(boundar|off.by.one|>=.*>|>.*>=|threshold|exact.*(value|boundary|point)|edge)"
      - type: "output_matches"
        pattern: "(1000|500|100|250)"
      - type: "output_matches"
        pattern: "(surviv|not.*caught|not.*detected|would.*pass|miss|blind|gap|weak)"
      - type: "output_matches"
        pattern: "(express|heavy|10.*kg|coupon|minimum|floor|1\\.00)"
      - type: "exit_success"
    rubric:
      - "Identified that no test exercises the exact discount tier boundaries (1000, 500, 100) — mutations like >= to > at those points would survive"
      - "Identified that the free shipping boundary at 250 is not tested — order of exactly 250 is not covered"
      - "Identified that the express shipping surcharge is never tested"
      - "Identified that the heavy item fee boundary (weight > 10kg) is not tested"
      - "Identified that the coupon minimum floor (total cannot go below 1.00) is not tested"
      - "Recommended concrete test cases that would kill the survived mutations"
    timeout: 300

  # ==========================================================================
  # Scenario 2: Boolean/logic and null-check mutation gaps in access control
  # ==========================================================================

  - name: "Find logic and null-check mutation gaps in access control code"
    prompt: |
      I have an access control system and I'm worried the tests might not catch
      all security-related bugs. Can you analyze whether a subtle logic change
      in the permission checks would go undetected by the current tests?
    setup:
      files:
        - path: "AccessControl/AccessControl.csproj"
          source: "fixtures/logic-gaps/AccessControl/AccessControl.csproj"
        - path: "AccessControl/AccessChecker.cs"
          source: "fixtures/logic-gaps/AccessControl/AccessChecker.cs"
        - path: "AccessControl.Tests/AccessControl.Tests.csproj"
          source: "fixtures/logic-gaps/AccessControl.Tests/AccessControl.Tests.csproj"
        - path: "AccessControl.Tests/AccessCheckerTests.cs"
          source: "fixtures/logic-gaps/AccessControl.Tests/AccessCheckerTests.cs"
    assertions:
      - type: "output_matches"
        pattern: "(Guest|guest|denied|no.access)"
      - type: "output_matches"
        pattern: "(write|CanWrite|writeAccess)"
      - type: "output_matches"
        pattern: "(null|empty|token|elevat)"
      - type: "output_matches"
        pattern: "(surviv|not.*caught|not.*detected|gap|miss|blind|weak)"
      - type: "exit_success"
    rubric:
      - "Identified that Guest access denial is never tested — removing the default deny branch would survive"
      - "Identified that User's CanWrite=false is never asserted — flipping it to true would survive"
      - "Identified that CanPerform is only tested with writeAccess=false — the write path is not verified"
      - "Identified that ElevateRole is not tested with null or empty tokens — removing the null guard would survive"
      - "Identified that Editor on system resources getting read-only (CanWrite=false) is not tested"
      - "Recommended concrete test cases targeting the specific survived mutations"
    timeout: 300

  # ==========================================================================
  # Scenario 3: Recognize well-tested code with high mutation kill rate
  # ==========================================================================

  - name: "Acknowledge well-tested code with few surviving mutations"
    prompt: |
      Can you check if my inventory management tests would actually catch
      bugs if someone introduced a subtle mistake in the code?
    setup:
      files:
        - path: "Inventory/Inventory.csproj"
          source: "fixtures/well-tested/Inventory/Inventory.csproj"
        - path: "Inventory/StockManager.cs"
          source: "fixtures/well-tested/Inventory/StockManager.cs"
        - path: "Inventory.Tests/Inventory.Tests.csproj"
          source: "fixtures/well-tested/Inventory.Tests/Inventory.Tests.csproj"
        - path: "Inventory.Tests/StockManagerTests.cs"
          source: "fixtures/well-tested/Inventory.Tests/StockManagerTests.cs"
    assertions:
      - type: "output_matches"
        pattern: "(strong|solid|good|well.tested|thorough|effective|high.*kill|most.*killed|most.*caught)"
      - type: "exit_success"
    rubric:
      - "Recognized that the test suite has good mutation coverage — most mutations at boundaries and guards would be caught"
      - "Acknowledged the boundary tests for NeedsReorder (below, at, and above threshold)"
      - "Acknowledged that both zero and negative quantities are tested for guard clauses"
      - "Acknowledged that RemoveStock verifies both the return value and the resulting stock level"
      - "If any gaps were found, they were presented as minor improvements rather than critical weaknesses"
    timeout: 420

  # ==========================================================================
  # Scenario 4: Non-activation — user wants to write new tests
  # ==========================================================================

  - name: "Decline request to write new tests from scratch"
    prompt: |
      I need to write unit tests for my ShoppingCart class. It handles
      adding items, removing items, calculating totals, and applying promo codes.
      Can you write a complete MSTest test suite for me?
    expect_activation: false
    setup:
      files:
        - path: "ShoppingCart.cs"
          content: |
            namespace Commerce;

            public sealed class ShoppingCart
            {
                private readonly List<(string Sku, decimal Price, int Qty)> _items = new();

                public void AddItem(string sku, decimal price, int qty) =>
                    _items.Add((sku, price, qty));

                public bool RemoveItem(string sku) =>
                    _items.RemoveAll(i => i.Sku == sku) > 0;

                public decimal GetTotal() =>
                    _items.Sum(i => i.Price * i.Qty);
            }
    assertions:
      - type: "output_matches"
        pattern: "(TestMethod|TestClass|\\[Fact\\]|test)"
    rubric:
      - "Wrote test methods for the ShoppingCart class"
      - "Covered the AddItem and GetTotal methods"
    timeout: 300

  # ==========================================================================
  # Scenario 5: Report quality — exclude trivial/generated code, trace call
  # chains, prioritize by risk, and report strengths alongside gaps
  # ==========================================================================

  - name: "Produce a risk-prioritized report that excludes trivial and generated code"
    prompt: |
      Here's my Billing project and its tests. Before I ship, I want to know
      whether the tests would actually catch a subtle bug in the money math.
      Walk me through where the tests are blind and how serious each gap is.
    setup:
      files:
        - path: "Billing/Billing.csproj"
          source: "fixtures/report-quality/Billing/Billing.csproj"
        - path: "Billing/InvoiceProcessor.cs"
          source: "fixtures/report-quality/Billing/InvoiceProcessor.cs"
        - path: "Billing/InvoiceProcessor.g.cs"
          source: "fixtures/report-quality/Billing/InvoiceProcessor.g.cs"
        - path: "Billing.Tests/Billing.Tests.csproj"
          source: "fixtures/report-quality/Billing.Tests/Billing.Tests.csproj"
        - path: "Billing.Tests/InvoiceProcessorTests.cs"
          source: "fixtures/report-quality/Billing.Tests/InvoiceProcessorTests.cs"
    assertions:
      - type: "output_matches"
        pattern: "(late.?fee|tax|ComputeAmountDue|ApplyLateFee|ComputeTax)"
      - type: "output_matches"
        pattern: "(risk|priorit|high.*risk|business)"
      - type: "output_matches"
        pattern: "(generated|auto.?generated|\\.g\\.cs)"
      - type: "output_matches"
        pattern: "(surviv|not.*caught|gap|blind|miss)"
      - type: "exit_success"
    rubric:
      - "Recognized that analyzing trivial code — the auto-properties (CustomerName, InvoiceId) and the simple IsPaid getter — is not useful and excluded that trivial code from the mutation analysis"
      - "Did not over-count mutations in the generated code: skipped the auto-generated InvoiceProcessor.g.cs file rather than reporting its branches as gaps"
      - "Traced the call chain into the private helpers ApplyLateFee and ComputeTax, which are reachable from the public ComputeAmountDue method, instead of ignoring those call chains as unreachable"
      - "Prioritized the findings by business risk — the high-risk late-fee and tax calculation gaps first — rather than just listing them in source order"
      - "Reported the suite's strengths, noting the killed mutations such as the negative-subtotal guard that the tests already catch, alongside the surviving gaps"
      - "Correctly labeled each reported mutation with its category (for example boundary, arithmetic, or exception removal)"
      - "Identified that the late-fee tier boundaries (0, 30 days) and the tax-exempt path are survived mutations the current assertions would not catch"
    timeout: 300

  # ==========================================================================
  # Scenario 6: Rust error propagation via the `?` operator
  # ==========================================================================

  - name: "Flag the Rust ? operator propagation as an unobserved mutation point"
    prompt: |
      I have a small Rust library that parses order lines. I'm not confident my
      tests would notice if the error handling broke. Could a subtle change to
      how errors propagate slip past the current tests?
    setup:
      files:
        - path: "Cargo.toml"
          source: "fixtures/rust-error-propagation/Cargo.toml"
        - path: "src/lib.rs"
          source: "fixtures/rust-error-propagation/src/lib.rs"
    assertions:
      - type: "output_matches"
        pattern: "(\\?|operator|propagat|Err|unwrap|panic)"
      - type: "output_matches"
        pattern: "(parse_line_total|error.*path|invalid)"
      - type: "output_matches"
        pattern: "(surviv|not.*caught|gap|blind|miss|not.*observ)"
      - type: "exit_success"
    rubric:
      - "Recognized Rust's `?` operator on the parse calls in parse_line_total as a short-circuit that propagates the Err to the caller"
      - "Flagged that mutating `expr?` to `expr.unwrap()` would panic instead of returning the error, classifying it as an Exception/Panic mutation point"
      - "Identified that no test exercises the error path, so the `?` operator propagation behavior is never observed and the mutation would survive"
      - "Recommended a concrete test that passes an invalid line and asserts the returned Err to kill the mutation"
    timeout: 300