mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
Amaury Levé
ce75c35569
Several dotnet-test eval scenarios repeatedly hit their wall-clock timeout in the evaluation dashboard (skilled + plugin arms), producing empty output and failing all assertions. Increase per-scenario timeouts for the ones that timed out most often across recent runs: - writing-mstest-tests: 'Write unit tests for a service class', 'Write data-driven tests for a calculator', 'Use string assertions for format validation' 180->360; 'Use comparison assertions for boundary testing' 120->240 - coverage-analysis: 'Coverage plateau diagnosis', 'Project-wide coverage analysis with existing Cobertura data' 300->480 - test-gap-analysis: 'Decline request to write new tests from scratch' 120->300; 'Acknowledge well-tested code with few surviving mutations' 300->420 - run-tests: 'Filter xUnit v3 tests by trait on MTP', 'Filter xUnit v3 tests by class pattern and trait using query filter language' 120->240 Per eng/skill-validator/src/docs/InvestigatingResults.md, timeouts are the highest-priority failure mode; code-generation scenarios often need 360s. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
422 lines
19 KiB
YAML
422 lines
19 KiB
YAML
scenarios:
|
|
# ============================================================================
|
|
# Goal 1: Calling dotnet test correctly
|
|
# ============================================================================
|
|
|
|
- name: "Run tests in a VSTest MSTest project"
|
|
prompt: "I need to run all the tests in this project. Can you help me? Show me the exact command to use."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-mstest/TestProject.csproj"
|
|
- path: "OrderServiceTests.cs"
|
|
source: "fixtures/vstest-mstest/OrderServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Identifies the project as a VSTest project (Microsoft.NET.Test.Sdk + MSTest.TestAdapter)"
|
|
- "Uses the correct `dotnet test` invocation for the detected platform and SDK version — a plain VSTest invocation with no MTP-specific arguments or '--' separator"
|
|
- "Does not suggest MTP-style arguments like --report-trx or --treenode-filter"
|
|
expect_tools: ["bash"]
|
|
timeout: 180
|
|
|
|
- name: "Run tests with trx reporting on MTP project (SDK 9)"
|
|
prompt: "I have an MSTest project using Microsoft.Testing.Platform. I need to run the tests and produce a TRX report. I'm on .NET SDK 9. Show me the exact command."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-mstest-sdk9/TestProject.csproj"
|
|
- path: "UserAuthTests.cs"
|
|
source: "fixtures/mtp-mstest-sdk9/UserAuthTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-mstest-sdk9/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "--\\s+--report-trx"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects TestingPlatformDotnetTestSupport property in the csproj indicating MTP"
|
|
- "Recognizes that SDK 9 requires MTP arguments after the '--' separator"
|
|
- "Uses '-- --report-trx' (with separator) instead of just '--report-trx'"
|
|
- "Does not suggest the VSTest-style '--logger trx' for an MTP project"
|
|
expect_tools: ["bash"]
|
|
timeout: 480
|
|
|
|
- name: "Run tests with blame-hang on MTP project (SDK 10)"
|
|
prompt: "My tests are occasionally hanging. I want to run them with dotnet test using a hang timeout of 5 minutes to detect the culprit. Here's my project. Show me the exact command."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-mstest-sdk10/TestProject.csproj"
|
|
- path: "CatalogServiceTests.cs"
|
|
source: "fixtures/mtp-mstest-sdk10/CatalogServiceTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-mstest-sdk10/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "blame-hang"
|
|
- type: "output_not_matches"
|
|
pattern: "-- --blame"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects MTP via the global.json runner setting (Microsoft.Testing.Platform)"
|
|
- "Recognizes that SDK 10 passes MTP arguments directly without '--' separator"
|
|
- "Uses '--blame-hang-timeout 5min' or similar directly, not after '--'"
|
|
- "Does not use the VSTest-style '--blame --blame-hang-timeout' combination"
|
|
expect_tools: ["bash"]
|
|
timeout: 420
|
|
|
|
- name: "Run tests on a specific TFM with TRX in a multi-TFM MTP project (SDK 9)"
|
|
prompt: "This MSTest project uses Microsoft.Testing.Platform and targets both net8.0 and net9.0. I'm on .NET SDK 9. I want to run only the net9.0 tests and produce a TRX report saved as net9-results.trx. Show me the exact command."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/multi-tfm-mtp-sdk9/TestProject.csproj"
|
|
- path: "global.json"
|
|
source: "fixtures/multi-tfm-mtp-sdk9/global.json"
|
|
- path: "UserAuthTests.cs"
|
|
source: "fixtures/multi-tfm-mtp-sdk9/UserAuthTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test[^\\r\\n]*--framework\\s+net9\\.0[^\\r\\n]*--\\s+--report-trx"
|
|
- type: "output_matches"
|
|
pattern: "--report-trx-filename\\s+\"?net9-results\\.trx"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Runs only the net9.0 target framework rather than both TFMs"
|
|
- "Identifies that the project uses MTP (TestingPlatformDotnetTestSupport) with SDK 9 argument-separator behavior"
|
|
- "Keeps '--framework net9.0' before the '--' separator (built-in dotnet test flag)"
|
|
- "Passes BOTH MTP extension flags after the single '--' separator: '--report-trx' and '--report-trx-filename net9-results.trx' — it does not split them across the separator or drop the '--report-trx-filename' extension"
|
|
- "Avoids VSTest-style TRX syntax such as '--logger trx'"
|
|
expect_tools: ["bash"]
|
|
timeout: 480
|
|
|
|
# ============================================================================
|
|
# Goal 2: Filtering tests correctly
|
|
# ============================================================================
|
|
|
|
- name: "Filter MSTest tests by category on VSTest"
|
|
prompt: "I only want to run the Integration tests using dotnet test, but I need to exclude the ones in the PaymentServiceTests class. Show me the exact command."
|
|
setup:
|
|
additional_required_skills: ["filter-syntax"]
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-mstest/TestProject.csproj"
|
|
- path: "OrderServiceTests.cs"
|
|
source: "fixtures/vstest-mstest/OrderServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter"
|
|
- type: "output_matches"
|
|
pattern: "TestCategory=Integration"
|
|
- type: "output_contains"
|
|
value: "&"
|
|
- type: "output_matches"
|
|
pattern: "(?:FullyQualifiedName|ClassName|Name)\\s*(?:!~|!=)[^\"&|]*PaymentService"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Uses the VSTest --filter flag and keeps TestCategory=Integration to select the integration tests"
|
|
- "Combines the category filter with a class exclusion using the '&' (AND) combinator"
|
|
- "Excludes PaymentServiceTests with a negation operator — '!~' (does-not-contain) or '!=' (not-equals) against FullyQualifiedName/ClassName — rather than a positive match"
|
|
- "Does not run all tests and post-filter, and does not use MTP-only flags like --filter-not-class for this VSTest project"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|
|
|
|
- name: "Filter NUnit tests by class name on VSTest"
|
|
prompt: "I want to run only the ReportExporterTests class using dotnet test, not the ReportGeneratorTests. How do I do that?"
|
|
setup:
|
|
additional_required_skills: ["filter-syntax"]
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-nunit/TestProject.csproj"
|
|
- path: "ReportServiceTests.cs"
|
|
source: "fixtures/vstest-nunit/ReportServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter"
|
|
- type: "output_matches"
|
|
pattern: "(FullyQualifiedName~ReportExporterTests|ClassName=.*ReportExporterTests)"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Identifies NUnit on VSTest (NUnit + NUnit3TestAdapter + Microsoft.NET.Test.Sdk)"
|
|
- "Uses --filter with FullyQualifiedName or ClassName property to select the class"
|
|
- "Correctly targets ReportExporterTests and excludes ReportGeneratorTests"
|
|
- "Uses VSTest expression syntax, not MTP-specific flags"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|
|
|
|
- name: "Filter xUnit v3 tests by class on MTP"
|
|
prompt: "I need to run only the ShoppingCartTests class using dotnet test, not the InventoryServiceTests. Can you show me the command?"
|
|
setup:
|
|
additional_required_skills: ["filter-syntax"]
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-xunit-v3/TestProject.csproj"
|
|
- path: "ShoppingCartTests.cs"
|
|
source: "fixtures/mtp-xunit-v3/ShoppingCartTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-xunit-v3/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter-class"
|
|
- type: "output_matches"
|
|
pattern: "ShoppingCartTests"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects xUnit v3 on MTP (xunit.v3 package + TestingPlatformDotnetTestSupport)"
|
|
- "Uses --filter-class instead of the VSTest --filter expression syntax"
|
|
- "xUnit v3 on MTP does NOT support the VSTest --filter \"ClassName=...\" syntax"
|
|
- "Correctly passes the filter after '--' separator since this is SDK 9"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|
|
|
|
- name: "Filter xUnit v3 tests by trait on MTP"
|
|
prompt: "I want to run only the tests with the Category=Integration trait in this xUnit project using dotnet test."
|
|
setup:
|
|
additional_required_skills: ["filter-syntax"]
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-xunit-v3/TestProject.csproj"
|
|
- path: "ShoppingCartTests.cs"
|
|
source: "fixtures/mtp-xunit-v3/ShoppingCartTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-xunit-v3/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter-trait"
|
|
- type: "output_matches"
|
|
pattern: "Category=Integration"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects xUnit v3 on MTP from the project references"
|
|
- "Uses --filter-trait \"Category=Integration\" instead of VSTest --filter \"Traits.Category=Integration\""
|
|
- "xUnit v3 on MTP requires framework-specific filter flags, not VSTest expression syntax"
|
|
- "Correctly handles the '--' separator for SDK 9"
|
|
expect_tools: ["bash"]
|
|
timeout: 240
|
|
|
|
- name: "Filter xUnit v3 tests by class pattern and trait using query filter language"
|
|
prompt: "I have an xUnit v3 project with several test classes. I need to run only the Smoke tests from classes whose name contains 'Integration' — I want a single combined filter expression for dotnet test rather than separate flags."
|
|
setup:
|
|
additional_required_skills: ["filter-syntax"]
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-xunit-v3/TestProject.csproj"
|
|
- path: "EcommerceTests.cs"
|
|
source: "fixtures/mtp-xunit-v3-query/EcommerceTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-xunit-v3/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--\\s+--filter-query"
|
|
- type: "output_matches"
|
|
pattern: '--filter-query\s+"(?=[^"]*Integration)(?=[^"]*\[Category=Smoke\])[^"]+"'
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Recognized that xUnit v3 on MTP supports a query filter language for complex combined expressions"
|
|
- "Uses SDK 9 MTP argument passing with '-- --filter-query'"
|
|
- "Produced a filter-query expression that combines a class name pattern with a trait filter in square brackets"
|
|
- "The expression correctly targets only Smoke tests from integration test classes, not all Smoke tests or all integration tests"
|
|
- "Used the path-segment syntax with wildcard segments and a trait qualifier like [Category=Smoke]"
|
|
expect_tools: ["bash"]
|
|
timeout: 240
|
|
|
|
- name: "Filter TUnit tests by class using treenode-filter"
|
|
prompt: "I want to run only the SmsNotificationTests from this TUnit project using dotnet test. What's the command?"
|
|
setup:
|
|
additional_required_skills: ["filter-syntax"]
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-tunit/TestProject.csproj"
|
|
- path: "NotificationTests.cs"
|
|
source: "fixtures/mtp-tunit/NotificationTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--treenode-filter"
|
|
- type: "output_matches"
|
|
pattern: "SmsNotificationTests"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Identifies TUnit as MTP-only (no VSTest support)"
|
|
- "Uses --treenode-filter with the path-based syntax to filter by class name"
|
|
- "Uses the correct wildcard pattern like /*/*/SmsNotificationTests/*"
|
|
- "Does not use VSTest --filter syntax which is incompatible with TUnit"
|
|
- "Uses 'dotnet run' or 'dotnet test' with correct MTP argument passing"
|
|
expect_tools: ["bash"]
|
|
timeout: 240
|
|
|
|
- name: "Combine multiple filter criteria on VSTest MSTest"
|
|
prompt: "I want to run only the unit tests in the OrderServiceTests class using dotnet test. How do I combine these two filter conditions?"
|
|
setup:
|
|
additional_required_skills: ["filter-syntax"]
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-mstest/TestProject.csproj"
|
|
- path: "OrderServiceTests.cs"
|
|
source: "fixtures/vstest-mstest/OrderServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter"
|
|
- type: "output_matches"
|
|
pattern: "TestCategory=Unit"
|
|
- type: "output_matches"
|
|
pattern: "OrderServiceTests"
|
|
- type: "output_contains"
|
|
value: "&"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Combines ClassName and TestCategory filters using the '&' (AND) operator"
|
|
- "Uses correct VSTest expression syntax: --filter \"ClassName~OrderServiceTests&TestCategory=Unit\""
|
|
- "Does not suggest running all tests and post-filtering results"
|
|
- "Explains the AND/OR combinator syntax for future reference"
|
|
expect_tools: ["bash"]
|
|
timeout: 300
|
|
|
|
# ============================================================================
|
|
# Goal 3: Using the right test platform CLI options
|
|
# ============================================================================
|
|
|
|
- name: "MTP project on SDK 9 must use -- separator for args"
|
|
prompt: "I want to run these tests with dotnet test and collect a crash dump if any test crashes the host. Show me the exact command."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-mstest-sdk9/TestProject.csproj"
|
|
- path: "UserAuthTests.cs"
|
|
source: "fixtures/mtp-mstest-sdk9/UserAuthTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-mstest-sdk9/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "--\\s+--blame-crash"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects TestingPlatformDotnetTestSupport in the csproj indicating MTP"
|
|
- "Recognizes that SDK 9 requires MTP arguments after the '--' separator"
|
|
- "Places --blame-crash after the '--' separator: 'dotnet test -- --blame-crash'"
|
|
- "Does not use the VSTest-style '--blame' flag for this MTP project"
|
|
expect_tools: ["bash"]
|
|
timeout: 240
|
|
|
|
- name: "MTP project on SDK 10 passes args directly"
|
|
prompt: "I want to run these tests with dotnet test and produce a TRX report. Show me the exact command."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-mstest-sdk10/TestProject.csproj"
|
|
- path: "CatalogServiceTests.cs"
|
|
source: "fixtures/mtp-mstest-sdk10/CatalogServiceTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-mstest-sdk10/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "--report-trx"
|
|
- type: "output_not_matches"
|
|
pattern: "-- --report-trx"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects MTP via global.json runner setting and SDK 10"
|
|
- "Passes --report-trx directly without the '--' separator"
|
|
- "Uses the SDK 10+ dotnet test syntax (e.g., --project flag instead of bare positional argument)"
|
|
- "Does not use VSTest-style '--logger trx' for an MTP project on SDK 10"
|
|
expect_tools: ["bash"]
|
|
timeout: 300
|
|
|
|
- name: "Detect test platform from Directory.Build.props"
|
|
prompt: "I want to run the integration tests in this project with dotnet test. Can you figure out which test platform it uses and run them? Show me the exact command you run."
|
|
setup:
|
|
additional_required_skills: ["platform-detection", "filter-syntax"]
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
content: |
|
|
<Project Sdk="Microsoft.NET.Sdk">
|
|
<PropertyGroup>
|
|
<TargetFramework>net9.0</TargetFramework>
|
|
<IsPackable>false</IsPackable>
|
|
</PropertyGroup>
|
|
<ItemGroup>
|
|
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
|
|
<PackageReference Include="MSTest" Version="3.8.0" />
|
|
</ItemGroup>
|
|
</Project>
|
|
- path: "Directory.Build.props"
|
|
content: |
|
|
<Project>
|
|
<PropertyGroup>
|
|
<TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
|
|
</PropertyGroup>
|
|
</Project>
|
|
- path: "global.json"
|
|
content: |
|
|
{
|
|
"sdk": {
|
|
"version": "9.0.200",
|
|
"rollForward": "latestFeature"
|
|
}
|
|
}
|
|
- path: "ServiceTests.cs"
|
|
content: |
|
|
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
|
namespace App.Tests;
|
|
|
|
[TestClass]
|
|
public class ServiceTests
|
|
{
|
|
[TestMethod]
|
|
[TestCategory("Unit")]
|
|
public void GetAll_ReturnsItems() { Assert.IsTrue(true); }
|
|
|
|
[TestMethod]
|
|
[TestCategory("Integration")]
|
|
public void GetAll_QueriesDatabase() { Assert.IsTrue(true); }
|
|
}
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "(MTP|Microsoft\\.Testing\\.Platform|TestingPlatformDotnetTestSupport)"
|
|
- type: "output_matches"
|
|
pattern: "--filter.*TestCategory=Integration"
|
|
- type: "output_matches"
|
|
pattern: "-- --filter"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Reads Directory.Build.props (not just the csproj) and finds TestingPlatformDotnetTestSupport=true"
|
|
- "Correctly identifies this as an MTP project even though the csproj alone looks like VSTest"
|
|
- "Uses MSTest filter syntax for TestCategory since MSTest on MTP uses the same filter syntax as VSTest"
|
|
- "Uses '--' separator before MTP filter args since this is SDK 9: 'dotnet test -- --filter TestCategory=Integration'"
|
|
expect_tools: ["bash"]
|
|
timeout: 240
|
|
|
|
- name: "Negative test: do not use MTP syntax for a VSTest project"
|
|
prompt: "Run my tests with dotnet test and generate a TRX report. Show me the exact command."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-mstest/TestProject.csproj"
|
|
- path: "OrderServiceTests.cs"
|
|
source: "fixtures/vstest-mstest/OrderServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "--logger.*(trx|\"trx\")"
|
|
- type: "output_not_contains"
|
|
value: "--report-trx"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Identifies this as a VSTest project (Microsoft.NET.Test.Sdk without MTP signals)"
|
|
- "Uses '--logger trx' which is the VSTest way to produce TRX output"
|
|
- "Does NOT use '--report-trx' which is the MTP-specific flag"
|
|
- "Does not add unnecessary '--' separator for a VSTest project"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|