mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
Amaury Levé
ca21fd61c1
* run-tests: use prompt-shaped multi-TFM keywords for plugin activation The 'Run tests in a multi-TFM project targeting a specific framework' eval scenario kept failing plugin-mode activation after #667. The skill description had 'multi-TFM projects (--framework)' as a brief parenthetical, but the scenario prompt uses different language: 'This test project targets both net8.0 and net9.0. I only want to run the tests on net9.0 using dotnet test.' Per the InvestigatingResults.md guidance for 'skill not activated' failures, expand that clause to include keywords from the prompt — 'running tests against a single target framework when a project targets multiple TFMs' plus a concrete '<TargetFrameworks>net8.0;net9.0</TargetFrameworks>' example and the '--framework <TFM>' invocation. Per-skill description is 947 chars, well under the 1024 spec cap. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Replace simple multi-TFM scenario with a harder MTP+SDK 9 variant that requires skill knowledge The previous 'Run tests in a multi-TFM project targeting a specific framework' scenario tested a trivial prompt (single --framework flag) that the model already answers perfectly from training (baseline 4.7/5). This matches pattern #8 (baseline already good, no headroom) in InvestigatingResults.md, and explains why two prior attempts to fix plugin-mode activation via description tweaks (PR #667, #688) failed: the router rationally skips loading the skill for a trivial question. This change replaces the scenario with a meaningfully harder one that combines several signals the model often confuses: - Multi-TFM project (TargetFrameworks plural) - MTP runner (TestingPlatformDotnetTestSupport) - .NET SDK 9 (-- separator required for MTP extension args) - TRX report (MTP extension flag) The correct answer requires knowing that --framework is a dotnet test/MSBuild flag that always goes BEFORE --, while --report-trx is an MTP extension flag that goes AFTER -- on SDK 9: dotnet test --framework net9.0 -- --report-trx The ordering assertion in eval.yaml enforces this on a single line, so swapping --framework after the separator no longer satisfies the scenario. Also fix a documentation bug in SKILL.md surfaced by this work: the 'Common MTP flags > Built-in flags' table previously listed --framework and --no-build under 'On SDK 8/9, pass after --', which is wrong. Those are dotnet test/MSBuild flags consumed by dotnet test itself; they always go before --. The table now lists only flags genuinely consumed by MTP, and an Important note clarifies the rule. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
408 lines
18 KiB
YAML
408 lines
18 KiB
YAML
scenarios:
|
|
# ============================================================================
|
|
# Goal 1: Calling dotnet test correctly
|
|
# ============================================================================
|
|
|
|
- name: "Run tests in a VSTest MSTest project"
|
|
prompt: "I need to run all the tests in this project. Can you help me?"
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-mstest/TestProject.csproj"
|
|
- path: "OrderServiceTests.cs"
|
|
source: "fixtures/vstest-mstest/OrderServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Identifies the project as a VSTest project (Microsoft.NET.Test.Sdk + MSTest.TestAdapter)"
|
|
- "Uses 'dotnet test' without MTP-specific arguments or '--' separator"
|
|
- "Does not suggest MTP-style arguments like --report-trx or --treenode-filter"
|
|
expect_tools: ["bash"]
|
|
timeout: 180
|
|
|
|
- name: "Run tests with trx reporting on MTP project (SDK 9)"
|
|
prompt: "I have an MSTest project using Microsoft.Testing.Platform. I need to run the tests and produce a TRX report. I'm on .NET SDK 9."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-mstest-sdk9/TestProject.csproj"
|
|
- path: "UserAuthTests.cs"
|
|
source: "fixtures/mtp-mstest-sdk9/UserAuthTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-mstest-sdk9/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "--\\s+--report-trx"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects TestingPlatformDotnetTestSupport property in the csproj indicating MTP"
|
|
- "Recognizes that SDK 9 requires MTP arguments after the '--' separator"
|
|
- "Uses '-- --report-trx' (with separator) instead of just '--report-trx'"
|
|
- "Does not suggest the VSTest-style '--logger trx' for an MTP project"
|
|
expect_tools: ["bash"]
|
|
timeout: 480
|
|
|
|
- name: "Run tests with blame-hang on MTP project (SDK 10)"
|
|
prompt: "My tests are occasionally hanging. I want to run them with dotnet test using a hang timeout of 5 minutes to detect the culprit. Here's my project."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-mstest-sdk10/TestProject.csproj"
|
|
- path: "CatalogServiceTests.cs"
|
|
source: "fixtures/mtp-mstest-sdk10/CatalogServiceTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-mstest-sdk10/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "blame-hang"
|
|
- type: "output_not_matches"
|
|
pattern: "-- --blame"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects MTP via the global.json runner setting (Microsoft.Testing.Platform)"
|
|
- "Recognizes that SDK 10 passes MTP arguments directly without '--' separator"
|
|
- "Uses '--blame-hang-timeout 5min' or similar directly, not after '--'"
|
|
- "Does not use the VSTest-style '--blame --blame-hang-timeout' combination"
|
|
expect_tools: ["bash"]
|
|
timeout: 420
|
|
|
|
- name: "Run tests on a specific TFM with TRX in a multi-TFM MTP project (SDK 9)"
|
|
prompt: "This MSTest project uses Microsoft.Testing.Platform and targets both net8.0 and net9.0. I'm on .NET SDK 9. I want to run only the net9.0 tests and produce a TRX report."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/multi-tfm-mtp-sdk9/TestProject.csproj"
|
|
- path: "global.json"
|
|
source: "fixtures/multi-tfm-mtp-sdk9/global.json"
|
|
- path: "UserAuthTests.cs"
|
|
source: "fixtures/multi-tfm-mtp-sdk9/UserAuthTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test[^\\r\\n]*--framework\\s+net9\\.0[^\\r\\n]*--\\s+--report-trx"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Runs only the net9.0 target framework rather than both TFMs"
|
|
- "Identifies that the project uses MTP (TestingPlatformDotnetTestSupport) with SDK 9 argument-separator behavior"
|
|
- "Keeps '--framework net9.0' before the '--' separator (built-in dotnet test flag)"
|
|
- "Passes '--report-trx' after the '--' separator (MTP extension flag, required on SDK 9)"
|
|
- "Avoids VSTest-style TRX syntax such as '--logger trx'"
|
|
expect_tools: ["bash"]
|
|
timeout: 480
|
|
|
|
# ============================================================================
|
|
# Goal 2: Filtering tests correctly
|
|
# ============================================================================
|
|
|
|
- name: "Filter MSTest tests by category on VSTest"
|
|
prompt: "I only want to run the integration tests using dotnet test. How do I filter by the Integration category?"
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-mstest/TestProject.csproj"
|
|
- path: "OrderServiceTests.cs"
|
|
source: "fixtures/vstest-mstest/OrderServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter"
|
|
- type: "output_matches"
|
|
pattern: "TestCategory=Integration"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Uses the VSTest --filter flag with the TestCategory property"
|
|
- "Uses the correct syntax: --filter \"TestCategory=Integration\""
|
|
- "Does not use MTP-style filter flags like --filter-trait"
|
|
- "Does not suggest xUnit-style Trait filtering for an MSTest project"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|
|
|
|
- name: "Filter NUnit tests by class name on VSTest"
|
|
prompt: "I want to run only the ReportExporterTests class using dotnet test, not the ReportGeneratorTests. How do I do that?"
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-nunit/TestProject.csproj"
|
|
- path: "ReportServiceTests.cs"
|
|
source: "fixtures/vstest-nunit/ReportServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter"
|
|
- type: "output_matches"
|
|
pattern: "(FullyQualifiedName~ReportExporterTests|ClassName=.*ReportExporterTests)"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Identifies NUnit on VSTest (NUnit + NUnit3TestAdapter + Microsoft.NET.Test.Sdk)"
|
|
- "Uses --filter with FullyQualifiedName or ClassName property to select the class"
|
|
- "Correctly targets ReportExporterTests and excludes ReportGeneratorTests"
|
|
- "Uses VSTest expression syntax, not MTP-specific flags"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|
|
|
|
- name: "Filter xUnit v3 tests by class on MTP"
|
|
prompt: "I need to run only the ShoppingCartTests class using dotnet test, not the InventoryServiceTests. Can you show me the command?"
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-xunit-v3/TestProject.csproj"
|
|
- path: "ShoppingCartTests.cs"
|
|
source: "fixtures/mtp-xunit-v3/ShoppingCartTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-xunit-v3/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter-class"
|
|
- type: "output_matches"
|
|
pattern: "ShoppingCartTests"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects xUnit v3 on MTP (xunit.v3 package + TestingPlatformDotnetTestSupport)"
|
|
- "Uses --filter-class instead of the VSTest --filter expression syntax"
|
|
- "xUnit v3 on MTP does NOT support the VSTest --filter \"ClassName=...\" syntax"
|
|
- "Correctly passes the filter after '--' separator since this is SDK 9"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|
|
|
|
- name: "Filter xUnit v3 tests by trait on MTP"
|
|
prompt: "I want to run only the tests with the Category=Integration trait in this xUnit project using dotnet test."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-xunit-v3/TestProject.csproj"
|
|
- path: "ShoppingCartTests.cs"
|
|
source: "fixtures/mtp-xunit-v3/ShoppingCartTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-xunit-v3/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter-trait"
|
|
- type: "output_matches"
|
|
pattern: "Category=Integration"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects xUnit v3 on MTP from the project references"
|
|
- "Uses --filter-trait \"Category=Integration\" instead of VSTest --filter \"Traits.Category=Integration\""
|
|
- "xUnit v3 on MTP requires framework-specific filter flags, not VSTest expression syntax"
|
|
- "Correctly handles the '--' separator for SDK 9"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|
|
|
|
- name: "Filter xUnit v3 tests by class pattern and trait using query filter language"
|
|
prompt: "I have an xUnit v3 project with several test classes. I need to run only the Smoke tests from classes whose name contains 'Integration' — I want a single combined filter expression for dotnet test rather than separate flags."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-xunit-v3/TestProject.csproj"
|
|
- path: "EcommerceTests.cs"
|
|
source: "fixtures/mtp-xunit-v3-query/EcommerceTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-xunit-v3/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--\\s+--filter-query"
|
|
- type: "output_matches"
|
|
pattern: '--filter-query\\s+"(?=[^"]*Integration)(?=[^"]*\\[Category=Smoke\\])[^"]+"'
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Recognized that xUnit v3 on MTP supports a query filter language for complex combined expressions"
|
|
- "Uses SDK 9 MTP argument passing with '-- --filter-query'"
|
|
- "Produced a filter-query expression that combines a class name pattern with a trait filter in square brackets"
|
|
- "The expression correctly targets only Smoke tests from integration test classes, not all Smoke tests or all integration tests"
|
|
- "Used the path-segment syntax with wildcard segments and a trait qualifier like [Category=Smoke]"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|
|
|
|
- name: "Filter TUnit tests by class using treenode-filter"
|
|
prompt: "I want to run only the SmsNotificationTests from this TUnit project using dotnet test. What's the command?"
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-tunit/TestProject.csproj"
|
|
- path: "NotificationTests.cs"
|
|
source: "fixtures/mtp-tunit/NotificationTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--treenode-filter"
|
|
- type: "output_matches"
|
|
pattern: "SmsNotificationTests"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Identifies TUnit as MTP-only (no VSTest support)"
|
|
- "Uses --treenode-filter with the path-based syntax to filter by class name"
|
|
- "Uses the correct wildcard pattern like /*/*/SmsNotificationTests/*"
|
|
- "Does not use VSTest --filter syntax which is incompatible with TUnit"
|
|
- "Uses 'dotnet run' or 'dotnet test' with correct MTP argument passing"
|
|
expect_tools: ["bash"]
|
|
timeout: 240
|
|
|
|
- name: "Combine multiple filter criteria on VSTest MSTest"
|
|
prompt: "I want to run only the unit tests in the OrderServiceTests class using dotnet test. How do I combine these two filter conditions?"
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-mstest/TestProject.csproj"
|
|
- path: "OrderServiceTests.cs"
|
|
source: "fixtures/vstest-mstest/OrderServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "--filter"
|
|
- type: "output_matches"
|
|
pattern: "TestCategory=Unit"
|
|
- type: "output_matches"
|
|
pattern: "OrderServiceTests"
|
|
- type: "output_contains"
|
|
value: "&"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Combines ClassName and TestCategory filters using the '&' (AND) operator"
|
|
- "Uses correct VSTest expression syntax: --filter \"ClassName~OrderServiceTests&TestCategory=Unit\""
|
|
- "Does not suggest running all tests and post-filtering results"
|
|
- "Explains the AND/OR combinator syntax for future reference"
|
|
expect_tools: ["bash"]
|
|
timeout: 300
|
|
|
|
# ============================================================================
|
|
# Goal 3: Using the right test platform CLI options
|
|
# ============================================================================
|
|
|
|
- name: "MTP project on SDK 9 must use -- separator for args"
|
|
prompt: "I want to run these tests with dotnet test and collect a crash dump if any test crashes the host."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-mstest-sdk9/TestProject.csproj"
|
|
- path: "UserAuthTests.cs"
|
|
source: "fixtures/mtp-mstest-sdk9/UserAuthTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-mstest-sdk9/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "--\\s+--blame-crash"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects TestingPlatformDotnetTestSupport in the csproj indicating MTP"
|
|
- "Recognizes that SDK 9 requires MTP arguments after the '--' separator"
|
|
- "Places --blame-crash after the '--' separator: 'dotnet test -- --blame-crash'"
|
|
- "Does not use the VSTest-style '--blame' flag for this MTP project"
|
|
expect_tools: ["bash"]
|
|
timeout: 240
|
|
|
|
- name: "MTP project on SDK 10 passes args directly"
|
|
prompt: "I want to run these tests with dotnet test and produce a TRX report."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/mtp-mstest-sdk10/TestProject.csproj"
|
|
- path: "CatalogServiceTests.cs"
|
|
source: "fixtures/mtp-mstest-sdk10/CatalogServiceTests.cs"
|
|
- path: "global.json"
|
|
source: "fixtures/mtp-mstest-sdk10/global.json"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "--report-trx"
|
|
- type: "output_not_matches"
|
|
pattern: "-- --report-trx"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Detects MTP via global.json runner setting and SDK 10"
|
|
- "Passes --report-trx directly without the '--' separator"
|
|
- "Uses the SDK 10+ dotnet test syntax (e.g., --project flag instead of bare positional argument)"
|
|
- "Does not use VSTest-style '--logger trx' for an MTP project on SDK 10"
|
|
expect_tools: ["bash"]
|
|
timeout: 300
|
|
|
|
- name: "Detect test platform from Directory.Build.props"
|
|
prompt: "I want to run the integration tests in this project with dotnet test. Can you figure out which test platform it uses and run them?"
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
content: |
|
|
<Project Sdk="Microsoft.NET.Sdk">
|
|
<PropertyGroup>
|
|
<TargetFramework>net9.0</TargetFramework>
|
|
<IsPackable>false</IsPackable>
|
|
</PropertyGroup>
|
|
<ItemGroup>
|
|
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
|
|
<PackageReference Include="MSTest" Version="3.8.0" />
|
|
</ItemGroup>
|
|
</Project>
|
|
- path: "Directory.Build.props"
|
|
content: |
|
|
<Project>
|
|
<PropertyGroup>
|
|
<TestingPlatformDotnetTestSupport>true</TestingPlatformDotnetTestSupport>
|
|
</PropertyGroup>
|
|
</Project>
|
|
- path: "global.json"
|
|
content: |
|
|
{
|
|
"sdk": {
|
|
"version": "9.0.200",
|
|
"rollForward": "latestFeature"
|
|
}
|
|
}
|
|
- path: "ServiceTests.cs"
|
|
content: |
|
|
using Microsoft.VisualStudio.TestTools.UnitTesting;
|
|
namespace App.Tests;
|
|
|
|
[TestClass]
|
|
public class ServiceTests
|
|
{
|
|
[TestMethod]
|
|
[TestCategory("Unit")]
|
|
public void GetAll_ReturnsItems() { Assert.IsTrue(true); }
|
|
|
|
[TestMethod]
|
|
[TestCategory("Integration")]
|
|
public void GetAll_QueriesDatabase() { Assert.IsTrue(true); }
|
|
}
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "(MTP|Microsoft\\.Testing\\.Platform|TestingPlatformDotnetTestSupport)"
|
|
- type: "output_matches"
|
|
pattern: "--filter.*TestCategory=Integration"
|
|
- type: "output_matches"
|
|
pattern: "-- --filter"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Reads Directory.Build.props (not just the csproj) and finds TestingPlatformDotnetTestSupport=true"
|
|
- "Correctly identifies this as an MTP project even though the csproj alone looks like VSTest"
|
|
- "Uses MSTest filter syntax for TestCategory since MSTest on MTP uses the same filter syntax as VSTest"
|
|
- "Uses '--' separator before MTP filter args since this is SDK 9: 'dotnet test -- --filter TestCategory=Integration'"
|
|
expect_tools: ["bash"]
|
|
timeout: 240
|
|
|
|
- name: "Negative test: do not use MTP syntax for a VSTest project"
|
|
prompt: "Run my tests with dotnet test and generate a TRX report."
|
|
setup:
|
|
files:
|
|
- path: "TestProject.csproj"
|
|
source: "fixtures/vstest-mstest/TestProject.csproj"
|
|
- path: "OrderServiceTests.cs"
|
|
source: "fixtures/vstest-mstest/OrderServiceTests.cs"
|
|
assertions:
|
|
- type: "output_matches"
|
|
pattern: "dotnet test"
|
|
- type: "output_matches"
|
|
pattern: "--logger.*(trx|\"trx\")"
|
|
- type: "output_not_contains"
|
|
value: "--report-trx"
|
|
- type: "exit_success"
|
|
rubric:
|
|
- "Identifies this as a VSTest project (Microsoft.NET.Test.Sdk without MTP signals)"
|
|
- "Uses '--logger trx' which is the VSTest way to produce TRX output"
|
|
- "Does NOT use '--report-trx' which is the MTP-specific flag"
|
|
- "Does not add unnecessary '--' separator for a VSTest project"
|
|
expect_tools: ["bash"]
|
|
timeout: 120
|