mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
592a53008e
* skill-validator: restore 15K aggregate cap, document it as the real Copilot CLI skill-menu budget The per-plugin aggregate description cap had been raised 15,000 -> 20,000 -> 22,000 under the belief that 15K was 'a local repo policy, NOT a documented Copilot constraint'. That belief was wrong: the GitHub Copilot CLI renders the model-facing <available_skills> menu under a hard 15,000- char budget (the agent SDK's SKILL_CHAR_BUDGET, default 15e3, confirmed in CLI 1.0.36 and 1.0.61). Skills are listed alphabetically and emitted with their full <description> only until the budget is exhausted; every skill past the cut-off collapses to a bare name with no description and can no longer be reliably model-activated. Raising the validator cap merely masked this silent menu truncation — e.g. dotnet-test's run-tests and test-* skills stopped activating in plugin eval runs because they fell into the name-only overflow. Changes: - SkillProfiler.MaxAggregateDescriptionLength: 22,000 -> 15,000, with the comment rewritten to document the real Copilot CLI budget (and correct the prior 'not a documented constraint' claim). - CheckCommand aggregate now excludes skills marked 'disable-model-invocation: true' — the CLI drops those from the menu, so they do not consume the budget. This makes the cap satisfiable by hiding reference / agent-orchestrated primitives rather than only by trimming. - InvestigatingResults.md: document plugin-arm-only non-activation caused by skill-menu budget overflow, and how to fix it. Note: dotnet-test currently exceeds 15K and must be slimmed below it (via disable-model-invocation on reference/primitive skills plus description trims) before this cap can go green repo-wide. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * skill-validator: use source-generated regex for disable-model-invocation check Address review: replace Regex.IsMatch(pattern-string) with a [GeneratedRegex] partial method (AOT-friendly, no per-call cache lookup), matching FrontmatterParser's style. Runs once per skill during checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * skill-validator: parse disable-model-invocation via YAML to avoid block-scalar false positives The regex-based check matched any line in the frontmatter, so a block-scalar description that merely mentioned 'disable-model-invocation: true' on its own line was wrongly treated as disabling model invocation. Parse the frontmatter with the existing YAML deserializer (which correctly handles block scalars) by adding a DisableModelInvocation field to SkillFrontmatter, and drop the regex entirely. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
163 lines
5.8 KiB
C#
163 lines
5.8 KiB
C#
using SkillValidator.Shared;
|
|
|
|
namespace SkillValidator.Tests;
|
|
|
|
public class DiscoverSkillsTests
|
|
{
|
|
private static string FixturesPath => Path.Combine(AppContext.BaseDirectory, "fixtures");
|
|
|
|
[Fact]
|
|
public async Task DiscoversASingleSkillDirectly()
|
|
{
|
|
var skills = await SkillDiscovery.DiscoverSkills(Path.Combine(FixturesPath, "sample-skill"));
|
|
Assert.Single(skills);
|
|
Assert.Equal("sample-skill", skills[0].Name);
|
|
Assert.Contains("greeting", skills[0].Description);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DiscoversSkillsInParentDirectory()
|
|
{
|
|
var skills = await SkillDiscovery.DiscoverSkills(FixturesPath);
|
|
Assert.True(skills.Count >= 2);
|
|
var names = skills.Select(s => s.Name).ToList();
|
|
Assert.Contains("sample-skill", names);
|
|
Assert.Contains("no-eval-skill", names);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task HandlesSkillWithNoEvalYaml()
|
|
{
|
|
var skills = await SkillDiscovery.DiscoverSkills(Path.Combine(FixturesPath, "no-eval-skill"));
|
|
Assert.Single(skills);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ReturnsEmptyForNonSkillDirectory()
|
|
{
|
|
var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}");
|
|
Directory.CreateDirectory(tmpDir);
|
|
try
|
|
{
|
|
var skills = await SkillDiscovery.DiscoverSkills(tmpDir);
|
|
Assert.Empty(skills);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(tmpDir, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DiscoverSkillsRecursiveFindsNestedSkills()
|
|
{
|
|
// Simulates plugins/<plugin>/skills/<skill>/SKILL.md layout
|
|
var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}");
|
|
var skill1Dir = Path.Combine(tmpDir, "plugin-a", "skills", "skill-one");
|
|
var skill2Dir = Path.Combine(tmpDir, "plugin-b", "skills", "skill-two");
|
|
Directory.CreateDirectory(skill1Dir);
|
|
Directory.CreateDirectory(skill2Dir);
|
|
try
|
|
{
|
|
await File.WriteAllTextAsync(Path.Combine(skill1Dir, "SKILL.md"), "---\nname: skill-one\ndescription: first\n---\nBody", TestContext.Current.CancellationToken);
|
|
await File.WriteAllTextAsync(Path.Combine(skill2Dir, "SKILL.md"), "---\nname: skill-two\ndescription: second\n---\nBody", TestContext.Current.CancellationToken);
|
|
|
|
var skills = await SkillDiscovery.DiscoverSkillsRecursive(tmpDir);
|
|
Assert.Equal(2, skills.Count);
|
|
var names = skills.Select(s => s.Name).OrderBy(n => n).ToList();
|
|
Assert.Equal("skill-one", names[0]);
|
|
Assert.Equal("skill-two", names[1]);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(tmpDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DiscoverSkillsRecursiveReturnsEmptyForMissingDir()
|
|
{
|
|
var skills = await SkillDiscovery.DiscoverSkillsRecursive(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")));
|
|
Assert.Empty(skills);
|
|
}
|
|
}
|
|
|
|
public class ParseFrontmatterTests
|
|
{
|
|
[Fact]
|
|
public void DisableModelInvocation_True_WhenTopLevelKeySet()
|
|
{
|
|
var content = "---\nname: my-skill\ndescription: A skill.\ndisable-model-invocation: true\n---\nBody";
|
|
var (metadata, _) = SkillDiscovery.ParseFrontmatter(content);
|
|
Assert.True(metadata.DisableModelInvocation);
|
|
}
|
|
|
|
[Fact]
|
|
public void DisableModelInvocation_False_WhenKeyAbsent()
|
|
{
|
|
var content = "---\nname: my-skill\ndescription: A skill.\n---\nBody";
|
|
var (metadata, _) = SkillDiscovery.ParseFrontmatter(content);
|
|
Assert.False(metadata.DisableModelInvocation);
|
|
}
|
|
|
|
[Fact]
|
|
public void DisableModelInvocation_False_WhenKeyAppearsInsideBlockScalarDescription()
|
|
{
|
|
// Regression: a previous regex-based check matched any line in the YAML,
|
|
// so a block-scalar description that merely mentions the key on its own
|
|
// line was wrongly treated as disabling model invocation. Proper YAML
|
|
// parsing must not be fooled by indented block-scalar content.
|
|
var content =
|
|
"---\n" +
|
|
"name: my-skill\n" +
|
|
"description: |\n" +
|
|
" This skill explains config options.\n" +
|
|
" disable-model-invocation: true\n" +
|
|
"---\n" +
|
|
"Body";
|
|
var (metadata, _) = SkillDiscovery.ParseFrontmatter(content);
|
|
Assert.False(metadata.DisableModelInvocation);
|
|
}
|
|
}
|
|
|
|
public class PluginDiscoveryTests
|
|
{
|
|
[Fact]
|
|
public void FindPluginContextReturnsNullWithoutPluginJson()
|
|
{
|
|
var tmpDir = Path.Combine(Path.GetTempPath(), $"ctx-test-{Guid.NewGuid():N}");
|
|
Directory.CreateDirectory(tmpDir);
|
|
try
|
|
{
|
|
var skill = new SkillInfo("test", "T", tmpDir, Path.Combine(tmpDir, "SKILL.md"), "# T");
|
|
var result = PluginDiscovery.FindPluginContext(skill);
|
|
Assert.Null(result);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(tmpDir, true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public void FindPluginContextReturnsPluginInfo()
|
|
{
|
|
var tmpDir = Path.Combine(Path.GetTempPath(), $"ctx-test-{Guid.NewGuid():N}");
|
|
var pluginDir = Path.Combine(tmpDir, "my-plugin");
|
|
var skillDir = Path.Combine(pluginDir, "skills", "test-skill");
|
|
Directory.CreateDirectory(skillDir);
|
|
try
|
|
{
|
|
File.WriteAllText(Path.Combine(pluginDir, "plugin.json"), """{ "name": "my-plugin" }""");
|
|
var skill = new SkillInfo("test-skill", "T", skillDir, Path.Combine(skillDir, "SKILL.md"), "# T");
|
|
var result = PluginDiscovery.FindPluginContext(skill);
|
|
Assert.NotNull(result);
|
|
Assert.Equal(pluginDir, result!.Value.PluginRoot);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(tmpDir, true);
|
|
}
|
|
}
|
|
}
|