Files
Amaury Levé 592a53008e skill-validator: restore 15K aggregate cap as the real Copilot CLI skill-menu budget (#803)
* skill-validator: restore 15K aggregate cap, document it as the real Copilot CLI skill-menu budget

The per-plugin aggregate description cap had been raised 15,000 -> 20,000
-> 22,000 under the belief that 15K was 'a local repo policy, NOT a
documented Copilot constraint'. That belief was wrong: the GitHub Copilot
CLI renders the model-facing <available_skills> menu under a hard 15,000-
char budget (the agent SDK's SKILL_CHAR_BUDGET, default 15e3, confirmed in
CLI 1.0.36 and 1.0.61). Skills are listed alphabetically and emitted with
their full <description> only until the budget is exhausted; every skill
past the cut-off collapses to a bare name with no description and can no
longer be reliably model-activated. Raising the validator cap merely
masked this silent menu truncation — e.g. dotnet-test's run-tests and
test-* skills stopped activating in plugin eval runs because they fell
into the name-only overflow.

Changes:
- SkillProfiler.MaxAggregateDescriptionLength: 22,000 -> 15,000, with the
  comment rewritten to document the real Copilot CLI budget (and correct
  the prior 'not a documented constraint' claim).
- CheckCommand aggregate now excludes skills marked
  'disable-model-invocation: true' — the CLI drops those from the menu, so
  they do not consume the budget. This makes the cap satisfiable by hiding
  reference / agent-orchestrated primitives rather than only by trimming.
- InvestigatingResults.md: document plugin-arm-only non-activation caused
  by skill-menu budget overflow, and how to fix it.

Note: dotnet-test currently exceeds 15K and must be slimmed below it
(via disable-model-invocation on reference/primitive skills plus
description trims) before this cap can go green repo-wide.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* skill-validator: use source-generated regex for disable-model-invocation check

Address review: replace Regex.IsMatch(pattern-string) with a
[GeneratedRegex] partial method (AOT-friendly, no per-call cache lookup),
matching FrontmatterParser's style. Runs once per skill during checks.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* skill-validator: parse disable-model-invocation via YAML to avoid block-scalar false positives

The regex-based check matched any line in the frontmatter, so a block-scalar description that merely mentioned 'disable-model-invocation: true' on its own line was wrongly treated as disabling model invocation. Parse the frontmatter with the existing YAML deserializer (which correctly handles block scalars) by adding a DisableModelInvocation field to SkillFrontmatter, and drop the regex entirely.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-06-25 14:59:05 +02:00

163 lines
5.8 KiB
C#

using SkillValidator.Shared;
namespace SkillValidator.Tests;
public class DiscoverSkillsTests
{
private static string FixturesPath => Path.Combine(AppContext.BaseDirectory, "fixtures");
[Fact]
public async Task DiscoversASingleSkillDirectly()
{
var skills = await SkillDiscovery.DiscoverSkills(Path.Combine(FixturesPath, "sample-skill"));
Assert.Single(skills);
Assert.Equal("sample-skill", skills[0].Name);
Assert.Contains("greeting", skills[0].Description);
}
[Fact]
public async Task DiscoversSkillsInParentDirectory()
{
var skills = await SkillDiscovery.DiscoverSkills(FixturesPath);
Assert.True(skills.Count >= 2);
var names = skills.Select(s => s.Name).ToList();
Assert.Contains("sample-skill", names);
Assert.Contains("no-eval-skill", names);
}
[Fact]
public async Task HandlesSkillWithNoEvalYaml()
{
var skills = await SkillDiscovery.DiscoverSkills(Path.Combine(FixturesPath, "no-eval-skill"));
Assert.Single(skills);
}
[Fact]
public async Task ReturnsEmptyForNonSkillDirectory()
{
var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}");
Directory.CreateDirectory(tmpDir);
try
{
var skills = await SkillDiscovery.DiscoverSkills(tmpDir);
Assert.Empty(skills);
}
finally
{
Directory.Delete(tmpDir, recursive: true);
}
}
[Fact]
public async Task DiscoverSkillsRecursiveFindsNestedSkills()
{
// Simulates plugins/<plugin>/skills/<skill>/SKILL.md layout
var tmpDir = Path.Combine(Path.GetTempPath(), $"skill-test-{Guid.NewGuid():N}");
var skill1Dir = Path.Combine(tmpDir, "plugin-a", "skills", "skill-one");
var skill2Dir = Path.Combine(tmpDir, "plugin-b", "skills", "skill-two");
Directory.CreateDirectory(skill1Dir);
Directory.CreateDirectory(skill2Dir);
try
{
await File.WriteAllTextAsync(Path.Combine(skill1Dir, "SKILL.md"), "---\nname: skill-one\ndescription: first\n---\nBody", TestContext.Current.CancellationToken);
await File.WriteAllTextAsync(Path.Combine(skill2Dir, "SKILL.md"), "---\nname: skill-two\ndescription: second\n---\nBody", TestContext.Current.CancellationToken);
var skills = await SkillDiscovery.DiscoverSkillsRecursive(tmpDir);
Assert.Equal(2, skills.Count);
var names = skills.Select(s => s.Name).OrderBy(n => n).ToList();
Assert.Equal("skill-one", names[0]);
Assert.Equal("skill-two", names[1]);
}
finally
{
Directory.Delete(tmpDir, true);
}
}
[Fact]
public async Task DiscoverSkillsRecursiveReturnsEmptyForMissingDir()
{
var skills = await SkillDiscovery.DiscoverSkillsRecursive(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")));
Assert.Empty(skills);
}
}
public class ParseFrontmatterTests
{
[Fact]
public void DisableModelInvocation_True_WhenTopLevelKeySet()
{
var content = "---\nname: my-skill\ndescription: A skill.\ndisable-model-invocation: true\n---\nBody";
var (metadata, _) = SkillDiscovery.ParseFrontmatter(content);
Assert.True(metadata.DisableModelInvocation);
}
[Fact]
public void DisableModelInvocation_False_WhenKeyAbsent()
{
var content = "---\nname: my-skill\ndescription: A skill.\n---\nBody";
var (metadata, _) = SkillDiscovery.ParseFrontmatter(content);
Assert.False(metadata.DisableModelInvocation);
}
[Fact]
public void DisableModelInvocation_False_WhenKeyAppearsInsideBlockScalarDescription()
{
// Regression: a previous regex-based check matched any line in the YAML,
// so a block-scalar description that merely mentions the key on its own
// line was wrongly treated as disabling model invocation. Proper YAML
// parsing must not be fooled by indented block-scalar content.
var content =
"---\n" +
"name: my-skill\n" +
"description: |\n" +
" This skill explains config options.\n" +
" disable-model-invocation: true\n" +
"---\n" +
"Body";
var (metadata, _) = SkillDiscovery.ParseFrontmatter(content);
Assert.False(metadata.DisableModelInvocation);
}
}
public class PluginDiscoveryTests
{
[Fact]
public void FindPluginContextReturnsNullWithoutPluginJson()
{
var tmpDir = Path.Combine(Path.GetTempPath(), $"ctx-test-{Guid.NewGuid():N}");
Directory.CreateDirectory(tmpDir);
try
{
var skill = new SkillInfo("test", "T", tmpDir, Path.Combine(tmpDir, "SKILL.md"), "# T");
var result = PluginDiscovery.FindPluginContext(skill);
Assert.Null(result);
}
finally
{
Directory.Delete(tmpDir, true);
}
}
[Fact]
public void FindPluginContextReturnsPluginInfo()
{
var tmpDir = Path.Combine(Path.GetTempPath(), $"ctx-test-{Guid.NewGuid():N}");
var pluginDir = Path.Combine(tmpDir, "my-plugin");
var skillDir = Path.Combine(pluginDir, "skills", "test-skill");
Directory.CreateDirectory(skillDir);
try
{
File.WriteAllText(Path.Combine(pluginDir, "plugin.json"), """{ "name": "my-plugin" }""");
var skill = new SkillInfo("test-skill", "T", skillDir, Path.Combine(skillDir, "SKILL.md"), "# T");
var result = PluginDiscovery.FindPluginContext(skill);
Assert.NotNull(result);
Assert.Equal(pluginDir, result!.Value.PluginRoot);
}
finally
{
Directory.Delete(tmpDir, true);
}
}
}