Files
dotnet__skills/eng/skill-validator/tests/CheckCommandTests.cs
Viktor Hofer 1a20e39a8d Refactor skill-validator: add check command, rename validate to evaluate (#388)
* Refactor skill-validator: add check command, rename validate to evaluate

- Add new 'check' command for CI skill validation (frontmatter, structure)
- Rename ValidateCommand to EvaluateCommand to clarify its purpose
- Add skill-check.yml GitHub Actions workflow
- Refactor SkillDiscovery for improved skill detection
- Simplify OverfittingJudge and SkillProfiler
- Update tests to match refactored APIs

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Fix check subcommand: use EvaluateCommand as root command

System.CommandLine 2.0.4 requires the primary command to be the
RootCommand. Restores the upstream pattern where the evaluate/validate
command is the root and check/consolidate/rejudge are subcommands.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Pass all plugins in single check invocation

Build plugin args dynamically and pass them all at once instead of
looping per plugin.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* Address PR review comments

- Remove 'evaluate' subcommand token from README, evaluation-run.yml,
  and RunArguments in csproj (evaluate is now the root command)
- Wrap ParsePluginJson in try/catch for JsonException with clean error
- Add CheckCommandTests with aggregate description limit coverage
  (under/at/over limit and multi-plugin cases)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-03-17 17:00:47 +01:00

134 lines
4.9 KiB
C#

using SkillValidator.Commands;
using SkillValidator.Models;
using SkillValidator.Services;
namespace SkillValidator.Tests;
public class CheckCommandAggregateDescriptionTests
{
private static string CreatePluginFixture(string pluginName, params (string skillName, string description)[] skills)
{
var root = Path.Combine(Path.GetTempPath(), $"check-test-{Guid.NewGuid():N}");
var pluginDir = Path.Combine(root, pluginName);
var skillsDir = Path.Combine(pluginDir, "skills");
Directory.CreateDirectory(pluginDir);
Directory.CreateDirectory(skillsDir);
File.WriteAllText(Path.Combine(pluginDir, "plugin.json"),
$$"""{"name":"{{pluginName}}","version":"1.0.0","description":"Test plugin.","skills":"./skills/"}""");
foreach (var (skillName, description) in skills)
{
var skillDir = Path.Combine(skillsDir, skillName);
Directory.CreateDirectory(skillDir);
File.WriteAllText(Path.Combine(skillDir, "SKILL.md"),
$"---\nname: {skillName}\ndescription: {description}\n---\n# {skillName}\n\nContent.\n");
}
return root;
}
[Fact]
public async Task UnderAggregateLimit_Passes()
{
var root = CreatePluginFixture("test-plugin",
("skill-a", "Short description A."),
("skill-b", "Short description B."));
try
{
var config = new CheckConfig { PluginPaths = [Path.Combine(root, "test-plugin")] };
var result = await CheckCommand.Run(config);
Assert.Equal(0, result);
}
finally { Directory.Delete(root, true); }
}
[Fact]
public async Task AtAggregateLimit_Passes()
{
// Create skills whose descriptions sum exactly to the limit, each under per-skill max (1024)
int limit = SkillProfiler.MaxAggregateDescriptionLength;
int perSkill = 1024;
int skillCount = limit / perSkill;
int remainder = limit - (skillCount * perSkill);
var skills = Enumerable.Range(0, skillCount)
.Select(i => ($"skill-{i}", new string('a', perSkill)))
.ToList();
if (remainder > 0)
skills.Add(($"skill-extra", new string('a', remainder)));
var root = CreatePluginFixture("test-plugin", skills.ToArray());
try
{
var config = new CheckConfig { PluginPaths = [Path.Combine(root, "test-plugin")] };
var result = await CheckCommand.Run(config);
Assert.Equal(0, result);
}
finally { Directory.Delete(root, true); }
}
[Fact]
public async Task OverAggregateLimit_Fails()
{
int limit = SkillProfiler.MaxAggregateDescriptionLength;
int perSkill = 1024;
// Enough skills to exceed the aggregate limit
int skillCount = (limit / perSkill) + 1;
var skills = Enumerable.Range(0, skillCount)
.Select(i => ($"skill-{i}", new string('a', perSkill)))
.ToArray();
var root = CreatePluginFixture("test-plugin", skills);
try
{
var config = new CheckConfig { PluginPaths = [Path.Combine(root, "test-plugin")] };
var result = await CheckCommand.Run(config);
Assert.Equal(1, result);
}
finally { Directory.Delete(root, true); }
}
[Fact]
public async Task MultiplePlugins_IndependentLimits()
{
// Each plugin is under limit individually — both should pass
var root = Path.Combine(Path.GetTempPath(), $"check-test-{Guid.NewGuid():N}");
var plugin1 = CreatePluginInDir(root, "plugin-one",
("skill-a", "Short A."));
var plugin2 = CreatePluginInDir(root, "plugin-two",
("skill-b", "Short B."));
try
{
var config = new CheckConfig { PluginPaths = [plugin1, plugin2] };
var result = await CheckCommand.Run(config);
Assert.Equal(0, result);
}
finally { Directory.Delete(root, true); }
}
private static string CreatePluginInDir(string root, string pluginName, params (string skillName, string description)[] skills)
{
var pluginDir = Path.Combine(root, pluginName);
var skillsDir = Path.Combine(pluginDir, "skills");
Directory.CreateDirectory(pluginDir);
Directory.CreateDirectory(skillsDir);
File.WriteAllText(Path.Combine(pluginDir, "plugin.json"),
$$"""{"name":"{{pluginName}}","version":"1.0.0","description":"Test plugin.","skills":"./skills/"}""");
foreach (var (skillName, description) in skills)
{
var skillDir = Path.Combine(skillsDir, skillName);
Directory.CreateDirectory(skillDir);
File.WriteAllText(Path.Combine(skillDir, "SKILL.md"),
$"---\nname: {skillName}\ndescription: {description}\n---\n# {skillName}\n\nContent.\n");
}
return pluginDir;
}
}