mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
1a20e39a8d
* Refactor skill-validator: add check command, rename validate to evaluate - Add new 'check' command for CI skill validation (frontmatter, structure) - Rename ValidateCommand to EvaluateCommand to clarify its purpose - Add skill-check.yml GitHub Actions workflow - Refactor SkillDiscovery for improved skill detection - Simplify OverfittingJudge and SkillProfiler - Update tests to match refactored APIs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix check subcommand: use EvaluateCommand as root command System.CommandLine 2.0.4 requires the primary command to be the RootCommand. Restores the upstream pattern where the evaluate/validate command is the root and check/consolidate/rejudge are subcommands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Pass all plugins in single check invocation Build plugin args dynamically and pass them all at once instead of looping per plugin. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Address PR review comments - Remove 'evaluate' subcommand token from README, evaluation-run.yml, and RunArguments in csproj (evaluate is now the root command) - Wrap ParsePluginJson in try/catch for JsonException with clean error - Add CheckCommandTests with aggregate description limit coverage (under/at/over limit and multi-plugin cases) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
134 lines
4.9 KiB
C#
134 lines
4.9 KiB
C#
using SkillValidator.Commands;
|
|
using SkillValidator.Models;
|
|
using SkillValidator.Services;
|
|
|
|
namespace SkillValidator.Tests;
|
|
|
|
public class CheckCommandAggregateDescriptionTests
|
|
{
|
|
private static string CreatePluginFixture(string pluginName, params (string skillName, string description)[] skills)
|
|
{
|
|
var root = Path.Combine(Path.GetTempPath(), $"check-test-{Guid.NewGuid():N}");
|
|
var pluginDir = Path.Combine(root, pluginName);
|
|
var skillsDir = Path.Combine(pluginDir, "skills");
|
|
|
|
Directory.CreateDirectory(pluginDir);
|
|
Directory.CreateDirectory(skillsDir);
|
|
|
|
File.WriteAllText(Path.Combine(pluginDir, "plugin.json"),
|
|
$$"""{"name":"{{pluginName}}","version":"1.0.0","description":"Test plugin.","skills":"./skills/"}""");
|
|
|
|
foreach (var (skillName, description) in skills)
|
|
{
|
|
var skillDir = Path.Combine(skillsDir, skillName);
|
|
Directory.CreateDirectory(skillDir);
|
|
File.WriteAllText(Path.Combine(skillDir, "SKILL.md"),
|
|
$"---\nname: {skillName}\ndescription: {description}\n---\n# {skillName}\n\nContent.\n");
|
|
}
|
|
|
|
return root;
|
|
}
|
|
|
|
[Fact]
|
|
public async Task UnderAggregateLimit_Passes()
|
|
{
|
|
var root = CreatePluginFixture("test-plugin",
|
|
("skill-a", "Short description A."),
|
|
("skill-b", "Short description B."));
|
|
try
|
|
{
|
|
var config = new CheckConfig { PluginPaths = [Path.Combine(root, "test-plugin")] };
|
|
var result = await CheckCommand.Run(config);
|
|
Assert.Equal(0, result);
|
|
}
|
|
finally { Directory.Delete(root, true); }
|
|
}
|
|
|
|
[Fact]
|
|
public async Task AtAggregateLimit_Passes()
|
|
{
|
|
// Create skills whose descriptions sum exactly to the limit, each under per-skill max (1024)
|
|
int limit = SkillProfiler.MaxAggregateDescriptionLength;
|
|
int perSkill = 1024;
|
|
int skillCount = limit / perSkill;
|
|
int remainder = limit - (skillCount * perSkill);
|
|
|
|
var skills = Enumerable.Range(0, skillCount)
|
|
.Select(i => ($"skill-{i}", new string('a', perSkill)))
|
|
.ToList();
|
|
if (remainder > 0)
|
|
skills.Add(($"skill-extra", new string('a', remainder)));
|
|
|
|
var root = CreatePluginFixture("test-plugin", skills.ToArray());
|
|
try
|
|
{
|
|
var config = new CheckConfig { PluginPaths = [Path.Combine(root, "test-plugin")] };
|
|
var result = await CheckCommand.Run(config);
|
|
Assert.Equal(0, result);
|
|
}
|
|
finally { Directory.Delete(root, true); }
|
|
}
|
|
|
|
[Fact]
|
|
public async Task OverAggregateLimit_Fails()
|
|
{
|
|
int limit = SkillProfiler.MaxAggregateDescriptionLength;
|
|
int perSkill = 1024;
|
|
// Enough skills to exceed the aggregate limit
|
|
int skillCount = (limit / perSkill) + 1;
|
|
|
|
var skills = Enumerable.Range(0, skillCount)
|
|
.Select(i => ($"skill-{i}", new string('a', perSkill)))
|
|
.ToArray();
|
|
|
|
var root = CreatePluginFixture("test-plugin", skills);
|
|
try
|
|
{
|
|
var config = new CheckConfig { PluginPaths = [Path.Combine(root, "test-plugin")] };
|
|
var result = await CheckCommand.Run(config);
|
|
Assert.Equal(1, result);
|
|
}
|
|
finally { Directory.Delete(root, true); }
|
|
}
|
|
|
|
[Fact]
|
|
public async Task MultiplePlugins_IndependentLimits()
|
|
{
|
|
// Each plugin is under limit individually — both should pass
|
|
var root = Path.Combine(Path.GetTempPath(), $"check-test-{Guid.NewGuid():N}");
|
|
var plugin1 = CreatePluginInDir(root, "plugin-one",
|
|
("skill-a", "Short A."));
|
|
var plugin2 = CreatePluginInDir(root, "plugin-two",
|
|
("skill-b", "Short B."));
|
|
try
|
|
{
|
|
var config = new CheckConfig { PluginPaths = [plugin1, plugin2] };
|
|
var result = await CheckCommand.Run(config);
|
|
Assert.Equal(0, result);
|
|
}
|
|
finally { Directory.Delete(root, true); }
|
|
}
|
|
|
|
private static string CreatePluginInDir(string root, string pluginName, params (string skillName, string description)[] skills)
|
|
{
|
|
var pluginDir = Path.Combine(root, pluginName);
|
|
var skillsDir = Path.Combine(pluginDir, "skills");
|
|
|
|
Directory.CreateDirectory(pluginDir);
|
|
Directory.CreateDirectory(skillsDir);
|
|
|
|
File.WriteAllText(Path.Combine(pluginDir, "plugin.json"),
|
|
$$"""{"name":"{{pluginName}}","version":"1.0.0","description":"Test plugin.","skills":"./skills/"}""");
|
|
|
|
foreach (var (skillName, description) in skills)
|
|
{
|
|
var skillDir = Path.Combine(skillsDir, skillName);
|
|
Directory.CreateDirectory(skillDir);
|
|
File.WriteAllText(Path.Combine(skillDir, "SKILL.md"),
|
|
$"---\nname: {skillName}\ndescription: {description}\n---\n# {skillName}\n\nContent.\n");
|
|
}
|
|
|
|
return pluginDir;
|
|
}
|
|
}
|