mirror of
https://github.com/dotnet/skills.git
synced 2026-09-20 09:49:54 +08:00
Aditya Mandaleeka
432 lines
21 KiB
YAML
432 lines
21 KiB
YAML
# Shadow evaluation with vally — runs alongside skill-validator for comparison.
|
|
#
|
|
# This workflow reuses the gate and discover jobs from evaluation.yml via
|
|
# workflow_call, or can be triggered manually. Results are informational
|
|
# only — they never gate PRs.
|
|
#
|
|
# NOTE: This is a transitional workflow for the replat. It will be removed
|
|
# once vally fully replaces skill-validator.
|
|
name: vally-evaluation
|
|
|
|
on:
|
|
# Triggered by the main evaluation workflow after discover completes
|
|
workflow_call:
|
|
inputs:
|
|
entries:
|
|
description: 'JSON array of matrix entries from discover job'
|
|
required: true
|
|
type: string
|
|
head_sha:
|
|
description: 'PR head SHA to checkout'
|
|
required: false
|
|
type: string
|
|
default: ''
|
|
secrets:
|
|
# These are NOT passed by the caller — the copilot-pat-pool secrets are
|
|
# injected at runtime via the vally-evaluate job's `environment:`. They are
|
|
# declared here only so actionlint can type the `secrets.COPILOT_PAT_*`
|
|
# references; without this block the actionlint CI gate fails with
|
|
# "property is not defined in object type". No wildcard exists, so GitHub
|
|
# requires each name (0..9) enumerated, matching shared/pat_pool.md. Names
|
|
# with no backing secret simply resolve empty and are skipped by the selector.
|
|
COPILOT_PAT_0: { required: false }
|
|
COPILOT_PAT_1: { required: false }
|
|
COPILOT_PAT_2: { required: false }
|
|
COPILOT_PAT_3: { required: false }
|
|
COPILOT_PAT_4: { required: false }
|
|
COPILOT_PAT_5: { required: false }
|
|
COPILOT_PAT_6: { required: false }
|
|
COPILOT_PAT_7: { required: false }
|
|
COPILOT_PAT_8: { required: false }
|
|
COPILOT_PAT_9: { required: false }
|
|
# Manual trigger for testing
|
|
workflow_dispatch:
|
|
inputs:
|
|
plugin:
|
|
description: 'Plugin to evaluate (e.g., dotnet-maui)'
|
|
required: true
|
|
type: string
|
|
skill:
|
|
description: 'Skill to evaluate (optional, evaluates whole plugin if empty)'
|
|
required: false
|
|
type: string
|
|
|
|
env:
|
|
# Fallback model/judge used only if the experiment file's `overrides:` block
|
|
# can't be parsed. The experiment file is the source of truth (see below).
|
|
FALLBACK_MODEL: claude-opus-4.6
|
|
FALLBACK_JUDGE_MODEL: claude-opus-4.6
|
|
|
|
permissions:
|
|
contents: read
|
|
|
|
jobs:
|
|
# --------------------------------------------------------------------------
|
|
# For workflow_dispatch: build a single-entry matrix
|
|
# --------------------------------------------------------------------------
|
|
prepare:
|
|
if: github.event_name == 'workflow_dispatch'
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
entries: ${{ steps.build.outputs.entries }}
|
|
steps:
|
|
- id: build
|
|
shell: pwsh
|
|
env:
|
|
PLUGIN: ${{ inputs.plugin }}
|
|
SKILL: ${{ inputs.skill }}
|
|
run: |
|
|
# Validate dispatch inputs against a strict allowlist, then build the
|
|
# entries JSON. Values arrive via env (not inlined into the script) to
|
|
# avoid expression injection, and \A..\z anchor to the true string
|
|
# bounds (no trailing-newline bypass, unlike a bare $).
|
|
$p = $env:PLUGIN
|
|
$s = $env:SKILL
|
|
$namePattern = '\A[A-Za-z0-9._-]+\z'
|
|
# Reject '.' as well: it matches the charset and contains no '..', but
|
|
# a lone-dot component is a path-normalization token (e.g. plugin='.'
|
|
# broadens tests/. to the whole tree), so it must not pass.
|
|
if ($p -notmatch $namePattern -or $p -match '\.\.' -or $p -eq '.') {
|
|
throw "Invalid plugin '$p' (must match $namePattern, not be '.', and not contain '..')"
|
|
}
|
|
if ($s -and ($s -notmatch $namePattern -or $s -match '\.\.' -or $s -eq '.')) {
|
|
throw "Invalid skill '$s' (must match $namePattern, not be '.', and not contain '..')"
|
|
}
|
|
if ($s) {
|
|
$json = @(@{ name = "$p--$s"; plugin = $p; skills_path = "plugins/$p/skills/$s" }) | ConvertTo-Json -Compress -AsArray
|
|
} else {
|
|
$json = @(@{ name = $p; plugin = $p; skills_path = "plugins/$p/skills" }) | ConvertTo-Json -Compress -AsArray
|
|
}
|
|
"entries=$json" >> $env:GITHUB_OUTPUT
|
|
|
|
# --------------------------------------------------------------------------
|
|
# Run vally evaluation (shadow mode)
|
|
# --------------------------------------------------------------------------
|
|
vally-evaluate:
|
|
needs: [prepare]
|
|
if: always() && !cancelled()
|
|
runs-on: ubuntu-latest
|
|
environment: copilot-pat-pool
|
|
timeout-minutes: 180
|
|
continue-on-error: true
|
|
name: vally (${{ matrix.entry.name }})
|
|
strategy:
|
|
fail-fast: false
|
|
# Cap concurrent Copilot sessions: max-parallel jobs * --workers per job.
|
|
# Tokens come from the copilot-pat-pool (COPILOT_PAT_0..9), but keep this
|
|
# well below skill-validator's fan-out to stay under rate limits.
|
|
max-parallel: 4
|
|
matrix:
|
|
entry: ${{ fromJson(inputs.entries || needs.prepare.outputs.entries || '[]') }}
|
|
|
|
steps:
|
|
- name: Validate matrix entry
|
|
# Validate matrix values against a strict allowlist before use.
|
|
env:
|
|
ENTRY_PLUGIN: ${{ matrix.entry.plugin }}
|
|
ENTRY_NAME: ${{ matrix.entry.name }}
|
|
ENTRY_SKILLS_PATH: ${{ matrix.entry.skills_path }}
|
|
run: |
|
|
set -f
|
|
# LC_ALL=C avoids locale-dependent collation quirks in [A-Za-z] ranges.
|
|
# bash's [[ =~ ]] (POSIX ERE, no REG_NEWLINE) anchors ^/$ to the true
|
|
# start/end of the string, unlike .NET's $ which matches before a
|
|
# trailing newline — so ^...$ is sufficient here without \A/\z.
|
|
export LC_ALL=C
|
|
name_re='^[A-Za-z0-9._-]+$'
|
|
path_re='^plugins/[A-Za-z0-9._-]+/skills(/[A-Za-z0-9._-]+)?$'
|
|
# A lone-dot component ('.') matches the charset and holds no '..', but
|
|
# it is a path-normalization token, so reject it for names and for any
|
|
# path component ('.../skills/.' or 'plugins/./skills').
|
|
for val in "$ENTRY_PLUGIN" "$ENTRY_NAME"; do
|
|
if ! [[ "$val" =~ $name_re ]] || [[ "$val" == "." ]] || [[ "$val" == *".."* ]]; then
|
|
echo "::error::Invalid matrix value '$val' (must match $name_re, not be '.', and not contain '..')"; exit 1
|
|
fi
|
|
done
|
|
# Collect path segments via word-splitting (set -f prevents globbing).
|
|
# A whitespace-only value word-splits to zero segments and must
|
|
# hard-fail exactly like a truly empty one — a bare `-z` test would let
|
|
# e.g. a single space slip past and then skip the per-segment loop,
|
|
# leaving an effectively-empty skills_path to reach CLI-arg building.
|
|
segs=()
|
|
for seg in $ENTRY_SKILLS_PATH; do segs+=("$seg"); done
|
|
if [ ${#segs[@]} -eq 0 ]; then
|
|
echo "::error::Empty or whitespace-only skills_path in matrix entry"; exit 1
|
|
fi
|
|
# Each segment must additionally belong to THIS entry's plugin: the
|
|
# generic path_re only checks the shape, so bind the prefix to
|
|
# ENTRY_PLUGIN via literal comparison (ENTRY_PLUGIN is already
|
|
# allowlist-validated above). This blocks a mismatched entry like
|
|
# plugin=foo + skills_path=plugins/bar/skills, which the eval-filter
|
|
# builder would otherwise turn into nonsense paths under tests/foo.
|
|
skills_prefix="plugins/$ENTRY_PLUGIN/skills"
|
|
for seg in "${segs[@]}"; do
|
|
if ! [[ "$seg" =~ $path_re ]] || [[ "$seg" == *".."* ]]; then
|
|
echo "::error::Invalid skills_path segment '$seg' (must match $path_re and not contain '..')"; exit 1
|
|
fi
|
|
# Reject a lone-dot path component ('plugins/./skills' or '.../skills/.').
|
|
if [[ "$seg" == *"/./"* || "$seg" == */. ]]; then
|
|
echo "::error::skills_path segment '$seg' contains a '.' path component"; exit 1
|
|
fi
|
|
if [[ "$seg" != "$skills_prefix" && "$seg" != "$skills_prefix"/* ]]; then
|
|
echo "::error::skills_path segment '$seg' does not belong to plugin '$ENTRY_PLUGIN'"; exit 1
|
|
fi
|
|
done
|
|
|
|
- name: Checkout skills content
|
|
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
|
|
with:
|
|
ref: ${{ inputs.head_sha || '' }}
|
|
persist-credentials: false
|
|
|
|
- name: Setup Node.js
|
|
uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
|
|
with:
|
|
node-version: 22
|
|
|
|
- name: Install vally
|
|
run: npm install -g @microsoft/vally-cli
|
|
|
|
- name: Select Copilot token from pool
|
|
id: select-token
|
|
env:
|
|
COPILOT_PAT_0: ${{ secrets.COPILOT_PAT_0 }}
|
|
COPILOT_PAT_1: ${{ secrets.COPILOT_PAT_1 }}
|
|
COPILOT_PAT_2: ${{ secrets.COPILOT_PAT_2 }}
|
|
COPILOT_PAT_3: ${{ secrets.COPILOT_PAT_3 }}
|
|
COPILOT_PAT_4: ${{ secrets.COPILOT_PAT_4 }}
|
|
COPILOT_PAT_5: ${{ secrets.COPILOT_PAT_5 }}
|
|
COPILOT_PAT_6: ${{ secrets.COPILOT_PAT_6 }}
|
|
COPILOT_PAT_7: ${{ secrets.COPILOT_PAT_7 }}
|
|
COPILOT_PAT_8: ${{ secrets.COPILOT_PAT_8 }}
|
|
COPILOT_PAT_9: ${{ secrets.COPILOT_PAT_9 }}
|
|
run: |
|
|
# Select from the copilot-pat-pool secrets (COPILOT_PAT_0..9). There is
|
|
# no fallback secret to plumb through and track usage of — an empty
|
|
# pool is a hard configuration failure.
|
|
TOKENS=()
|
|
for i in $(seq 0 9); do
|
|
var="COPILOT_PAT_$i"
|
|
val="${!var}"
|
|
if [ -n "$val" ]; then TOKENS+=("$val"); fi
|
|
done
|
|
if [ ${#TOKENS[@]} -eq 0 ]; then
|
|
echo "::error::No copilot-pat-pool entries (COPILOT_PAT_0..9) are configured"
|
|
exit 1
|
|
fi
|
|
IDX=$(( ${RANDOM} % ${#TOKENS[@]} ))
|
|
echo "::add-mask::${TOKENS[$IDX]}"
|
|
echo "token=${TOKENS[$IDX]}" >> $GITHUB_OUTPUT
|
|
|
|
- name: Find eval specs
|
|
id: find-evals
|
|
env:
|
|
DISPATCH_SKILL: ${{ inputs.skill }}
|
|
PLUGIN: ${{ matrix.entry.plugin }}
|
|
SKILLS_PATH: ${{ matrix.entry.skills_path }}
|
|
run: |
|
|
set -f
|
|
# For workflow_dispatch targeting a specific skill, only run that one
|
|
# eval. Otherwise derive the specs from this entry's skills_path so a
|
|
# subset leg (per-skill PR entry or shard) reports has_evals accurately
|
|
# instead of proceeding through token selection/install only to exit
|
|
# empty. A whole-plugin entry (skills_path is exactly
|
|
# plugins/<plugin>/skills) keeps the recursive find. Matrix values
|
|
# arrive via env (validated upstream), and set -f keeps the
|
|
# skills_path word-split glob-safe.
|
|
if [ -n "$DISPATCH_SKILL" ]; then
|
|
CANDIDATE="tests/$PLUGIN/${DISPATCH_SKILL}/eval.vally.yaml"
|
|
if [ ! -f "$CANDIDATE" ]; then
|
|
echo "::error::workflow_dispatch requested skill '${DISPATCH_SKILL}' in plugin '${PLUGIN}' but no eval spec exists at '${CANDIDATE}'. Check the skill name (case-sensitive) and that the eval file is present."
|
|
exit 1
|
|
fi
|
|
EVALS="$CANDIDATE"
|
|
else
|
|
EVALS=""
|
|
for token in $SKILLS_PATH; do
|
|
rel="${token#plugins/}" # <plugin>/skills[/<skill>]
|
|
tp="${rel%%/skills*}" # <plugin>
|
|
skill="${rel#*/skills}" # "" or "/<skill>"
|
|
skill="${skill#/}" # "" or "<skill>"
|
|
if [ -n "$skill" ]; then
|
|
# Sharded/PR entry: only this skill's spec, if one exists.
|
|
# agent.* skills are excluded from the vally experiment.
|
|
case "$skill" in
|
|
agent.*) continue ;;
|
|
esac
|
|
CANDIDATE="tests/$tp/$skill/eval.vally.yaml"
|
|
[ -f "$CANDIDATE" ] && EVALS="${EVALS}${CANDIDATE}"$'\n'
|
|
else
|
|
# Whole-plugin entry: every eval spec under the plugin, except
|
|
# agent.* skills, which the vally experiment glob excludes (so
|
|
# counting them would report has_evals=true for zero effective
|
|
# evals).
|
|
FOUND=$(find "tests/$tp" -name "eval.vally.yaml" -type f -not -path "*/agent.*/*")
|
|
[ -n "$FOUND" ] && EVALS="${EVALS}${FOUND}"$'\n'
|
|
fi
|
|
done
|
|
EVALS=$(printf '%s' "$EVALS" | sed '/^$/d' | sort -u)
|
|
fi
|
|
if [ -z "$EVALS" ]; then
|
|
echo "No eval.vally.yaml files found for ${PLUGIN}"
|
|
echo "has_evals=false" >> $GITHUB_OUTPUT
|
|
else
|
|
echo "has_evals=true" >> $GITHUB_OUTPUT
|
|
echo "Found $(printf '%s\n' "$EVALS" | wc -l | tr -d ' ') eval specs"
|
|
fi
|
|
|
|
- name: Run vally evaluations
|
|
if: steps.find-evals.outputs.has_evals == 'true'
|
|
env:
|
|
GITHUB_TOKEN: ${{ steps.select-token.outputs.token }}
|
|
RESULTS_DIR: artifacts/TestResults/vally/${{ matrix.entry.name }}
|
|
PLUGIN: ${{ matrix.entry.plugin }}
|
|
SKILLS_PATH: ${{ matrix.entry.skills_path }}
|
|
DISPATCH_SKILL: ${{ inputs.skill }}
|
|
run: |
|
|
set -f
|
|
# PLUGIN/SKILLS_PATH/DISPATCH_SKILL arrive via env (validated upstream)
|
|
# rather than inline GitHub-expression interpolation, to avoid
|
|
# injection; set -f keeps the SKILLS_PATH word-split glob-safe.
|
|
#
|
|
# Scope the canonical experiment with --eval-filter (repeatable),
|
|
# which intersects each pattern with the experiment's `evals:` glob.
|
|
# `agent.*` evals are excluded by that glob, so they drop out
|
|
# automatically; a missing plugins/<plugin>/skills/<skill> dir surfaces
|
|
# as a skilled-variant failure that adapt.mjs warns on.
|
|
#
|
|
# Honor the matrix entry's skills_path so per-skill PR entries and
|
|
# sharded entries run only their targeted skills, not the whole plugin:
|
|
# plugins/<plugin>/skills -> tests/<plugin>/** (whole plugin)
|
|
# plugins/<plugin>/skills/<skill> -> tests/<plugin>/<skill>
|
|
# A workflow_dispatch skill input overrides to that single skill.
|
|
FILTER_ARGS=()
|
|
if [ -n "$DISPATCH_SKILL" ]; then
|
|
FILTER_ARGS=(--eval-filter "tests/$PLUGIN/$DISPATCH_SKILL/eval.vally.yaml")
|
|
else
|
|
for token in $SKILLS_PATH; do
|
|
rel="${token#plugins/}" # <plugin>/skills[/<skill>]
|
|
tp="${rel%%/skills*}" # <plugin>
|
|
skill="${rel#*/skills}" # "" or "/<skill>"
|
|
skill="${skill#/}" # "" or "<skill>"
|
|
if [ -n "$skill" ]; then
|
|
# agent.* evals are excluded from the experiment's `evals:` glob,
|
|
# and a skill without an eval.vally.yaml has nothing to run — skip
|
|
# both so we never pass a filter that matches zero evals.
|
|
if [[ "$skill" == agent.* ]]; then
|
|
echo "Skipping $tp/$skill (agent.* evals are excluded from the vally experiment)"
|
|
continue
|
|
fi
|
|
candidate="tests/$tp/$skill/eval.vally.yaml"
|
|
if [ ! -f "$candidate" ]; then
|
|
echo "Skipping $tp/$skill (no eval.vally.yaml)"
|
|
continue
|
|
fi
|
|
FILTER_ARGS+=(--eval-filter "$candidate")
|
|
else
|
|
FILTER_ARGS+=(--eval-filter "tests/$tp/**/eval.vally.yaml")
|
|
fi
|
|
done
|
|
fi
|
|
if [ ${#FILTER_ARGS[@]} -eq 0 ]; then
|
|
# Every targeted skill was skipped above (agent.* or eval-less), so
|
|
# there is nothing to evaluate for this entry.
|
|
echo "No matching vally eval filters for $PLUGIN"
|
|
exit 0
|
|
fi
|
|
|
|
# Model + comparison-judge model come from the experiment file's
|
|
# `overrides:` block (single source of truth), matching run-vally-evals.sh.
|
|
read_override() {
|
|
awk -v k="$1" '
|
|
/^overrides:/ { f = 1; next }
|
|
/^[^[:space:]]/ { f = 0 }
|
|
f && $1 == k":" { print $2; exit }
|
|
' dotnet-skills.experiment.yaml
|
|
}
|
|
MODEL="$(read_override model)"; MODEL="${MODEL:-${{ env.FALLBACK_MODEL }}}"
|
|
JUDGE_MODEL="$(read_override judge_model)"; JUDGE_MODEL="${JUDGE_MODEL:-${{ env.FALLBACK_JUDGE_MODEL }}}"
|
|
|
|
# Run baseline (no skills) + skilled (the one skill) in a single
|
|
# experiment pass. The baseline is a true skill-free control because
|
|
# vally does no ambient skill discovery — skills load only from
|
|
# environment.skills.
|
|
EXPERIMENT_OUT="$RESULTS_DIR/_experiment"
|
|
mkdir -p "$EXPERIMENT_OUT"
|
|
vally experiment run dotnet-skills.experiment.yaml \
|
|
"${FILTER_ARGS[@]}" \
|
|
--output-dir "$EXPERIMENT_OUT" \
|
|
--workers 5 \
|
|
2>&1 || echo "::warning::vally experiment run reported failures for $PLUGIN"
|
|
|
|
RUN_DIR=$(find "$EXPERIMENT_OUT" -mindepth 1 -maxdepth 1 -type d | sort | tail -1)
|
|
if [ -z "$RUN_DIR" ]; then
|
|
echo "::warning::No experiment output produced for $PLUGIN"
|
|
exit 0
|
|
fi
|
|
|
|
# Split the per-variant experiment output by eval and run `vally
|
|
# compare` per skill (in the adapter) to produce the per-skill
|
|
# results.json the shadow summary consumes. `vally` is on PATH here.
|
|
node eng/vally-adapter/adapt.mjs \
|
|
--experiment-dir "$RUN_DIR" \
|
|
--output-root "$RESULTS_DIR" \
|
|
--vally "vally" \
|
|
--model "$MODEL" \
|
|
--judge-model "$JUDGE_MODEL"
|
|
|
|
# Informational shadow job: surface how many verdicts were produced.
|
|
PRODUCED=$(find "$RESULTS_DIR" -name results.json -not -path "$EXPERIMENT_OUT/*" | wc -l | tr -d ' ')
|
|
echo "Produced $PRODUCED skill verdict(s) for $PLUGIN"
|
|
if [ "$PRODUCED" -eq 0 ]; then
|
|
echo "::warning::vally produced no skill verdicts for $PLUGIN — evals failed to run or validate (see logs above)"
|
|
fi
|
|
|
|
- name: Upload results
|
|
if: always() && steps.find-evals.outputs.has_evals == 'true'
|
|
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
|
|
with:
|
|
name: vally-results-${{ matrix.entry.name }}
|
|
path: artifacts/TestResults/vally/${{ matrix.entry.name }}/
|
|
include-hidden-files: true
|
|
retention-days: 14
|
|
|
|
- name: Write summary
|
|
if: always() && steps.find-evals.outputs.has_evals == 'true'
|
|
env:
|
|
ENTRY_NAME: ${{ matrix.entry.name }}
|
|
run: |
|
|
echo "## 🔬 Vally Shadow Results: $ENTRY_NAME" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
echo "Skilled vs baseline, judged head-to-head by \`vally compare\`. A skill passes only on a credible improvement (mean preference > 0 with its 95% CI above 0)." >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
RESULTS_DIR="artifacts/TestResults/vally/$ENTRY_NAME"
|
|
while IFS= read -r RESULTS_JSON; do
|
|
if [ ! -f "$RESULTS_JSON" ]; then continue; fi
|
|
EVAL_NAME=$(basename "$(dirname "$RESULTS_JSON")")
|
|
|
|
PASSED=$(node -e "const r=JSON.parse(require('fs').readFileSync('$RESULTS_JSON','utf-8')); console.log(r.verdicts[0].passed)")
|
|
REASON=$(node -e "const r=JSON.parse(require('fs').readFileSync('$RESULTS_JSON','utf-8')); console.log(r.verdicts[0].reason)")
|
|
ICON=$([ "$PASSED" = "true" ] && echo "✅" || echo "❌")
|
|
|
|
echo "### $ICON $EVAL_NAME" >> $GITHUB_STEP_SUMMARY
|
|
echo "$REASON" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
|
|
# Per-scenario table: mean preference score and win/tie/loss.
|
|
echo "| Scenario | Mean preference | Trials (W/T/L) |" >> $GITHUB_STEP_SUMMARY
|
|
echo "|----------|-----------------|----------------|" >> $GITHUB_STEP_SUMMARY
|
|
node -e "
|
|
const r = JSON.parse(require('fs').readFileSync('$RESULTS_JSON', 'utf-8'));
|
|
for (const s of r.verdicts[0].scenarios) {
|
|
const icon = s.meanScore > 0 ? '▲' : s.meanScore < 0 ? '▼' : '=';
|
|
const pct = (s.meanScore >= 0 ? '+' : '') + (s.meanScore * 100).toFixed(1) + '%';
|
|
let w=0,t=0,l=0;
|
|
for (const tr of (s.trials||[])) { if (tr.score>0) w++; else if (tr.score<0) l++; else t++; }
|
|
console.log('| ' + icon + ' ' + s.scenarioName + ' | ' + pct + ' | ' + w + '/' + t + '/' + l + ' |');
|
|
}
|
|
" >> $GITHUB_STEP_SUMMARY
|
|
echo "" >> $GITHUB_STEP_SUMMARY
|
|
done < <(find "$RESULTS_DIR" -name results.json | sort)
|