mirror of
https://github.com/rtk-ai/rtk.git
synced 2026-09-19 07:33:17 +08:00
cc15ea7a8c
Cloud-init VM now installs 50+ real tools (arch-aware for ARM/x86), clones 14 open-source projects (ripgrep, Flask, Hugo, Vite, Redis, Newtonsoft.Json, Polars, dbt-core, scikit-learn, Vue, T3, Sinatra, Laravel, Spring Petclinic), and pre-builds dependencies. New Phase 12: Token savings measured on real codebases instead of toy projects. Covers Rust, Python, Go, JS/TS, C, C#, Ruby, Java. New Phase 13: AI Comprehension Quality benchmark — asks Claude Haiku the same factual questions on raw vs filtered output to validate that RTK preserves semantic content while removing 85-99% of tokens. Uses claude CLI auth (no API key needed) with ANTHROPIC_API_KEY fallback. Signed-off-by: Patrick szymkowiak <pszymkowiak@innovtech.eu> Signed-off-by: Patrick szymkowiak <patrick.szymkowiak@innovtech.eu>
763 lines
27 KiB
YAML
763 lines
27 KiB
YAML
#cloud-config
|
|
# RTK Integration Test VM — Ubuntu 24.04
|
|
# Installs all tools needed for comprehensive RTK testing (~100 commands)
|
|
# Usage: multipass launch --name rtk-test --cloud-init scripts/benchmark/cloud-init.yaml --cpus 4 --memory 6G --disk 30G 24.04
|
|
#
|
|
# Architecture-aware: detects aarch64 vs x86_64 automatically
|
|
|
|
package_update: true
|
|
package_upgrade: false
|
|
|
|
packages:
|
|
# System tools
|
|
- curl
|
|
- wget
|
|
- jq
|
|
- git
|
|
- make
|
|
- cmake
|
|
- rsync
|
|
- sqlite3
|
|
- shellcheck
|
|
- yamllint
|
|
- postgresql-client
|
|
- docker.io
|
|
- containerd
|
|
- python3
|
|
- python3-pip
|
|
- python3-venv
|
|
- pipx
|
|
# Build essentials (for Rust compilation)
|
|
- build-essential
|
|
- pkg-config
|
|
- libssl-dev
|
|
- libsqlite3-dev
|
|
# Additional system tools for TOML filters
|
|
- iptables
|
|
- net-tools
|
|
- sysstat
|
|
- fail2ban
|
|
- openssh-server
|
|
# Ruby ecosystem
|
|
- ruby
|
|
- ruby-dev
|
|
- bundler
|
|
# Java/Gradle/Maven
|
|
- default-jdk
|
|
- maven
|
|
- gradle
|
|
# Misc
|
|
- hyperfine
|
|
- unzip
|
|
- tree
|
|
- gcc
|
|
- g++
|
|
# PHP
|
|
- php-cli
|
|
- composer
|
|
|
|
runcmd:
|
|
# ── Detect architecture ──
|
|
- |
|
|
ARCH=$(uname -m)
|
|
if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then
|
|
GO_ARCH="arm64"
|
|
KUBECTL_ARCH="arm64"
|
|
HADOLINT_ARCH="arm64"
|
|
else
|
|
GO_ARCH="amd64"
|
|
KUBECTL_ARCH="amd64"
|
|
HADOLINT_ARCH="x86_64"
|
|
fi
|
|
echo "ARCH=$ARCH GO_ARCH=$GO_ARCH" > /tmp/rtk-arch.env
|
|
|
|
# ── Rust toolchain ──
|
|
- su - ubuntu -c 'curl --proto "=https" --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y'
|
|
- su - ubuntu -c '$HOME/.cargo/bin/cargo install cargo-nextest 2>/dev/null || true'
|
|
|
|
# ── Node.js 22 + package managers + JS tools ──
|
|
- curl -fsSL https://deb.nodesource.com/setup_22.x | bash -
|
|
- apt-get install -y nodejs
|
|
- npm install -g pnpm yarn
|
|
- npm install -g eslint prettier typescript
|
|
- npm install -g markdownlint-cli
|
|
- npm install -g @biomejs/biome
|
|
- npm install -g oxlint
|
|
- npm install -g turbo
|
|
- npm install -g nx
|
|
- npm install -g vitest jest
|
|
- npm install -g next
|
|
- npm install -g prisma
|
|
# Playwright needs browsers — skip install, just need the CLI
|
|
- npm install -g @playwright/test || true
|
|
|
|
# ── Go (arch-aware) ──
|
|
- |
|
|
. /tmp/rtk-arch.env
|
|
curl -fsSL "https://go.dev/dl/go1.22.5.linux-${GO_ARCH}.tar.gz" | tar -C /usr/local -xz
|
|
- echo 'export PATH=$PATH:/usr/local/go/bin:/home/ubuntu/go/bin' >> /home/ubuntu/.bashrc
|
|
- su - ubuntu -c 'export PATH=$PATH:/usr/local/go/bin && go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest'
|
|
|
|
# ── Python tools ──
|
|
- pipx install ruff
|
|
- pipx install mypy
|
|
- pipx install poetry
|
|
- pip3 install --break-system-packages pytest uv pre-commit basedpyright
|
|
|
|
# ── .NET 8 SDK ──
|
|
- |
|
|
wget https://dot.net/v1/dotnet-install.sh -O /tmp/dotnet-install.sh
|
|
chmod +x /tmp/dotnet-install.sh
|
|
/tmp/dotnet-install.sh --channel 8.0 --install-dir /usr/local/share/dotnet
|
|
ln -sf /usr/local/share/dotnet/dotnet /usr/local/bin/dotnet
|
|
echo 'export DOTNET_ROOT=/usr/local/share/dotnet' >> /home/ubuntu/.bashrc
|
|
|
|
# ── Terraform ──
|
|
- |
|
|
wget -qO- https://apt.releases.hashicorp.com/gpg | gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
|
|
echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" > /etc/apt/sources.list.d/hashicorp.list
|
|
apt-get update && apt-get install -y terraform
|
|
|
|
# ── OpenTofu ──
|
|
- |
|
|
curl -fsSL https://get.opentofu.org/install-opentofu.sh | bash -s -- --install-method deb 2>/dev/null || true
|
|
|
|
# ── Helm ──
|
|
- curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
|
|
|
|
# ── Hadolint (arch-aware) ──
|
|
- |
|
|
. /tmp/rtk-arch.env
|
|
wget -qO /usr/local/bin/hadolint "https://github.com/hadolint/hadolint/releases/latest/download/hadolint-Linux-${HADOLINT_ARCH}" 2>/dev/null || true
|
|
chmod +x /usr/local/bin/hadolint 2>/dev/null || true
|
|
|
|
# ── Docker setup ──
|
|
- usermod -aG docker ubuntu
|
|
- systemctl enable docker
|
|
- systemctl start docker
|
|
|
|
# ── kubectl (arch-aware) ──
|
|
- |
|
|
. /tmp/rtk-arch.env
|
|
curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${KUBECTL_ARCH}/kubectl"
|
|
install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
|
|
rm -f kubectl
|
|
|
|
# ── ansible ──
|
|
- pip3 install --break-system-packages ansible-core
|
|
|
|
# ── Ruby tools ──
|
|
- gem install rspec rubocop rake bundler --no-document 2>/dev/null || true
|
|
|
|
# ── just (task runner) ──
|
|
- |
|
|
curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to /usr/local/bin 2>/dev/null || true
|
|
|
|
# ── jj (Jujutsu VCS) ──
|
|
- |
|
|
. /tmp/rtk-arch.env
|
|
if [ "$GO_ARCH" = "amd64" ]; then
|
|
curl -fsSL https://github.com/jj-vcs/jj/releases/latest/download/jj-latest-linux-x86_64.tar.gz | tar xz -C /usr/local/bin jj 2>/dev/null || true
|
|
fi
|
|
|
|
# ── AWS CLI ──
|
|
- |
|
|
. /tmp/rtk-arch.env
|
|
if [ "$GO_ARCH" = "amd64" ]; then
|
|
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o /tmp/awscliv2.zip
|
|
else
|
|
curl "https://awscli.amazonaws.com/awscli-exe-linux-aarch64.zip" -o /tmp/awscliv2.zip
|
|
fi
|
|
cd /tmp && unzip -q awscliv2.zip && ./aws/install 2>/dev/null || true
|
|
rm -rf /tmp/aws /tmp/awscliv2.zip
|
|
|
|
# ── gh CLI ──
|
|
- |
|
|
curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg | dd of=/usr/share/keyrings/githubcli-archive-keyring.gpg
|
|
echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" > /etc/apt/sources.list.d/github-cli.list
|
|
apt-get update && apt-get install -y gh
|
|
|
|
# ── Mock tools (too heavy or proprietary to install for real) ──
|
|
- |
|
|
cat > /usr/local/bin/gcloud << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "version" ] || [ "$1" = "--version" ]; then
|
|
echo "Google Cloud SDK 500.0.0"
|
|
echo "alpha 2026.04.01"
|
|
echo "beta 2026.04.01"
|
|
echo "bq 2.1.10"
|
|
echo "bundled-python3-unix 3.12.8"
|
|
echo "core 2026.04.01"
|
|
echo "gcloud-crc32c 1.0.0"
|
|
echo "gsutil 5.31"
|
|
else
|
|
echo "gcloud mock: $*"
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/gcloud
|
|
|
|
- |
|
|
cat > /usr/local/bin/shopify << 'MOCK'
|
|
#!/bin/bash
|
|
echo "Shopify CLI 3.70.0 (mock)"
|
|
if [ "$1" = "theme" ] && [ "$2" = "check" ]; then
|
|
echo "Running theme check..."
|
|
echo " 3 issues found"
|
|
echo " [error] templates/index.liquid: Missing required section 'header'"
|
|
echo " [warn] assets/theme.css: Missing alt text on image in line 42"
|
|
echo " [info] config/settings_schema.json: Consider adding description to setting 'font_size'"
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/shopify
|
|
|
|
- |
|
|
cat > /usr/local/bin/pio << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "--version" ]; then echo "PlatformIO Core, version 6.1.15"
|
|
elif [ "$1" = "run" ]; then
|
|
echo "Processing esp32dev (platform: espressif32; board: esp32dev; framework: arduino)"
|
|
echo "PACKAGES:"
|
|
echo " - framework-arduinoespressif32 @ 3.20017.0 (3.0.7)"
|
|
echo " - tool-esptoolpy @ 1.40501.0 (4.5.1)"
|
|
echo "Compiling .pio/build/esp32dev/src/main.cpp.o"
|
|
echo "Linking .pio/build/esp32dev/firmware.elf"
|
|
echo "Building .pio/build/esp32dev/firmware.bin"
|
|
echo "RAM: [= ] 12.5% (used 40960 bytes from 327680 bytes)"
|
|
echo "Flash: [== ] 24.3% (used 318464 bytes from 1310720 bytes)"
|
|
echo "========================= [SUCCESS] Took 12.34 seconds ========================="
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/pio
|
|
|
|
- |
|
|
cat > /usr/local/bin/quarto << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "--version" ]; then echo "1.5.57"
|
|
elif [ "$1" = "render" ]; then
|
|
echo " Cell 1/5: Running Python code..."
|
|
echo " Cell 2/5: Generating plot..."
|
|
echo " Cell 3/5: Creating table..."
|
|
echo " Cell 4/5: Processing markdown..."
|
|
echo " Cell 5/5: Finalizing..."
|
|
echo "Output created: _output/document.html"
|
|
echo "Output created: _output/document.pdf"
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/quarto
|
|
|
|
- |
|
|
cat > /usr/local/bin/sops << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "--version" ]; then echo "sops 3.9.4"
|
|
elif [ "$1" = "-d" ]; then echo '{"database_url": "postgres://user:***@localhost/db", "api_key": "sk-***"}'
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/sops
|
|
|
|
- |
|
|
cat > /usr/local/bin/swift << 'MOCK'
|
|
#!/bin/bash
|
|
case "$1" in
|
|
--version) echo "Swift version 5.10.1 (swift-5.10.1-RELEASE)" ;;
|
|
build)
|
|
echo "Compiling MyApp (14 sources)..."
|
|
echo "Compiling MyLib (8 sources)..."
|
|
echo "Linking MyApp"
|
|
echo "Build complete! (2.34s)" ;;
|
|
test)
|
|
echo "Test Suite 'All tests' started at 2026-04-16 10:00:00"
|
|
echo "Test Suite 'MyAppTests.xctest' started"
|
|
echo "Test Case '-[MyAppTests.MathTests testAdd]' passed (0.001s)"
|
|
echo "Test Case '-[MyAppTests.MathTests testSubtract]' passed (0.001s)"
|
|
echo "Test Case '-[MyAppTests.StringTests testConcat]' failed (0.002s)"
|
|
echo " XCTAssertEqual failed: (\"hello\") is not equal to (\"world\")"
|
|
echo "Test Suite 'MyAppTests.xctest' finished: 2 passed, 1 failed (0.004s)"
|
|
echo "Test Suite 'All tests' finished: 2 passed, 1 failed (0.005s)"
|
|
exit 1 ;;
|
|
package) echo "Resolving dependencies..." ; echo "Resolved 3 packages (0.5s)" ;;
|
|
*) echo "swift mock: $*" ;;
|
|
esac
|
|
MOCK
|
|
chmod +x /usr/local/bin/swift
|
|
|
|
- |
|
|
cat > /usr/local/bin/swiftlint << 'MOCK'
|
|
#!/bin/bash
|
|
echo "Linting Swift files in current directory..."
|
|
echo "Sources/App/main.swift:12:1: warning: Line Length Violation: Line should be 120 characters or less; currently it has 145 characters (line_length)"
|
|
echo "Sources/App/main.swift:25:5: warning: Force Cast Violation: Force casts should be avoided (force_cast)"
|
|
echo "Sources/App/Models/User.swift:8:1: error: Type Body Length Violation: Type body should span 250 lines or less excluding comments and whitespace: currently spans 312 lines (type_body_length)"
|
|
echo "Sources/App/Views/Home.swift:15:10: warning: Unused Import Violation: All imported modules should be used (unused_import)"
|
|
echo "Done linting! Found 3 warnings and 1 error in 4 files."
|
|
exit 1
|
|
MOCK
|
|
chmod +x /usr/local/bin/swiftlint
|
|
|
|
- |
|
|
cat > /usr/local/bin/swiftformat << 'MOCK'
|
|
#!/bin/bash
|
|
echo "Running SwiftFormat..."
|
|
echo "Sources/App/main.swift: formatted (2 changes)"
|
|
echo "Sources/App/Models/User.swift: formatted (1 change)"
|
|
echo "Sources/App/Views/Home.swift: already formatted"
|
|
echo "3 files processed, 2 formatted (3 changes total)"
|
|
MOCK
|
|
chmod +x /usr/local/bin/swiftformat
|
|
|
|
- |
|
|
cat > /usr/local/bin/ollama << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "list" ]; then
|
|
printf "NAME\tID\tSIZE\tMODIFIED\n"
|
|
printf "llama3.2:latest\ta1b2c3d4\t4.7 GB\t2 days ago\n"
|
|
printf "codellama:13b\te5f6a7b8\t7.4 GB\t5 days ago\n"
|
|
printf "mistral:latest\tc9d0e1f2\t4.1 GB\t1 week ago\n"
|
|
printf "phi3:mini\tg3h4i5j6\t2.3 GB\t2 weeks ago\n"
|
|
elif [ "$1" = "ps" ]; then
|
|
printf "NAME\tID\tSIZE\tPROCESSOR\tUNTIL\n"
|
|
printf "llama3.2:latest\ta1b2c3d4\t6.2 GB\t100%% GPU\t4 minutes from now\n"
|
|
elif [ "$1" = "--version" ]; then
|
|
echo "ollama version 0.6.2"
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/ollama
|
|
|
|
- |
|
|
cat > /usr/local/bin/skopeo << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "--version" ]; then echo "skopeo 1.16.1"
|
|
elif [ "$1" = "inspect" ]; then
|
|
echo '{"Name":"docker.io/library/ubuntu","Tag":"latest","Digest":"sha256:abc123","Created":"2026-04-01T00:00:00Z","Architecture":"amd64","Os":"linux","Layers":["sha256:layer1","sha256:layer2","sha256:layer3"]}'
|
|
elif [ "$1" = "list-tags" ]; then
|
|
echo '{"Repository":"docker.io/library/ubuntu","Tags":["24.04","22.04","20.04","latest","noble","jammy","focal"]}'
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/skopeo
|
|
|
|
- |
|
|
cat > /usr/local/bin/mise << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "--version" ]; then echo "mise 2026.4.1"
|
|
elif [ "$1" = "ls" ]; then
|
|
echo "Tool Version Config Source Requested"
|
|
echo "node 22.0.0 ~/.tool-versions 22"
|
|
echo "python 3.12.3 ~/.tool-versions 3.12"
|
|
echo "ruby 3.3.1 ~/.tool-versions 3.3"
|
|
echo "go 1.22.5 ~/.tool-versions 1.22"
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/mise
|
|
|
|
- |
|
|
cat > /usr/local/bin/gt << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "--version" ]; then echo "gt 1.4.2"
|
|
elif [ "$1" = "log" ] || [ "$1" = "ls" ]; then
|
|
echo " main"
|
|
echo " ├── feat/auth-flow (3 commits, 2 days ago)"
|
|
echo " │ └── fix/auth-redirect (1 commit, 1 hour ago)"
|
|
echo " ├── feat/dashboard (5 commits, 1 day ago)"
|
|
echo " └── chore/deps-update (1 commit, 3 hours ago)"
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/gt
|
|
|
|
- |
|
|
cat > /usr/local/bin/psql << 'MOCK'
|
|
#!/bin/bash
|
|
if [ "$1" = "--version" ]; then echo "psql (PostgreSQL) 16.3"
|
|
else
|
|
echo " id | name | email | created_at"
|
|
echo "----+------------+--------------------+----------------------------"
|
|
echo " 1 | Alice | alice@example.com | 2026-01-15 10:30:00+00"
|
|
echo " 2 | Bob | bob@example.com | 2026-02-20 14:15:00+00"
|
|
echo " 3 | Charlie | charlie@test.com | 2026-03-01 09:00:00+00"
|
|
echo "(3 rows)"
|
|
fi
|
|
MOCK
|
|
chmod +x /usr/local/bin/psql
|
|
|
|
# ── Fake test projects ──
|
|
|
|
# Node.js project with errors (vitest + eslint + prettier + tsc)
|
|
- |
|
|
su - ubuntu -c '
|
|
mkdir -p /tmp/test-node/src && cd /tmp/test-node
|
|
npm init -y >/dev/null 2>&1
|
|
echo "{\"compilerOptions\":{\"strict\":true,\"noEmit\":true,\"target\":\"ES2020\",\"module\":\"ESNext\",\"moduleResolution\":\"node\"},\"include\":[\"src\"]}" > tsconfig.json
|
|
cat > src/index.ts << "TSEOF"
|
|
const x: number = "not a number";
|
|
const unused = 42;
|
|
function greet(name: string): string { return name }
|
|
greet(123);
|
|
export function add(a: number, b: number): number { return a + b; }
|
|
export function multiply(a: number, b: number) { return a * b; }
|
|
TSEOF
|
|
echo "{\"rules\":{\"no-unused-vars\":\"error\",\"semi\":[\"error\",\"always\"]}}" > .eslintrc.json
|
|
echo "const x = 1;const y=2; const z =3" > src/ugly.ts
|
|
pnpm init 2>/dev/null || true
|
|
'
|
|
|
|
# Python project with errors
|
|
- |
|
|
su - ubuntu -c '
|
|
mkdir -p /tmp/test-python && cd /tmp/test-python
|
|
cat > main.py << "PYEOF"
|
|
import os
|
|
import sys
|
|
import json
|
|
unused_import = 1
|
|
def add(a: int, b: int) -> str:
|
|
return a + b
|
|
x: int = "hello"
|
|
class User:
|
|
def __init__(self, name: str, age: int):
|
|
self.name = name
|
|
self.age = age
|
|
def greet(self) -> str:
|
|
return f"Hello, {self.name}"
|
|
PYEOF
|
|
cat > test_main.py << "PYEOF"
|
|
from main import add, User
|
|
def test_pass():
|
|
assert 1 + 1 == 2
|
|
def test_fail():
|
|
assert 1 + 1 == 3, "math is broken"
|
|
def test_user():
|
|
u = User("Alice", 30)
|
|
assert u.greet() == "Hello, Alice"
|
|
def test_add_types():
|
|
result = add(1, 2)
|
|
assert result == 3
|
|
PYEOF
|
|
cat > pyproject.toml << "PYEOF"
|
|
[tool.ruff]
|
|
line-length = 80
|
|
select = ["E", "F", "W", "I"]
|
|
[tool.mypy]
|
|
strict = true
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["."]
|
|
PYEOF
|
|
'
|
|
|
|
# Go project with errors
|
|
- |
|
|
su - ubuntu -c '
|
|
export PATH=$PATH:/usr/local/go/bin
|
|
mkdir -p /tmp/test-go && cd /tmp/test-go
|
|
go mod init test-go 2>/dev/null
|
|
cat > main.go << "GOEOF"
|
|
package main
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
func main() {
|
|
msg := strings.ToUpper("hello world")
|
|
fmt.Println(msg)
|
|
}
|
|
func unused() { var x int; _ = x }
|
|
func badStyle() string {
|
|
var s string = "test"
|
|
return s
|
|
}
|
|
GOEOF
|
|
cat > main_test.go << "GOEOF"
|
|
package main
|
|
import "testing"
|
|
func TestPass(t *testing.T) { if 1+1 != 2 { t.Fatal("math") } }
|
|
func TestPass2(t *testing.T) { if 2*3 != 6 { t.Fatal("math2") } }
|
|
func TestFail(t *testing.T) { t.Fatal("expected failure for testing") }
|
|
func TestBadStyle(t *testing.T) {
|
|
result := badStyle()
|
|
if result != "test" { t.Fatalf("expected test, got %s", result) }
|
|
}
|
|
GOEOF
|
|
'
|
|
|
|
# Rust project with errors and warnings
|
|
- |
|
|
su - ubuntu -c '
|
|
export PATH=$HOME/.cargo/bin:$PATH
|
|
mkdir -p /tmp/test-rust && cd /tmp/test-rust
|
|
cargo init --name test-rust 2>/dev/null
|
|
cat > src/main.rs << "RSEOF"
|
|
fn main() {
|
|
let x = vec![1, 2, 3, 4, 5];
|
|
let _y = x.iter().map(|i| i.clone()).collect::<Vec<_>>();
|
|
let _unused = 42;
|
|
println!("hello from test-rust");
|
|
}
|
|
fn unused_function() -> String {
|
|
"this is never called".to_string()
|
|
}
|
|
#[cfg(test)]
|
|
mod tests {
|
|
#[test] fn test_pass_1() { assert_eq!(1 + 1, 2); }
|
|
#[test] fn test_pass_2() { assert_eq!(2 + 2, 4); }
|
|
#[test] fn test_pass_3() { assert_eq!(3 * 3, 9); }
|
|
#[test] fn test_pass_4() { assert_eq!(10 / 2, 5); }
|
|
#[test] fn test_fail() { assert_eq!(1 + 1, 3, "intentional failure"); }
|
|
#[test] fn test_fail_2() { panic!("another intentional failure"); }
|
|
}
|
|
RSEOF
|
|
cargo build 2>/dev/null || true
|
|
'
|
|
|
|
# Ruby project with rspec + rubocop
|
|
- |
|
|
su - ubuntu -c '
|
|
mkdir -p /tmp/test-ruby/spec && cd /tmp/test-ruby
|
|
cat > app.rb << "RBEOF"
|
|
class Calculator
|
|
def add(a, b)
|
|
a + b
|
|
end
|
|
def divide(a, b)
|
|
a / b
|
|
end
|
|
end
|
|
RBEOF
|
|
cat > spec/calculator_spec.rb << "RBEOF"
|
|
require_relative "../app"
|
|
RSpec.describe Calculator do
|
|
it "adds two numbers" do
|
|
expect(Calculator.new.add(1, 2)).to eq(3)
|
|
end
|
|
it "fails intentionally" do
|
|
expect(Calculator.new.add(1, 2)).to eq(4)
|
|
end
|
|
end
|
|
RBEOF
|
|
'
|
|
|
|
# .NET project
|
|
- |
|
|
export DOTNET_ROOT=/usr/local/share/dotnet
|
|
su - ubuntu -c '
|
|
export DOTNET_ROOT=/usr/local/share/dotnet && export PATH=$PATH:$DOTNET_ROOT
|
|
mkdir -p /tmp/test-dotnet && cd /tmp/test-dotnet
|
|
dotnet new console -n TestApp --force 2>/dev/null || true
|
|
'
|
|
|
|
# Dockerfiles for hadolint
|
|
- |
|
|
su - ubuntu -c '
|
|
cat > /tmp/Dockerfile.bad << "DEOF"
|
|
FROM ubuntu:latest
|
|
RUN apt-get update && apt-get install -y curl wget git vim nano
|
|
RUN cd /tmp && wget http://example.com/script.sh && bash script.sh
|
|
RUN pip install flask django requests
|
|
EXPOSE 80 443 8080
|
|
COPY . /app
|
|
WORKDIR /app
|
|
CMD ["python", "app.py"]
|
|
DEOF
|
|
'
|
|
|
|
# Shell/YAML/Markdown test files
|
|
- |
|
|
su - ubuntu -c '
|
|
cat > /tmp/test.sh << "SHEOF"
|
|
#!/bin/bash
|
|
echo $foo
|
|
ls *.txt
|
|
cd $(pwd)
|
|
[ -f file ] && rm file
|
|
for i in 1 2 3; do
|
|
echo $i
|
|
done
|
|
cat /etc/passwd | grep root
|
|
SHEOF
|
|
printf "foo: bar\nbaz: qux\nlist:\n - item1\n - item2\ntruthy: yes\npassword: secret123\n" > /tmp/test.yaml
|
|
printf "#Header without space\nSome text with [broken link]()\n\n* List item\n+ Mixed markers\n- Another item\n\n## No blank line before heading\nParagraph\n" > /tmp/test.md
|
|
echo "{\"name\":\"test\",\"version\":\"1.0\",\"dependencies\":{\"express\":\"4.18\",\"lodash\":\"4.17\",\"axios\":\"1.6\"},\"scripts\":{\"start\":\"node index.js\",\"test\":\"jest\",\"build\":\"tsc\"}}" > /tmp/test.json
|
|
'
|
|
|
|
# Git repo with rich history (30 commits with bodies)
|
|
- |
|
|
su - ubuntu -c '
|
|
mkdir -p /tmp/test-git && cd /tmp/test-git
|
|
git init && git config user.email "test@rtk.dev" && git config user.name "RTK Test"
|
|
for i in $(seq 1 30); do
|
|
echo "line $i content for feature $i" >> file.txt
|
|
scope=$( [ $((i%3)) -eq 0 ] && echo "core" || ( [ $((i%3)) -eq 1 ] && echo "ui" || echo "api" ) )
|
|
git add file.txt
|
|
git commit -m "feat($scope): implement feature number $i - closes #$i"
|
|
done
|
|
echo "modified content for testing diff" >> file.txt
|
|
echo "brand new file" > new.txt
|
|
echo "another new file with content" > another.txt
|
|
git add new.txt
|
|
git branch feature/auth
|
|
git branch fix/login-bug
|
|
'
|
|
|
|
# Large log file for dedup testing (more varied)
|
|
- |
|
|
su - ubuntu -c '
|
|
for i in $(seq 1 200); do
|
|
printf "[2026-04-16 10:00:%02d] INFO Starting service on port 8080...\n" $((i%60))
|
|
printf "[2026-04-16 10:00:%02d] INFO Connected to database postgres://localhost:5432/app\n" $((i%60))
|
|
printf "[2026-04-16 10:00:%02d] WARN Connection pool exhausted, retry=%d backoff=100ms\n" $((i%60)) $i
|
|
printf "[2026-04-16 10:00:%02d] ERROR Failed to process request: connection refused (attempt %d/3)\n" $((i%60)) $((i%3+1))
|
|
done > /tmp/large.log
|
|
for i in $(seq 1 50); do echo "[2026-04-16 10:05:00] FATAL Out of memory: heap allocation failed (requested 256MB, available 12MB)"; done >> /tmp/large.log
|
|
for i in $(seq 1 100); do echo "[2026-04-16 10:06:$((i%60))] DEBUG Request processed in ${i}ms path=/api/v1/users method=GET status=200"; done >> /tmp/large.log
|
|
'
|
|
|
|
# .env file
|
|
- |
|
|
su - ubuntu -c '
|
|
printf "DATABASE_URL=postgres://user:pass@localhost:5432/db\nAPI_KEY=sk-1234567890abcdef\nSECRET_TOKEN=ghp_xxxx\nNODE_ENV=production\nPORT=3000\nREDIS_URL=redis://localhost:6379\n" > /tmp/.env
|
|
'
|
|
|
|
# Makefile (bigger)
|
|
- |
|
|
su - ubuntu -c '
|
|
cat > /tmp/Makefile << "MKEOF"
|
|
.PHONY: all test clean build lint
|
|
all: build test
|
|
build:
|
|
@echo "Compiling main module..."
|
|
@echo "Compiling auth module..."
|
|
@echo "Compiling database module..."
|
|
@echo "Compiling api module..."
|
|
@echo "Linking..."
|
|
@echo "Build complete (4 modules compiled)"
|
|
test:
|
|
@echo "Running unit tests..."
|
|
@echo " test_auth: PASS"
|
|
@echo " test_database: PASS"
|
|
@echo " test_api: PASS"
|
|
@echo " test_integration: FAIL"
|
|
@echo "4 tests run, 3 passed, 1 failed"
|
|
lint:
|
|
@echo "Linting 12 files..."
|
|
@echo " 2 warnings found"
|
|
clean:
|
|
@echo "Cleaning build artifacts..."
|
|
@echo "Removed 15 files"
|
|
MKEOF
|
|
'
|
|
|
|
# Terraform project
|
|
- |
|
|
su - ubuntu -c '
|
|
mkdir -p /tmp/test-terraform && cd /tmp/test-terraform
|
|
cat > main.tf << "TFEOF"
|
|
terraform {
|
|
required_version = ">= 1.0"
|
|
required_providers {
|
|
aws = { source = "hashicorp/aws", version = "~> 5.0" }
|
|
}
|
|
}
|
|
provider "aws" { region = "eu-west-1" }
|
|
resource "aws_instance" "web" {
|
|
ami = "ami-0abcdef1234567890"
|
|
instance_type = "t3.micro"
|
|
tags = { Name = "web-server" }
|
|
}
|
|
resource "aws_s3_bucket" "data" {
|
|
bucket = "my-data-bucket"
|
|
}
|
|
output "instance_ip" { value = aws_instance.web.public_ip }
|
|
TFEOF
|
|
'
|
|
|
|
# Helm chart
|
|
- su - ubuntu -c 'mkdir -p /tmp/test-helm && cd /tmp/test-helm && helm create test-chart 2>/dev/null || true'
|
|
|
|
# Justfile for just runner
|
|
- |
|
|
su - ubuntu -c '
|
|
cat > /tmp/justfile << "JEOF"
|
|
default: build test
|
|
|
|
build:
|
|
@echo "Building project..."
|
|
@echo "Done."
|
|
|
|
test:
|
|
@echo "Running tests..."
|
|
@echo "All tests passed."
|
|
|
|
lint:
|
|
@echo "Linting..."
|
|
JEOF
|
|
'
|
|
|
|
# ══════════════════════════════════════════════════════════
|
|
# Real open-source projects for token savings benchmarks
|
|
# Each project covers a developer profile/language ecosystem
|
|
# ══════════════════════════════════════════════════════════
|
|
|
|
# ── Rust: ripgrep (CLI, 200+ tests) ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/BurntSushi/ripgrep.git /tmp/real-ripgrep 2>/dev/null'
|
|
- su - ubuntu -c 'export PATH=$HOME/.cargo/bin:$PATH && cd /tmp/real-ripgrep && cargo build 2>/dev/null'
|
|
|
|
# ── Rust data: Polars (DataFrame engine, 1000+ tests) ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/pola-rs/polars.git /tmp/real-polars 2>/dev/null'
|
|
|
|
# ── Python backend: Flask (400+ tests) ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/pallets/flask.git /tmp/real-flask 2>/dev/null'
|
|
- su - ubuntu -c 'pip install --user --break-system-packages -e "/tmp/real-flask[dev]" 2>/dev/null'
|
|
|
|
# ── Python data engineering: dbt-core ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/dbt-labs/dbt-core.git /tmp/real-dbt 2>/dev/null'
|
|
|
|
# ── Python ML/Data Science: scikit-learn ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/scikit-learn/scikit-learn.git /tmp/real-sklearn 2>/dev/null'
|
|
|
|
# ── Go backend: Hugo (static site, 1800+ tests) ──
|
|
- |
|
|
su - ubuntu -c '
|
|
export PATH=$PATH:/usr/local/go/bin
|
|
git clone --depth 1 https://github.com/gohugoio/hugo.git /tmp/real-hugo 2>/dev/null
|
|
cd /tmp/real-hugo && go mod download 2>/dev/null
|
|
'
|
|
|
|
# ── JS/TS build tools: Vite (500+ tests) ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/vitejs/vite.git /tmp/real-vite 2>/dev/null'
|
|
- su - ubuntu -c 'cd /tmp/real-vite && pnpm install --no-frozen-lockfile 2>/dev/null'
|
|
|
|
# ── Vue frontend: vuejs/core ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/vuejs/core.git /tmp/real-vue 2>/dev/null'
|
|
- su - ubuntu -c 'cd /tmp/real-vue && pnpm install --no-frozen-lockfile 2>/dev/null || true'
|
|
|
|
# ── React/Next: T3 stack (fullstack TS) ──
|
|
- |
|
|
su - ubuntu -c '
|
|
mkdir -p /tmp/real-t3 && cd /tmp/real-t3
|
|
pnpm create t3-app@latest my-app --noGit --CI --tailwind --trpc --nextAuth --prisma --appRouter 2>/dev/null || true
|
|
'
|
|
|
|
# ── C systems: Redis ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/redis/redis.git /tmp/real-redis 2>/dev/null'
|
|
- su - ubuntu -c 'cd /tmp/real-redis && make -j$(nproc) 2>/dev/null'
|
|
|
|
# ── C#/.NET: Newtonsoft.Json ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/JamesNK/Newtonsoft.Json.git /tmp/real-newtonsoft 2>/dev/null'
|
|
|
|
# ── Java Spring: spring-petclinic ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/spring-projects/spring-petclinic.git /tmp/real-petclinic 2>/dev/null'
|
|
|
|
# ── Ruby: Sinatra (web framework) ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/sinatra/sinatra.git /tmp/real-sinatra 2>/dev/null'
|
|
- su - ubuntu -c 'cd /tmp/real-sinatra && bundle install 2>/dev/null || true'
|
|
|
|
# ── PHP: Laravel framework ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/laravel/framework.git /tmp/real-laravel 2>/dev/null'
|
|
- su - ubuntu -c 'cd /tmp/real-laravel && composer install --no-interaction 2>/dev/null || true'
|
|
|
|
# ── Kotlin: Ktor (web framework) ──
|
|
- su - ubuntu -c 'git clone --depth 1 https://github.com/ktorio/ktor.git /tmp/real-ktor 2>/dev/null'
|
|
|
|
# Signal completion
|
|
- touch /home/ubuntu/.cloud-init-complete
|
|
- chown ubuntu:ubuntu /home/ubuntu/.cloud-init-complete
|
|
- echo "RTK cloud-init setup complete" | tee /var/log/rtk-setup.log
|
|
|
|
final_message: "RTK test VM ready in $UPTIME seconds"
|