Files
Peter Schilling 78f1812493 Make inline image attachments discoverable, eval-validated
Agents asked to put a visible screenshot on an issue reach for
`issue attach`, which uploads the file but creates a sidebar link
attachment that never renders inline — while the success output
("Attachment created") convinces them the image is visible. The working
path, `issue comment add --attach`, has existed since v2.0.0 but nothing
pointed at it: the skill had no image guidance and the flag was buried in
a reference table.

Three coordinated changes, validated as experiment 2 of the skill eval:

- Skill: a Common Tasks recipe for visible images via
  `issue comment add --attach`, with an explicit warning about
  `issue attach`'s sidebar-only behavior.
- CLI: `issue attach` now says it created a sidebar link attachment,
  and for images prints a copy-pasteable hint suggesting
  `issue comment add --attach` (shell-quoted, --public preserved).
  Help descriptions updated on both commands.
- Eval: new frozen image family (trap-phrased development prompt,
  comment-phrased holdout) plus a sidebar-control case graded on
  positionals, with pre-declared outcome rules, CLI/API control split,
  binary-safe fixture checks, and version-matched shim output.

Result (rules frozen before baseline): image-development went 0/3 to 3/3
— every baseline trial fell into the attach trap and wrongly reported
success; every post-change trial routed straight to the recipe. Image
family 3/6 to 6/6 lands in the pre-declared partial-baseline band, so it
is reported as consistent with improvement (exploratory Fisher p = 0.09)
rather than confirmed. Controls held except one known npx-version-check
grader artifact, adjudicated by an Opus gold-label pass (17/18 agreement
with the deterministic grader).
2026-07-23 10:46:18 -07:00

223 lines
5.9 KiB
TypeScript

/**
* Mock Linear API server for testing
*
* Usage:
* const server = new MockLinearServer([
* {
* queryName: "GetIssueDetails",
* variables: { id: "TEST-123" },
* response: { data: { issue: { title: "Test Issue", ... } } }
* }
* ]);
*/
interface MockResponse {
queryName: string
queryIncludes?: string
variables?: Record<string, unknown>
response: Record<string, unknown>
status?: number
}
export interface UploadRequest {
pathname: string
contentType: string | null
headers: Record<string, string>
body: Uint8Array
}
export class MockLinearServer {
private server?: Deno.HttpServer
private port = 0
private mockResponses: MockResponse[]
/** Signed-URL file uploads received via PUT, in arrival order */
readonly uploadRequests: UploadRequest[] = []
constructor(responses: MockResponse[] = []) {
this.mockResponses = responses
}
async start(): Promise<void> {
this.server = Deno.serve({
hostname: "127.0.0.1",
port: this.port,
onListen: () => {},
}, (request) => {
// Handle CORS preflight
if (request.method === "OPTIONS") {
return new Response(null, {
status: 200,
headers: {
"Access-Control-Allow-Origin": "*",
"Access-Control-Allow-Methods": "POST, OPTIONS",
"Access-Control-Allow-Headers": "Content-Type, Authorization",
},
})
}
// Handle GraphQL requests
if (
request.method === "POST" &&
new URL(request.url).pathname === "/graphql"
) {
return this.handleGraphQL(request)
}
// Handle signed-URL file uploads (the PUT step of the fileUpload flow)
if (
request.method === "PUT" &&
new URL(request.url).pathname.startsWith("/upload")
) {
return this.handleUpload(request)
}
return new Response("Not Found", { status: 404 })
})
if ("port" in this.server.addr) {
this.port = this.server.addr.port
}
// Wait a bit for server to start
await new Promise((resolve) => setTimeout(resolve, 100))
}
async stop(): Promise<void> {
if (this.server) {
await this.server.shutdown()
this.server = undefined
}
}
getEndpoint(): string {
return `http://localhost:${this.port}/graphql`
}
/** URL to hand out as a fileUpload signed uploadUrl in mock responses */
getUploadUrl(): string {
return `http://localhost:${this.port}/upload`
}
private async handleUpload(request: Request): Promise<Response> {
const headers: Record<string, string> = {}
for (const [key, value] of request.headers) {
headers[key] = value
}
this.uploadRequests.push({
pathname: new URL(request.url).pathname,
contentType: request.headers.get("content-type"),
headers,
body: new Uint8Array(await request.arrayBuffer()),
})
return new Response(null, { status: 200 })
}
private async handleGraphQL(request: Request): Promise<Response> {
const headers = {
"Content-Type": "application/json",
"Access-Control-Allow-Origin": "*",
// Use fixed date header for deterministic snapshot tests
"Date": "Mon, 01 Jan 2024 00:00:00 GMT",
}
try {
const body = await request.json()
const { query, variables } = body
// Find matching mock response
const mockResponse = this.findMatchingResponse(query, variables)
if (mockResponse) {
return new Response(
JSON.stringify(mockResponse.response),
{ status: mockResponse.status ?? 200, headers },
)
}
// Default response for unhandled queries
return new Response(
JSON.stringify({
errors: [{
message: "No mock response configured for this query",
extensions: {
code: "NO_MOCK_CONFIGURED",
query: this.extractQueryName(query),
variables,
},
}],
}),
{ status: 200, headers },
)
} catch (_error) {
return new Response(
JSON.stringify({
errors: [{
message: "Invalid JSON in request body",
extensions: { code: "BAD_REQUEST" },
}],
}),
{ status: 400, headers },
)
}
}
private findMatchingResponse(
query: string,
variables: Record<string, unknown> = {},
): MockResponse | undefined {
const queryName = this.extractQueryName(query)
return this.mockResponses.find((mock) => {
// Check if query name matches
if (mock.queryName !== queryName) {
return false
}
if (mock.queryIncludes != null && !query.includes(mock.queryIncludes)) {
return false
}
// If no variables specified in mock, match any variables
if (!mock.variables) {
return true
}
// Check if all mock variables match the request variables (deep comparison)
return Object.entries(mock.variables).every(([key, value]) => {
return this.deepEqual(variables[key], value)
})
})
}
private deepEqual(a: unknown, b: unknown): boolean {
if (a === b) return true
if (a == null || b == null) return a === b
if (typeof a !== typeof b) return false
if (typeof a !== "object") return a === b
const aObj = a as Record<string, unknown>
const bObj = b as Record<string, unknown>
const aKeys = Object.keys(aObj)
const bKeys = Object.keys(bObj)
if (aKeys.length !== bKeys.length) return false
return aKeys.every((key) => this.deepEqual(aObj[key], bObj[key]))
}
private extractQueryName(query: string): string {
// Extract query name from GraphQL query string
// Examples: "query GetIssueDetails" -> "GetIssueDetails"
const match = query.match(/(?:query|mutation)\s+(\w+)/)
return match?.[1] || "UnknownQuery"
}
addResponse(response: MockResponse): void {
this.mockResponses.push(response)
}
clearResponses(): void {
this.mockResponses = []
}
}