Files
Jordi Enric d6ca0e5900 feat(studio): migrate API reports to OTEL (#50638)
## Problem

The API Gateway and Data API observability reports still send legacy
BigQuery SQL to the logs.all endpoint. The Data API shared-report hook
also hardcodes logs.all, so the otelReports flag cannot move that report
to ClickHouse.

## Fix

- Add the ClickHouse requests-by-country query needed by API Gateway.
- Select API Gateway SQL and endpoint atomically from otelReports.
- Route the Data API PostgREST report through the existing tested OTEL
API query builders and logs.all.otel.
- Wait for ConfigCat before the Data API sends a request, avoiding an
initial legacy request while the flag loads.
- Keep logs.all behavior when the flag is disabled and leave other
shared reports unchanged.

## How to test

1. Enable otelReports and open API Gateway, then confirm its report
requests use logs.all.otel.
2. Open Data API and confirm all report requests use logs.all.otel with
a request.path filter for /rest.
3. Change the date range, add a filter, and refresh each report.
4. Disable otelReports and confirm both reports use logs.all.

All OTEL query shapes were tested individually against logs.all.otel.
The focused query suite and lint pass locally.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

- **New Features**
  - API reports can use OpenTelemetry data when enabled.
- Added country-level request reporting, excluding requests without
country information.
- Added OpenTelemetry-backed PostgREST reports and Storage cache
hit/miss metrics.

- **Improvements**
  - Reports wait for required configuration before loading data.
  - Refreshing reports consistently refetches active metrics.
  - Improved error handling for analytics query failures.
- Improved report accuracy with numeric time buckets and more precise
attribute filtering.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->
2026-09-21 18:13:22 +02:00

133 lines
5.4 KiB
TypeScript

import { describe, expect, it } from 'vitest'
import { generateOtelWhereSafe, PRESET_CONFIG } from './Reports.constants'
import { Presets, type ReportFilterItem } from './Reports.types'
function getOtelSql(preset: Presets, queryName: string, filters: ReportFilterItem[] = []): string {
const query = PRESET_CONFIG[preset].queries[queryName]
if (query.queryType !== 'logs' || query.safeSqlOtel === undefined) {
throw new Error(`Missing OTEL logs query for ${preset}.${queryName}`)
}
return query.safeSqlOtel(filters)
}
describe('generateOtelWhereSafe', () => {
it.each([
['matches', 'api.*', "match(log_attributes['request.path'], 'api.*')"],
['is', 'GET', "log_attributes['request.path'] = 'GET'"],
['!=', 'GET', "log_attributes['request.path'] != 'GET'"],
['>=', '400', "toFloat64OrNull(log_attributes['request.path']) >= 400"],
['<=', '400', "toFloat64OrNull(log_attributes['request.path']) <= 400"],
['>', '400', "toFloat64OrNull(log_attributes['request.path']) > 400"],
['<', '400', "toFloat64OrNull(log_attributes['request.path']) < 400"],
] as const)('builds a safe %s predicate', (compare, value, expected) => {
expect(generateOtelWhereSafe([{ key: 'request.path', value, compare }])).toBe(
`WHERE ${expected}`
)
})
it('preserves dotted keys and escapes key and value input', () => {
expect(
generateOtelWhereSafe([
{
key: "request.headers.user_agent'] OR 1=1 --",
value: "agent' OR 1=1 --",
compare: 'is',
},
])
).toBe("WHERE log_attributes['request.headers.user_agent''] OR 1=1 --'] = 'agent'' OR 1=1 --'")
})
it('omits invalid numeric comparisons without dropping valid filters', () => {
expect(
generateOtelWhereSafe(
[
{ key: 'response.status_code', value: 'not-a-number', compare: '>=' },
{ key: 'request.method', value: 'GET', compare: 'is' },
],
false
)
).toBe("AND log_attributes['request.method'] = 'GET'")
})
})
describe('Report OTEL queries', () => {
it.each([
['totalRequests', 50000],
['topRoutes', 10],
['errorCounts', 50000],
['topErrorRoutes', 10],
['responseSpeed', 50000],
['topSlowRoutes', 10],
['networkTraffic', 50000],
['requestsByCountry', 250],
])('uses edge_logs and a fixed limit for API query %s', (queryName, limit) => {
const sql = getOtelSql(Presets.API, queryName)
expect(sql).toContain('from logs')
expect(sql).toContain("source = 'edge_logs'")
expect(sql).toContain(`limit ${limit}`)
expect(sql).not.toContain('from edge_logs')
expect(sql).not.toContain('cross join unnest')
})
it.each(['totalRequests', 'errorCounts'])(
'emits numeric counts and explicit unix-microsecond buckets for %s',
(queryName) => {
const sql = getOtelSql(Presets.API, queryName)
const bucket = 'toUnixTimestamp(toStartOfHour(logs.timestamp)) * 1000000'
expect(sql).toContain(`${bucket} as timestamp`)
expect(sql).toContain('toFloat64(count()) as count')
expect(sql).toContain(`group by ${bucket}`)
expect(sql).toContain(`order by ${bucket} asc`)
}
)
it('uses exact route attributes and numeric response values', () => {
const routesSql = getOtelSql(Presets.API, 'topRoutes')
const speedSql = getOtelSql(Presets.API, 'responseSpeed')
const trafficSql = getOtelSql(Presets.API, 'networkTraffic')
const countrySql = getOtelSql(Presets.API, 'requestsByCountry')
expect(routesSql).toContain("log_attributes['request.path'] as path")
expect(routesSql).toContain("log_attributes['request.method'] as method")
expect(routesSql).toContain("log_attributes['request.search'] as search")
expect(routesSql).toContain(
"toInt32OrZero(log_attributes['response.status_code']) as status_code"
)
expect(routesSql).toContain('toFloat64(count()) as count')
expect(speedSql).toContain(
"avg(toFloat64OrNull(log_attributes['response.origin_time'])) as avg"
)
expect(trafficSql).toContain(
"sum(toFloat64OrZero(log_attributes['request.headers.content_length'])) / 1000000 as ingress_mb"
)
expect(trafficSql).toContain(
"sum(toFloat64OrZero(log_attributes['response.headers.content_length'])) / 1000000 as egress_mb"
)
expect(countrySql).toContain("log_attributes['request.cf.country'] as country")
expect(countrySql).toContain("notEmpty(log_attributes['request.cf.country'])")
})
it('builds numeric Storage cache results from exact OTEL attributes', () => {
const hitRateSql = getOtelSql(Presets.STORAGE, 'cacheHitRate')
const missesSql = getOtelSql(Presets.STORAGE, 'topCacheMisses')
expect(hitRateSql).toContain("where source = 'edge_logs'")
expect(hitRateSql).toContain("startsWith(log_attributes['request.path'], '/storage/v1/object')")
expect(hitRateSql).toContain(
"toFloat64(countIf(log_attributes['response.headers.cf_cache_status'] in ('HIT', 'STALE', 'REVALIDATED', 'UPDATING'))) as hit_count"
)
expect(hitRateSql).toContain(
"toFloat64(countIf(log_attributes['response.headers.cf_cache_status'] in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC'))) as miss_count"
)
expect(hitRateSql).toContain('limit 50000')
expect(missesSql).toContain('toFloat64(count()) as count')
expect(missesSql).toContain(
"log_attributes['response.headers.cf_cache_status'] in ('MISS', 'NONE/UNKNOWN', 'EXPIRED', 'BYPASS', 'DYNAMIC')"
)
expect(missesSql).toContain('limit 12')
})
})