name: string

baseUrl: string

model: string

apiKeyEnvVar: string

turn: number

elapsedMs: number

ttftMs?: number

inputTokens: number

cachedTokens: number

outputTokens: number

outputTokensPerSec: number

cost: number

responseContent: string

error?: string

provider: ProviderConfig

turns: TurnResult[]

totalElapsedMs: number

wallClockMs: number

const inputTokens = typeof usage.prompt_tokens === 'number' ? usage.prompt_tokens : 0

const outputTokens = typeof usage.completion_tokens === 'number' ? usage.completion_tokens : 0

const promptDetails = usage.prompt_tokens_details as Record<string, unknown> | undefined

const cachedTokens = typeof promptDetails?.cached_tokens === 'number' ? promptDetails.cached_tokens : 0

const nonCachedInput = Math.max(0, inputTokens - cachedTokens)

return nonCachedInput * INPUT_COST_PER_TOKEN +

cachedTokens * CACHED_INPUT_COST_PER_TOKEN +

outputTokens * OUTPUT_COST_PER_TOKEN

const inputTokens = typeof usage.prompt_tokens === 'number' ? usage.prompt_tokens : 0

const outputTokens = typeof usage.completion_tokens === 'number' ? usage.completion_tokens : 0

const promptDetails = usage.prompt_tokens_details as Record<string, unknown> | undefined

const cachedTokens = typeof promptDetails?.cached_tokens === 'number' ? promptDetails.cached_tokens : 0

return { inputTokens, cachedTokens, outputTokens }

config: ProviderConfig,

apiKey: string,

messages: ConversationMessage[],

turnIndex: number,

): Promise<TurnResult> {

const startTime = Date.now()

let ttftMs: number | undefined

const response = await fetch(`${config.baseUrl}/chat/completions`, {

method: 'POST',

headers: {

Authorization: `Bearer ${apiKey}`,

'Content-Type': 'application/json',

},

body: JSON.stringify({

model: config.model,

messages,

max_tokens: MAX_TOKENS,

stream: true,

stream_options: { include_usage: true },

}),

})

if (!response.ok) {

const errorText = await response.text()

return {

turn: turnIndex + 1,

elapsedMs: Date.now() - startTime,

inputTokens: 0,

cachedTokens: 0,

outputTokens: 0,

outputTokensPerSec: 0,

cost: 0,

responseContent: '',

error: `${response.status}: ${errorText.slice(0, 200)}`,

}

}

const reader = response.body?.getReader()

if (!reader) {

return {

turn: turnIndex + 1,

elapsedMs: Date.now() - startTime,

inputTokens: 0,

cachedTokens: 0,

outputTokens: 0,

outputTokensPerSec: 0,

cost: 0,

responseContent: '',

error: 'No response body reader',

}

}

const decoder = new TextDecoder()

let streamContent = ''

let streamUsage: Record<string, unknown> | null = null

let firstContentChunkTime: number | undefined

let done = false

while (!done) {

const result = await reader.read()

done = result.done

if (done) break

const text = decoder.decode(result.value, { stream: true })

const lines = text.split('\n').filter((l) => l.startsWith('data: '))

for (const line of lines) {

const raw = line.slice('data: '.length)

if (raw === '[DONE]') continue

try {

const chunk = JSON.parse(raw)

const delta = chunk.choices?.[0]?.delta

if (delta?.content) {

if (firstContentChunkTime === undefined) {

firstContentChunkTime = Date.now()

ttftMs = firstContentChunkTime - startTime

}

streamContent += delta.content

}

if (chunk.usage) streamUsage = chunk.usage

} catch {

// skip non-JSON lines

}

}

}

const elapsedMs = Date.now() - startTime

const { inputTokens, cachedTokens, outputTokens } = streamUsage

? extractUsageFields(streamUsage)

: { inputTokens: 0, cachedTokens: 0, outputTokens: 0 }

const generationTimeMs = firstContentChunkTime !== undefined

? Date.now() - firstContentChunkTime

: elapsedMs

const outputTokensPerSec = generationTimeMs > 0

? (outputTokens / (generationTimeMs / 1000))

: 0

const cost = streamUsage ? computeCost(streamUsage) : 0

return {

turn: turnIndex + 1,

elapsedMs,

ttftMs,

inputTokens,

cachedTokens,

outputTokens,

outputTokensPerSec,

cost,

responseContent: streamContent,

}

const conversationHistory: ConversationMessage[] = [

{ role: 'system', content: SYSTEM_PROMPT },

]

const turns: TurnResult[] = []

const wallStart = Date.now()

let totalElapsedMs = 0

for (let i = 0; i < NUM_TURNS; i++) {

conversationHistory.push({ role: 'user', content: TURN_PROMPTS[i] })

const result = await runTurn(config, apiKey, [...conversationHistory], i)

turns.push(result)

totalElapsedMs += result.elapsedMs

if (result.responseContent) {

conversationHistory.push({ role: 'assistant', content: result.responseContent })

}

}

return {

provider: config,

turns,

totalElapsedMs,

wallClockMs: Date.now() - wallStart,

}

const p = r.provider

console.log()

console.log(`${'═'.repeat(100)}`)

console.log(` ${p.name} | Model: ${p.model} | Base URL: ${p.baseUrl}`)

console.log(`${'═'.repeat(100)}`)

console.log()

console.log(` ${'Turn'.padEnd(25)} | ${pad('Time', 8)} | ${pad('TTFT', 7)} | ${pad('Input', 6)} | ${pad('Cached', 6)} | ${pad('Cache%', 7)} | ${pad('Output', 6)} | ${pad('tok/s', 6)} | ${pad('e2e t/s', 7)} | Cost`)

console.log(' ' + '─'.repeat(105))

let totalCost = 0

let totalInput = 0

let totalCached = 0

let totalOutput = 0

for (const t of r.turns) {

const label = `Turn ${t.turn}/${NUM_TURNS}${t.turn === 1 ? ' (cold)' : ''}`

const time = `${(t.elapsedMs / 1000).toFixed(2)}s`

const ttft = t.ttftMs !== undefined ? `${(t.ttftMs / 1000).toFixed(2)}s` : 'n/a'

const cacheRate = pct(t.cachedTokens, t.inputTokens)

const tokSec = t.outputTokensPerSec.toFixed(1)

const e2eTokSec = t.elapsedMs > 0 ? (t.outputTokens / (t.elapsedMs / 1000)).toFixed(1) : 'n/a'

const costStr = t.error ? 'err' : `$${t.cost.toFixed(6)}`

totalCost += t.cost

totalInput += t.inputTokens

totalCached += t.cachedTokens

totalOutput += t.outputTokens

if (t.error) {

console.log(` ${label.padEnd(25)} | ${pad(time, 8)} | ${pad(ttft, 7)} | ❌ ${t.error.slice(0, 60)}`)

} else {

console.log(` ${label.padEnd(25)} | ${pad(time, 8)} | ${pad(ttft, 7)} | ${pad(String(t.inputTokens), 6)} | ${pad(String(t.cachedTokens), 6)} | ${pad(cacheRate, 7)} | ${pad(String(t.outputTokens), 6)} | ${pad(tokSec, 6)} | ${pad(e2eTokSec, 7)} | ${costStr}`)

}

}

console.log(' ' + '─'.repeat(105))

const totalTimeStr = `${(r.totalElapsedMs / 1000).toFixed(2)}s`

const overallCacheRate = pct(totalCached, totalInput)

const overallTokSec = r.totalElapsedMs > 0 ? (totalOutput / (r.totalElapsedMs / 1000)).toFixed(1) : 'n/a'

console.log(` ${'TOTAL'.padEnd(25)} | ${pad(totalTimeStr, 8)} | | ${pad(String(totalInput), 6)} | ${pad(String(totalCached), 6)} | ${pad(overallCacheRate, 7)} | ${pad(String(totalOutput), 6)} | | ${pad(overallTokSec, 7)} | $${totalCost.toFixed(6)}`)

console.log()

const costWithoutCaching = totalInput * INPUT_COST_PER_TOKEN + totalOutput * OUTPUT_COST_PER_TOKEN

const savings = costWithoutCaching - totalCost

const savingsPct = costWithoutCaching > 0 ? ((savings / costWithoutCaching) * 100).toFixed(1) : '0.0'

console.log(` Cost savings from caching: $${savings.toFixed(6)} (${savingsPct}%)`)

const ttfts = r.turns.filter((t) => t.ttftMs !== undefined).map((t) => t.ttftMs!)

if (ttfts.length > 0) {

const avgTtft = ttfts.reduce((a, b) => a + b, 0) / ttfts.length

console.log(` TTFT — avg: ${(avgTtft / 1000).toFixed(2)}s, min: ${(Math.min(...ttfts) / 1000).toFixed(2)}s, max: ${(Math.max(...ttfts) / 1000).toFixed(2)}s`)

}

name: string

totalTime: number

wallClock: number

cacheHitRate: number

costSavings: number

totalCost: number

costWithoutCaching: number

avgTtft: number | null

avgWarmTtft: number | null

e2eTokSec: number

totalInput: number

totalCached: number

totalOutput: number

cacheMissTurns: number

errorTurns: number

let totalInput = 0

let totalCached = 0

let totalOutput = 0

let totalCost = 0

let cacheMissTurns = 0

let errorTurns = 0

for (const t of r.turns) {

totalInput += t.inputTokens

totalCached += t.cachedTokens

totalOutput += t.outputTokens

totalCost += t.cost

if (t.error) errorTurns++

else if (t.cachedTokens === 0) cacheMissTurns++

}

const cacheHitRate = totalInput > 0 ? (totalCached / totalInput) * 100 : 0

const costWithoutCaching = totalInput * INPUT_COST_PER_TOKEN + totalOutput * OUTPUT_COST_PER_TOKEN

const savings = costWithoutCaching > 0 ? ((costWithoutCaching - totalCost) / costWithoutCaching) * 100 : 0

const e2eTokSec = r.totalElapsedMs > 0 ? totalOutput / (r.totalElapsedMs / 1000) : 0

const ttfts = r.turns.filter((t) => t.ttftMs !== undefined).map((t) => t.ttftMs!)

const avgTtft = ttfts.length > 0 ? ttfts.reduce((a, b) => a + b, 0) / ttfts.length : null

const warmTtfts = r.turns.slice(1).filter((t) => t.ttftMs !== undefined).map((t) => t.ttftMs!)

const avgWarmTtft = warmTtfts.length > 0 ? warmTtfts.reduce((a, b) => a + b, 0) / warmTtfts.length : null

return {

name: r.provider.name,

totalTime: r.totalElapsedMs,

wallClock: r.wallClockMs,

cacheHitRate,

costSavings: savings,

totalCost,

costWithoutCaching,

avgTtft,

avgWarmTtft,

e2eTokSec,

totalInput,

totalCached,

totalOutput,

cacheMissTurns,

errorTurns,

}

let best: ProviderSummary | null = null

for (const s of summaries) {

const val = s[key]

if (val === null || val === undefined) continue

if (!best) { best = s; continue }

const bestVal = best[key] as number

if (higherIsBetter ? (val as number) > bestVal : (val as number) < bestVal) best = s

}

return best ? `🏆 ${best.name}` : 'n/a'

console.log()

console.log('█'.repeat(100))

console.log(' HEAD-TO-HEAD COMPARISON')

console.log('█'.repeat(100))

console.log()

const nameWidth = 14

const colWidth = 16

const header = ` ${'Metric'.padEnd(24)} | ${summaries.map((s) => s.name.padStart(colWidth)).join(' | ')} | Winner`

console.log(header)

console.log(' ' + '─'.repeat(header.length - 3))

const rows: Array<{ label: string; values: string[]; winner: string }> = [

{

label: 'Total time',

values: summaries.map((s) => `${(s.totalTime / 1000).toFixed(2)}s`),

winner: pickWinner(summaries, 'totalTime', false),

},

{

label: 'Wall clock',

values: summaries.map((s) => `${(s.wallClock / 1000).toFixed(2)}s`),

winner: pickWinner(summaries, 'wallClock', false),

},

{

label: 'Cache hit rate',

values: summaries.map((s) => `${s.cacheHitRate.toFixed(1)}%`),

winner: pickWinner(summaries, 'cacheHitRate', true),

},

{

label: 'Cost savings',

values: summaries.map((s) => `${s.costSavings.toFixed(1)}%`),

winner: pickWinner(summaries, 'costSavings', true),

},

{

label: 'Total cost',

values: summaries.map((s) => `$${s.totalCost.toFixed(6)}`),

winner: pickWinner(summaries, 'totalCost', false),

},

{

label: 'Avg TTFT',

values: summaries.map((s) => s.avgTtft !== null ? `${(s.avgTtft / 1000).toFixed(2)}s` : 'n/a'),

winner: (() => {

const withTtft = summaries.filter((s) => s.avgTtft !== null)

if (withTtft.length === 0) return 'n/a'

return `🏆 ${withTtft.reduce((a, b) => a.avgTtft! < b.avgTtft! ? a : b).name}`

})(),

},

{

label: 'Avg warm TTFT',

values: summaries.map((s) => s.avgWarmTtft !== null ? `${(s.avgWarmTtft / 1000).toFixed(2)}s` : 'n/a'),

winner: (() => {

const withTtft = summaries.filter((s) => s.avgWarmTtft !== null)

if (withTtft.length === 0) return 'n/a'

return `🏆 ${withTtft.reduce((a, b) => a.avgWarmTtft! < b.avgWarmTtft! ? a : b).name}`

})(),

},

{

label: 'e2e tok/s',

values: summaries.map((s) => s.e2eTokSec.toFixed(1)),

winner: pickWinner(summaries, 'e2eTokSec', true),

},

{

label: 'Cache miss turns',

values: summaries.map((s) => `${s.cacheMissTurns}/${NUM_TURNS}`),

winner: pickWinner(summaries, 'cacheMissTurns', false),

},

{

label: 'Error turns',

values: summaries.map((s) => `${s.errorTurns}/${NUM_TURNS}`),

winner: pickWinner(summaries, 'errorTurns', false),

},

{

label: 'Total input tokens',

values: summaries.map((s) => String(s.totalInput)),

winner: '',

},

{

label: 'Total output tokens',

values: summaries.map((s) => String(s.totalOutput)),

winner: '',

},

]

for (const row of rows) {

const vals = row.values.map((v) => v.padStart(colWidth)).join(' | ')

console.log(` ${row.label.padEnd(24)} | ${vals} | ${row.winner}`)

}

console.log()

console.log('🏁 Combined Provider Benchmark — 10-Turn Conversation Caching Test')

console.log('='.repeat(100))

console.log(`Turns: ${NUM_TURNS}`)

console.log(`Max tokens: ${MAX_TOKENS} per turn`)

console.log(`Pricing: $0.30/M input, $0.03/M cached, $1.20/M output`)

console.log(`Seed: ${SEED_STRING}`)

console.log(`Providers: ${PROVIDERS.map((p) => p.name).join(', ')}`)

console.log('='.repeat(100))

console.log()

// Validate API keys

const validProviders: Array<{ config: ProviderConfig; apiKey: string }> = []

const skippedProviders: string[] = []

for (const config of PROVIDERS) {

const apiKey = process.env[config.apiKeyEnvVar]

if (!apiKey) {

console.log(`⚠️ Skipping ${config.name}: ${config.apiKeyEnvVar} not set`)

skippedProviders.push(config.name)

} else {

validProviders.push({ config, apiKey })

console.log(`✅ ${config.name}: API key found`)

}

}

if (validProviders.length === 0) {

console.error('\n❌ No API keys found. Set at least one of: FIREWORKS_API_KEY, SILICON_FLOW_API_KEY, CANOPYWAVE_API_KEY')

process.exit(1)

}

console.log()

console.log(`🚀 Running ${validProviders.length} provider(s) in parallel...`)

console.log()

const benchmarkStart = Date.now()

// Run all providers in parallel

const results = await Promise.all(

validProviders.map(({ config, apiKey }) => runProviderBenchmark(config, apiKey)),

)

const benchmarkElapsed = Date.now() - benchmarkStart

// Print individual provider summaries

for (const result of results) {

printProviderSummary(result)

}

// Print comparison table

if (results.length > 1) {

const summaries = results.map(summarize)

printComparisonTable(summaries)

}

// Final summary

console.log('━'.repeat(100))

console.log(` Benchmark complete in ${(benchmarkElapsed / 1000).toFixed(1)}s wall clock (all providers ran in parallel)`)

if (skippedProviders.length > 0) {

console.log(` Skipped: ${skippedProviders.join(', ')}`)

}

console.log('━'.repeat(100))

console.log()

console.log('Done!')