I Was Ready to Move Everything to OSS Models on Cloudflare Workers AI—Then the Benchmarks Said No

export CLOUDFLARE_ACCOUNT_ID='your-account-id'
export CLOUDFLARE_API_TOKEN='your-workers-ai-token'
export AI_GATEWAY_NAME='your-gateway-id'

export GOOGLE_APPLICATION_CREDENTIALS="$PWD/service-account.json"
export GOOGLE_VERTEX_LOCATION='global'
export ANTHROPIC_API_KEY='your-anthropic-api-key'
# AI Gatewayを認証ありで運用している場合だけ設定
export CF_AIG_TOKEN='your-ai-gateway-token'

{
  "type": "module",
  "dependencies": {
    "@ai-sdk/anthropic": "3.0.94",
    "@ai-sdk/google-vertex": "4.0.157",
    "@ai-sdk/openai": "3.0.81",
    "ai": "6.0.220",
    "zod": "4.5.4"
  },
  "devDependencies": {
    "tsx": "4.23.0"
  }
}

npm install

import { createAnthropic } from "@ai-sdk/anthropic";
import { createVertex } from "@ai-sdk/google-vertex/edge";
import { createOpenAI } from "@ai-sdk/openai";
import { generateText, Output } from "ai";
import { readFileSync } from "node:fs";
import { z } from "zod";

const { CLOUDFLARE_ACCOUNT_ID, CLOUDFLARE_API_TOKEN, AI_GATEWAY_NAME } =
  process.env;
if (!CLOUDFLARE_ACCOUNT_ID || !CLOUDFLARE_API_TOKEN || !AI_GATEWAY_NAME) {
  throw new Error("Cloudflareの3環境変数が必要です");
}

const PRICING = {
  "gpt-5.2": { input: 1.75, output: 14.0 },
  "@cf/openai/gpt-oss-120b": { input: 0.35, output: 0.75 },
  "@cf/openai/gpt-oss-20b": { input: 0.2, output: 0.3 },
  "@cf/qwen/qwen3-30b-a3b-fp8": { input: 0.051, output: 0.335 },
  "@cf/zai-org/glm-4.7-flash": { input: 0.06, output: 0.4 },
  "@cf/zai-org/glm-5.3-flash": { input: 0.15, output: 0.5 },
  "@cf/google/gemma-4-26b-a4b-it": { input: 0.1, output: 0.3 },
  "@cf/moonshotai/kimi-k2.6": { input: 0.95, output: 4.0 },
  "vertex@gemini-3.5-flash-lite": { input: 0.3, output: 2.5 },
  "claude@claude-haiku-4-5-20251001": { input: 1, output: 5 },
} as const;

type ModelId = keyof typeof PRICING;
const modelId = (process.env.MODEL_ID ||
  "@cf/qwen/qwen3-30b-a3b-fp8") as ModelId;
if (!PRICING[modelId]) throw new Error(`未登録モデル: ${modelId}`);

const schema = z.object({
  productName: z.string(),
  categoryName: z.string(),
  ingredients: z.string(),
  detectedAllergens: z.array(z.string()),
});

const cases = [
  {
    name: "パウンドケーキ",
    recipe:
      "薄力粉100g、バター100g、砂糖80g、卵2個、ベーキングパウダー小さじ1を混ぜて焼く。",
    expected: ["小麦", "卵", "乳"],
  },
  {
    name: "えびグラタン",
    recipe:
      "えび、玉ねぎ、マカロニをバターで炒め、小麦粉と牛乳でホワイトソースを作りチーズをのせて焼く。",
    expected: ["えび", "乳", "小麦"],
  },
  {
    name: "いちごジャム",
    recipe: "いちご500g、砂糖250g、レモン果汁大さじ1を煮詰める。",
    expected: [],
  },
  {
    name: "ポテトサラダ",
    recipe:
      "じゃがいも、にんじん、きゅうり、ハムを茹でて、マヨネーズと塩こしょうで和える。",
    expected: ["卵"],
  },
] as const;

const allergens = [
  "えび",
  "かに",
  "くるみ",
  "小麦",
  "そば",
  "卵",
  "乳",
  "落花生",
] as const;
const variants: Record<(typeof allergens)[number], RegExp> = {
  えび: /えび|エビ|海老/,
  かに: /かに|カニ|蟹/,
  くるみ: /くるみ|クルミ|胡桃/,
  小麦: /小麦|薄力粉|マカロニ|しょうゆ|醤油|パン粉|グルテン/,
  そば: /そば|ソバ|蕎麦/,
  卵: /卵|たまご|玉子|鶏卵|マヨネーズ/,
  乳: /乳|牛乳|バター|チーズ|ミルク|クリーム/,
  落花生: /落花生|ピーナッツ|ピーナツ/,
};

const detect = (text: string): string[] =>
  allergens.filter((a) => variants[a].test(text));
const canonical = (value: string): string | undefined =>
  allergens.find((a) => variants[a].test(value));
const asSet = (values: readonly string[]): Set<string> =>
  new Set(values.map(canonical).filter((v): v is string => Boolean(v)));
const equalSet = (a: Set<string>, b: Set<string>): boolean =>
  a.size === b.size && [...a].every((v) => b.has(v));

const annotation = (a: string): string => (a === "乳" ? "乳成分" : a);
const buildPrompt = (
  recipe: string,
  required: string[],
): string => `あなたは食品表示法の専門家。日本の食品表示法に基づき、以下のレシピから、食品表示ラベルに必要な情報を生成せよ。

# レシピ
${recipe}

# 【決定論検知】レシピに含まれる義務アレルゲン（必ず反映）
- 「${required.map(annotation).join("・")}」がレシピから検出されました。該当原材料に「（〇〇を含む）」注記を付け、detectedAllergens にも必ず含めてください（名称から読み取れない原材料由来＝しょうゆ→小麦・バター→乳成分・マヨネーズ→卵 等の取りこぼしに注意）。

# 生成ルール
1. productName: レシピから適切な商品名を生成。シンプルで分かりやすい名前にする。
2. categoryName: 食品表示法に基づく正式な食品分類名（例：惣菜、菓子、調理パン、弁当、漬物、佃煮、煮豆、調味料、ジャム類など）
3. ingredients::
  - 重量の多い順に記載（レシピの分量から推測）
  - アレルゲンは「（〇〇を含む）」形式で表示（義務8品目のみ。推奨品目は付けない）。一括表記に統一（例: 末尾に「（一部に小麦・乳成分・卵を含む）」をまとめて記載）。個別と一括の併用は不可。
  - 添加物は「/」で区切って記載
  - 一般的な名称を使用（例：「グラニュー糖」→「砂糖」、「薄力粉」→「小麦粉」）
4. detectedAllergens: レシピに含まれる義務8品目（特定原材料）のうち該当するものだけを配列で返す（推奨品目は含めない）

## アレルゲン（検出・注記は義務8品目のみ・推奨は出力しない）: えび、かに、くるみ、小麦、そば、卵、乳、落花生（ピーナッツ）
- detectedAllergens は義務品目名を格納。注記の「乳」は「乳成分」。味噌・豆乳・大豆は小麦ではない（誤検出注意）。

# 返却値: productName(商品名) / categoryName(名称) / ingredients(原材料名・重量順・アレルゲン注記含む) / detectedAllergens(配列)`;

type Usage = { input: number; output: number; neurons: number };
const usageQueue: Usage[] = [];
const workersAI = createOpenAI({
  apiKey: CLOUDFLARE_API_TOKEN,
  baseURL: `https://api.cloudflare.com/client/v4/accounts/${CLOUDFLARE_ACCOUNT_ID}/ai/v1`,
  headers: { "cf-aig-gateway-id": AI_GATEWAY_NAME },
  name: "workers-ai",
  fetch: async (input, init) => {
    const response = await fetch(input, init);
    const body = (await response
      .clone()
      .json()
      .catch(() => null)) as {
      usage?: {
        prompt_tokens?: number;
        completion_tokens?: number;
        neurons?: number;
      };
    } | null;
    if (body?.usage) {
      usageQueue.push({
        input: body.usage.prompt_tokens || 0,
        output: body.usage.completion_tokens || 0,
        neurons: body.usage.neurons || 0,
      });
    }
    return response;
  },
});

const isVertex = modelId === "vertex@gemini-3.5-flash-lite";
const isClaude = modelId === "claude@claude-haiku-4-5-20251001";
const isOpenAI = modelId === "gpt-5.2";
const vertexLocation = process.env.GOOGLE_VERTEX_LOCATION || "global";
const serviceAccountPath = process.env.GOOGLE_APPLICATION_CREDENTIALS;
const serviceAccount =
  isVertex && serviceAccountPath
    ? (JSON.parse(readFileSync(serviceAccountPath, "utf8")) as {
        project_id: string;
        client_email: string;
        private_key: string;
        private_key_id?: string;
      })
    : undefined;
if (isVertex && !serviceAccount) {
  throw new Error("VertexにはGOOGLE_APPLICATION_CREDENTIALSが必要です");
}
const vertex = serviceAccount
  ? createVertex({
      project: serviceAccount.project_id,
      location: vertexLocation,
      googleCredentials: {
        clientEmail: serviceAccount.client_email,
        privateKey: serviceAccount.private_key,
        privateKeyId: serviceAccount.private_key_id,
      },
      headers: process.env.CF_AIG_TOKEN
        ? { "cf-aig-authorization": `Bearer ${process.env.CF_AIG_TOKEN}` }
        : undefined,
      // 実測と同じくCloudflare AI Gatewayのprovider-native endpointを通す
      baseURL:
        `https://gateway.ai.cloudflare.com/v1/${CLOUDFLARE_ACCOUNT_ID}/` +
        `${AI_GATEWAY_NAME}/google-vertex-ai/v1/projects/${serviceAccount.project_id}/` +
        `locations/${vertexLocation}/publishers/google`,
    })
  : undefined;

if (isClaude && !process.env.ANTHROPIC_API_KEY) {
  throw new Error("ClaudeにはANTHROPIC_API_KEYが必要です");
}
const claude = isClaude
  ? createAnthropic({
      apiKey: process.env.ANTHROPIC_API_KEY!,
      headers: process.env.CF_AIG_TOKEN
        ? { "cf-aig-authorization": `Bearer ${process.env.CF_AIG_TOKEN}` }
        : undefined,
      baseURL:
        `https://gateway.ai.cloudflare.com/v1/${CLOUDFLARE_ACCOUNT_ID}/` +
        `${AI_GATEWAY_NAME}/anthropic`,
    })
  : undefined;

if (isOpenAI && !process.env.OPENAI_API_KEY) {
  throw new Error("gpt-5.2にはOPENAI_API_KEYが必要です");
}
const openai = isOpenAI
  ? createOpenAI({
      apiKey: process.env.OPENAI_API_KEY!,
      baseURL:
        `https://gateway.ai.cloudflare.com/v1/${CLOUDFLARE_ACCOUNT_ID}/` +
        `${AI_GATEWAY_NAME}/openai`,
    })
  : undefined;

let totalInput = 0;
let totalOutput = 0;
let totalNeurons = 0;
const started = performance.now();

for (const testCase of cases.slice(
  0,
  Number(process.env.CASE_LIMIT || cases.length),
)) {
  const required = detect(testCase.recipe);
  const caseStarted = performance.now();
  try {
    const result = await generateText({
      model: isVertex
        ? vertex!("gemini-3.5-flash-lite")
        : isClaude
          ? claude!("claude-haiku-4-5-20251001")
          : isOpenAI
            ? openai!.chat("gpt-5.2")
            : workersAI.chat(modelId),
      prompt: buildPrompt(testCase.recipe, required),
      output: Output.object({ schema }),
      temperature: 0,
      maxOutputTokens: 4096,
      maxRetries: 1,
      providerOptions: isVertex
        ? {
            google: {
              thinkingConfig: { thinkingLevel: "minimal" },
            },
          }
        : isOpenAI
          ? { openai: { reasoningEffort: "none" } }
          : undefined,
    });
    const parsed = result.output;
    // 本番と同じ考え方の決定論ガード:
    // 生成ingredientsまたは元レシピに根拠がある義務品目だけ残し、
    // ingredientsから確実に読める品目はモデル出力に和集合する。
    const supported = new Set([...detect(parsed.ingredients), ...required]);
    const guarded = parsed.detectedAllergens
      .map(canonical)
      .filter((a): a is string => Boolean(a) && supported.has(a));
    for (const a of detect(parsed.ingredients)) guarded.push(a);
    const actual = new Set(guarded);
    const expected = asSet(testCase.expected);
    const usage = usageQueue.shift() || {
      input: result.usage.inputTokens || 0,
      output: result.usage.outputTokens || 0,
      neurons: 0,
    };
    totalInput += usage.input;
    totalOutput += usage.output;
    totalNeurons += usage.neurons;
    console.log(
      JSON.stringify({
        case: testCase.name,
        ok: equalSet(actual, expected) && parsed.categoryName.length > 0,
        expected: [...expected],
        actual: [...actual],
        durationMs: Math.round(performance.now() - caseStarted),
        usage,
      }),
    );
  } catch (error) {
    console.error(
      JSON.stringify({
        case: testCase.name,
        ok: false,
        durationMs: Math.round(performance.now() - caseStarted),
        error: error instanceof Error ? error.message : String(error),
      }),
    );
    process.exitCode = 1;
    break;
  }
}

const price = PRICING[modelId];
console.log(
  JSON.stringify({
    model: modelId,
    durationMs: Math.round(performance.now() - started),
    inputTokens: totalInput,
    outputTokens: totalOutput,
    neurons: totalNeurons,
    estimatedUsd:
      (totalInput * price.input + totalOutput * price.output) / 1_000_000,
  }),
);

for MODEL_ID in \
  '@cf/openai/gpt-oss-120b' \
  '@cf/openai/gpt-oss-20b' \
  '@cf/qwen/qwen3-30b-a3b-fp8' \
  '@cf/zai-org/glm-4.7-flash' \
  '@cf/zai-org/glm-5.3-flash' \
  '@cf/google/gemma-4-26b-a4b-it' \
  '@cf/moonshotai/kimi-k2.6' \
  'vertex@gemini-3.5-flash-lite' \
  'claude@claude-haiku-4-5-20251001'
do
  MODEL_ID="$MODEL_ID" npx tsx benchmark.mts
done

for RUN in $(seq 1 5); do
  MODEL_ID='@cf/qwen/qwen3-30b-a3b-fp8' \
    npx tsx benchmark.mts 2>&1 | tee -a qwen-results.jsonl
done

import { createOpenAI } from "@ai-sdk/openai";

const workersAI = createOpenAI({
  apiKey: process.env.CLOUDFLARE_API_TOKEN,
  baseURL:
    `https://api.cloudflare.com/client/v4/accounts/` +
    `${process.env.CLOUDFLARE_ACCOUNT_ID}/ai/v1`,
  headers: {
    "cf-aig-gateway-id": process.env.AI_GATEWAY_NAME!,
  },
  name: "workers-ai",
});

const model = workersAI.chat("@cf/qwen/qwen3-30b-a3b-fp8");

const result = await generateText({
  model,
  prompt,
  output: Output.object({ schema }),
  temperature: 0,
  maxOutputTokens: 4096,
  maxRetries: 1,
});

export OPENAI_API_KEY='your-openai-api-key'
CASE_LIMIT=1 MODEL_ID='gpt-5.2' npx tsx benchmark.mts

{
  "model": "@cf/qwen/qwen3-30b-a3b-fp8",
  "cases": 4,
  "durationMs": 89377,
  "inputTokens": 2434,
  "outputTokens": 8325,
  "neurons": 264.9587001800537,
  "estimatedUsd": 0.002913009
}

{
  "model": "@cf/openai/gpt-oss-120b",
  "cases": 4,
  "durationMs": 60675,
  "inputTokens": 2801,
  "outputTokens": 4052,
  "neurons": 365.39485931396484,
  "estimatedUsd": 0.00401935
}

{"model":"@cf/openai/gpt-oss-120b","cases":1,"durationMs":17041,"inputTokens":735,"outputTokens":1054,"neurons":95.24984741210938,"estimatedUsd":0.00104775}
{"model":"gpt-5.2","cases":1,"durationMs":4643,"inputTokens":727,"outputTokens":80,"estimatedUsd":0.00239225}

@cf/openai/gpt-oss-20b
8294ms: No object generated: response did not match schema.
実際のcontent: -1.1

@cf/zai-org/glm-4.7-flash
148888ms: No output generated.

@cf/zai-org/glm-5.3-flash
76136ms: No output generated.

@cf/google/gemma-4-26b-a4b-it
83561ms: ポテトサラダ expected=["卵"] actual=["小麦","卵"]

@cf/moonshotai/kimi-k2.6
118519ms: No output generated.

{"durationMs":7453,"inputTokens":2538,"outputTokens":331,"estimatedUsd":0.0015889,"caseDurations":[3009,1825,1163,1454]}
{"durationMs":6933,"inputTokens":2538,"outputTokens":342,"estimatedUsd":0.0016164,"caseDurations":[2478,1529,1298,1627]}
{"durationMs":7202,"inputTokens":2538,"outputTokens":339,"estimatedUsd":0.0016089,"caseDurations":[2172,1796,1408,1824]}
{"durationMs":6293,"inputTokens":2538,"outputTokens":364,"estimatedUsd":0.0016714,"caseDurations":[1883,1590,1201,1617]}
{"durationMs":6598,"inputTokens":2538,"outputTokens":340,"estimatedUsd":0.0016114,"caseDurations":[2021,1381,1455,1739]}

{"durationMs":6760,"inputTokens":4181,"outputTokens":337,"estimatedUsd":0.005866,"caseDurations":[2388,1811,1246,1314]}
{"durationMs":5957,"inputTokens":4181,"outputTokens":350,"estimatedUsd":0.005931,"caseDurations":[1757,1766,1033,1399]}
{"durationMs":5883,"inputTokens":4181,"outputTokens":339,"estimatedUsd":0.005876,"caseDurations":[1714,1671,1147,1349]}
{"durationMs":6536,"inputTokens":4181,"outputTokens":336,"estimatedUsd":0.005861,"caseDurations":[1845,2027,1312,1351]}
{"durationMs":6211,"inputTokens":4181,"outputTokens":347,"estimatedUsd":0.005916,"caseDurations":[1692,1839,1259,1420]}