{"aiq":null,"arms":"fresh defaults for every historical ID; no feedback, training or family matching","baselines":"all eleven fixed models over the identical full set","configuration":{"frozen_hour":10,"objective":"balanced","plan_id":"pro-code"},"cost":"sum of selected cached USD values; no incurred spend or TT conversion","created_at":"2026-09-25T11:32:26Z","dataset":{"bytes":99567659,"file":"routerbench_0shot.pkl","revision":"784021482c3f320c6619ed4b3bb3b41a21424fcb","rows":36497,"sampling":"all_rows_in_original_order_no_exclusions","sha256":"ba4f77f19517610a707c374e99322d7750c30fc4ae7ff5527888595a1e65d36d"},"expected_weights_tt":{"WizardLM/WizardLM-13B-V1.2":1,"claude-instant-v1":1,"claude-v1":1,"claude-v2":1,"gpt-3.5-turbo-1106":1,"gpt-4-1106-preview":1,"meta/code-llama-instruct-34b-chat":1,"meta/llama-2-70b-chat":1,"mistralai/mistral-7b-chat":1,"mistralai/mixtral-8x7b-chat":1,"zero-one-ai/Yi-34B-Chat":1},"experiment":"titan-u-routerbench-0shot-historical-v1","missing_data":"fail the complete evaluation on any missing/nonfinite/out-of-range quality or cost; never drop or impute","model_ids":["gpt-3.5-turbo-1106","claude-instant-v1","claude-v1","claude-v2","gpt-4-1106-preview","meta/llama-2-70b-chat","mistralai/mixtral-8x7b-chat","zero-one-ai/Yi-34B-Chat","WizardLM/WizardLM-13B-V1.2","meta/code-llama-instruct-34b-chat","mistralai/mistral-7b-chat"],"official_submission":"not_submitted","oracle":"retrospective maximum cached quality, then minimum cached cost, then frozen pool order; never router input","phases":["extract prompt-only JSONL","seal every prediction","separate evaluation process"],"price_basis":"upstream static TOKEN_COSTS literals; divisor 10000000 retained; derive_tt_weight unchanged","prices_sha256":"014b1bed27fef27ea8dcf411b3005811861089fb4f80771235ef7989be346417","publication":"protocol, IDs, hashes, model selections and aggregates only; no prompts/responses/corpus redistribution","quality":"arithmetic mean of cached per-case normalized quality [0,1]; not binary accuracy","router_input":"prompt only; no sample_id, eval_name, answers, quality, cached costs or oracle","runtime":{"machine":"arm64","packages":{"numpy":"1.26.4","pandas":"2.2.3","python-dateutil":"2.9.0.post0","pytz":"2025.2","six":"1.17.0","tzdata":"2025.2"},"python":"3.11.15","system":"Darwin"},"schema_version":1,"scope":"historical_reference_no_inference","snapshot_sha256":"dd7ce8026b1a8b31a9edd339a7c98f046e7b85779b6103956fbb68b3297aa3a8","source_sha256":{"scripts/benchmarks/titan_cached/__init__.py":"6d96d481096aa7b68840f5014cecf1e1d6521d5d1c6bf596ef55959c7a582d9d","scripts/benchmarks/titan_cached/frozen.py":"d06c1247d3ec28f2d8a3463b75ecd0352bea080eb32961293e1015f2baa10f75","scripts/benchmarks/titan_routerbench/__init__.py":"eb1c7d2461b215b7572933d159931ba7a7bb47aae300fd8bd012eb65236a8459","scripts/benchmarks/titan_routerbench/corpus.py":"ffe5bd1768fd198f6a4bbebfb08c7b8d7c0ee7b51497f68b7fa5c26b07cae9f1","scripts/benchmarks/titan_routerbench/protocol.py":"73c4629cc6e6dd7949395fe85f8fff414c5ae144a195d10860d3b25dcb80b158","scripts/benchmarks/titan_routerbench/requirements.txt":"423381f393fc3092a9ce20dec6b539d337e2ed1f2fbd627c03943e4004e9579d","scripts/benchmarks/titan_routerbench/run.py":"50d94d14c38c867cb8726b1347fc21ca0cfb896ea376cc9b88cae3e34bd9eec8","server/native_approvals.py":"f35d727d074419d0ef5c21802b9ae82ca746c71364be31f5930813042e4022e3","server/theus_oauth_server.py":"fb318a5f8c4bacfec40e15b0057c49fc01e8e9245924ee6ffc1d19f7ca8e0a13","server/titan_contract.py":"6499a5a8bd102eec530ff5c93fbf5e47419f81bd150a58b4c02867c7b30c9004","server/titan_evidence.py":"997d5deb99cf43ea8d2ef29e7477fe50f8c61cee8c5196aacc9f82457877b742","server/titan_evidence_http.py":"814b3b53b8357c8cc0375d0d83d7c2e07374b9ab83b259f285b37df8f63e4fad","server/titan_gateway_adapter.py":"b13dc2e80f29bb8f15d3e8999df2714926adad0b9fd8b837a83804e34fa97c74"},"timing":"ReferenceTitan local selector latency; no provider latency, warmup or inference"}