{"meta":{"dataset_version":"06b1e59cb172b3ae","generated_at":"2026-11-17T10:15:03.043Z","upstream_fetched_at":"2026-11-17T04:24:14.188Z"},"data":{"provider":{"id":"devin","name":"Devin","doc":"https://docs.devin.ai","env":[],"access":"subscription","access_reviewed":true,"origin":"added","entry_count":17,"evidence_entry_count":17,"providers_dev":{"origin":"added","access":"subscription","access_basis":"Reached with a Devin subscription. Devin serves other labs' models under its own model ids.","access_reviewed":true,"access_source":null,"note":"Only models evaluated on Devin are listed, under the ids Devin uses. The link to a lab's model is the one recorded with each evaluation; where none was recorded, the entry has no canonical model.","evidence":[{"kind":"measured","url":"https://github.com/ngvoicu/groundtruth/blob/main/observed.json","date":"2026-11-12","note":"Cache evaluations taken through a Devin subscription."}],"provenance":[]}},"summary":{"ref":"devin/swe-2","provider":"devin","model":"swe-2","name":"swe-2","type":null,"family":null,"canonical_id":null,"status":null,"release_date":null,"access":"subscription","origin":"added","context":null,"input":null,"max_output":null,"input_window":{"tokens":300499,"basis":"measured","bound":"at_least","scope":"input","source":"providers.dev","date":"2026-11-16","documented_tokens":null,"documented_field":null,"documented_source":null,"documented_date":null,"documented_status":null,"measured_tokens":300499,"measured_bound":"at_least","measured_kind":"accepted","measured_refused":null,"measured_date":"2026-11-16","clients":1,"evaluations":1,"measurements":[{"client":"cliproxyapi","client_label":"Requests sent through CLIProxyAPI","route":"devin","route_label":"Devin subscription","date":"2026-11-16","tokens":300499,"bound":"at_least","kind":"accepted","accepted":300499,"refused":null,"named":null,"read":300499,"source":"https://github.com/ngvoicu/groundtruth/blob/main/results/limits-2026-11-16/swe-2-recall.jsonl"}]},"cache":"unknown","cache_modes":[],"cache_mode_support":{"implicit":"unknown","explicit":"unknown"},"cache_retention":"unknown","cache_lifetime":{"short":{"safe_s":0,"basis":"measured","source":"providers.dev","date":"2026-11-08","documented_s":null,"measured_s":0,"documented_status":null,"clients":3,"evaluations":3,"checks":23,"hits":1,"checks_inside":null,"hits_inside":null,"tested_from_s":300,"first_miss_s":0},"long":{"safe_s":null,"basis":"none","source":null,"date":null,"documented_s":null,"measured_s":null,"documented_status":null,"clients":0,"evaluations":0,"checks":0,"hits":0,"checks_inside":null,"hits_inside":null,"tested_from_s":null,"first_miss_s":null},"lifetimes":[{"retention_profile":"short","basis":"measured","source":"providers.dev","date":"2026-11-08","lifetime_s":0,"mode":null,"documented_as":null,"client":"agent+cliproxyapi","client_label":"A coding agent, through CLIProxyAPI","parameter":null,"value":null,"evaluations":1,"evaluation_ids":["devin-harness-2026-11-08/D12p"],"checks":11,"hits":1,"tested_from_s":300,"first_miss_s":300,"results":[{"kind":"control","age_s":0,"hits":1,"checks":1,"hit_rate_pct":100},{"kind":"timed","age_s":300,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":900,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":1800,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":3600,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":7200,"hits":0,"checks":2,"hit_rate_pct":0}]},{"retention_profile":"short","basis":"measured","source":"providers.dev","date":"2026-11-04","lifetime_s":0,"mode":null,"documented_as":null,"client":"cliproxyapi","client_label":"Requests sent through CLIProxyAPI","parameter":null,"value":null,"evaluations":1,"evaluation_ids":["cpa-8.0.53/D3"],"checks":1,"hits":0,"tested_from_s":null,"first_miss_s":0,"results":[{"kind":"control","age_s":0,"hits":0,"checks":1,"hit_rate_pct":0}]},{"retention_profile":"short","basis":"measured","source":"providers.dev","date":"2026-11-08","lifetime_s":0,"mode":null,"documented_as":null,"client":"devin-cli","client_label":"The Devin CLI","parameter":null,"value":null,"evaluations":1,"evaluation_ids":["devin-harness-2026-11-08/D12c"],"checks":11,"hits":0,"tested_from_s":300,"first_miss_s":0,"results":[{"kind":"control","age_s":0,"hits":0,"checks":1,"hit_rate_pct":0},{"kind":"timed","age_s":300,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":900,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":1800,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":3600,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":7200,"hits":0,"checks":2,"hit_rate_pct":0}]}]},"evaluation_count":3,"price_in":null,"price_out":null,"price_from":null,"pricing":{"coverage":"not_applicable","billing":"not_applicable","reference_ref":null,"rules":null,"rule_order":null,"unparsed":[],"evidence":[]},"request_controls":{"coverage":"unknown","items":[{"id":"cache.retention","support":"unknown","when":null,"bindings":null,"options":null,"default":null,"conditions":[],"evidence":[]},{"id":"service_tier","support":"unknown","when":null,"bindings":null,"options":null,"default":null,"conditions":[],"evidence":[]}]},"measured_input_accepted":300499,"measured_input_refused":null,"measured_limit_count":1,"overridden_fields":[],"verified":"2026-11-16","shard":0},"entry":{"id":"swe-2","providers_dev":{"origin":"added","access":"subscription","provenance":[],"inherited":null,"cache":{"documented":{"support":"unknown","modes":{"implicit":{"support":"unknown","min_tokens":null,"max_breakpoints":null,"targets":null},"explicit":{"support":"unknown","min_tokens":null,"max_breakpoints":null,"targets":null}}},"retention":{"support":"unknown","parameter":null,"accepted_values":null,"default":null,"profiles":[],"conditions":[],"evidence":[]},"lifetime":{"short":{"safe_s":0,"basis":"measured","source":"providers.dev","date":"2026-11-08","documented_s":null,"measured_s":0,"documented_status":null,"clients":3,"evaluations":3,"checks":23,"hits":1,"checks_inside":null,"hits_inside":null,"tested_from_s":300,"first_miss_s":0},"long":{"safe_s":null,"basis":"none","source":null,"date":null,"documented_s":null,"measured_s":null,"documented_status":null,"clients":0,"evaluations":0,"checks":0,"hits":0,"checks_inside":null,"hits_inside":null,"tested_from_s":null,"first_miss_s":null},"lifetimes":[{"retention_profile":"short","basis":"measured","source":"providers.dev","date":"2026-11-08","lifetime_s":0,"mode":null,"documented_as":null,"client":"agent+cliproxyapi","client_label":"A coding agent, through CLIProxyAPI","parameter":null,"value":null,"evaluations":1,"evaluation_ids":["devin-harness-2026-11-08/D12p"],"checks":11,"hits":1,"tested_from_s":300,"first_miss_s":300,"results":[{"kind":"control","age_s":0,"hits":1,"checks":1,"hit_rate_pct":100},{"kind":"timed","age_s":300,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":900,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":1800,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":3600,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":7200,"hits":0,"checks":2,"hit_rate_pct":0}]},{"retention_profile":"short","basis":"measured","source":"providers.dev","date":"2026-11-04","lifetime_s":0,"mode":null,"documented_as":null,"client":"cliproxyapi","client_label":"Requests sent through CLIProxyAPI","parameter":null,"value":null,"evaluations":1,"evaluation_ids":["cpa-8.0.53/D3"],"checks":1,"hits":0,"tested_from_s":null,"first_miss_s":0,"results":[{"kind":"control","age_s":0,"hits":0,"checks":1,"hit_rate_pct":0}]},{"retention_profile":"short","basis":"measured","source":"providers.dev","date":"2026-11-08","lifetime_s":0,"mode":null,"documented_as":null,"client":"devin-cli","client_label":"The Devin CLI","parameter":null,"value":null,"evaluations":1,"evaluation_ids":["devin-harness-2026-11-08/D12c"],"checks":11,"hits":0,"tested_from_s":300,"first_miss_s":0,"results":[{"kind":"control","age_s":0,"hits":0,"checks":1,"hit_rate_pct":0},{"kind":"timed","age_s":300,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":900,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":1800,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":3600,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":7200,"hits":0,"checks":2,"hit_rate_pct":0}]}]},"evaluations":[{"id":"devin-harness-2026-11-08/D12c","route":"devin","route_label":"Devin subscription","client":"devin-cli","client_label":"The Devin CLI","date":"2026-11-08","request_settings":{"reasoning":"low"},"retention_profile":"short","parameter":null,"value":null,"results":[{"kind":"control","age_s":0,"hits":0,"checks":1,"hit_rate_pct":0},{"kind":"timed","age_s":300,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":900,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":1800,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":3600,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":7200,"hits":0,"checks":2,"hit_rate_pct":0}],"late_excluded":0,"note":null,"source":"https://github.com/ngvoicu/groundtruth/blob/main/results/devin-harness-2026-11-08/D12c.jsonl"},{"id":"devin-harness-2026-11-08/D12p","route":"devin","route_label":"Devin subscription","client":"agent+cliproxyapi","client_label":"A coding agent, through CLIProxyAPI","date":"2026-11-08","request_settings":{"reasoning":"low"},"retention_profile":"short","parameter":null,"value":null,"results":[{"kind":"control","age_s":0,"hits":1,"checks":1,"hit_rate_pct":100},{"kind":"timed","age_s":300,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":900,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":1800,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":3600,"hits":0,"checks":2,"hit_rate_pct":0},{"kind":"timed","age_s":7200,"hits":0,"checks":2,"hit_rate_pct":0}],"late_excluded":0,"note":null,"source":"https://github.com/ngvoicu/groundtruth/blob/main/results/devin-harness-2026-11-08/D12p.jsonl"},{"id":"cpa-8.0.53/D3","route":"devin","route_label":"Devin subscription","client":"cliproxyapi","client_label":"Requests sent through CLIProxyAPI","date":"2026-11-04","request_settings":{},"retention_profile":"short","parameter":null,"value":null,"results":[{"kind":"control","age_s":0,"hits":0,"checks":1,"hit_rate_pct":0}],"late_excluded":0,"note":null,"source":"https://github.com/ngvoicu/groundtruth/blob/main/results/cpa-8.0.53/D3-2026-11-04T17-57-26.json"}]},"pricing":{"coverage":"not_applicable","billing":"not_applicable","reference_ref":null,"rules":null,"rule_order":null,"unparsed":[],"evidence":[]},"request_controls":{"coverage":"unknown","items":[{"id":"cache.retention","support":"unknown","when":null,"bindings":null,"options":null,"default":null,"conditions":[],"evidence":[]},{"id":"service_tier","support":"unknown","when":null,"bindings":null,"options":null,"default":null,"conditions":[],"evidence":[]}]},"limits":{"measured":[{"route":"devin","route_label":"Devin subscription","client":"cliproxyapi","client_label":"Requests sent through CLIProxyAPI","context":null,"output":null,"input_accepted":300499,"input_refused":null,"input_read":300499,"output_accepted":null,"note":"no edge found: no request was refused at any size tried, and Devin lists 262,000. Both code words (one before the filler, one after it) came back at 20,493, 200,497 and 300,499 tokens, so the model read those inputs to the end. Requests of up to 930,487 tokens were accepted on 2026-11-13 (results/limits-2026-11-13) with no check that they were read, and a both-ends request then got no answer at 250K and 300K within 5 minutes: those sizes are not in the row","date":"2026-11-16","source":"https://github.com/ngvoicu/groundtruth/blob/main/results/limits-2026-11-16/swe-2-recall.jsonl"}],"from_measurement":[],"input_window":{"tokens":300499,"basis":"measured","bound":"at_least","scope":"input","source":"providers.dev","date":"2026-11-16","documented_tokens":null,"documented_field":null,"documented_source":null,"documented_date":null,"documented_status":null,"measured_tokens":300499,"measured_bound":"at_least","measured_kind":"accepted","measured_refused":null,"measured_date":"2026-11-16","clients":1,"evaluations":1,"measurements":[{"client":"cliproxyapi","client_label":"Requests sent through CLIProxyAPI","route":"devin","route_label":"Devin subscription","date":"2026-11-16","tokens":300499,"bound":"at_least","kind":"accepted","accepted":300499,"refused":null,"named":null,"read":300499,"source":"https://github.com/ngvoicu/groundtruth/blob/main/results/limits-2026-11-16/swe-2-recall.jsonl"}]}},"evidence":[{"kind":"measured","url":"https://github.com/ngvoicu/groundtruth/blob/main/results/cpa-8.0.53/D3-2026-11-04T17-57-26.json","date":"2026-11-04","note":null}],"verified":"2026-11-16"},"name":"swe-2"}}}