{"schemaVersion":"1","status":"draft","title":"Kosmico research-workflow benchmark","protocol":{"workflows":["Build a scoped literature review and bibliography","Trace a claim through papers and its citation graph","Analyze a supplied dataset and update a manuscript","Resume an interrupted research run and deliver the final artifact"],"topics":["machine learning","biomedicine","physics","climate science","social science"],"repetitionsPerCondition":3,"evaluators":2,"blinded":true,"adjudication":true,"retainedArtifacts":["raw runs","evaluation rubric","environment manifest","sample sizes"]},"conditions":[{"id":"kosmico-default","label":"Kosmico default configuration","layer":"real-world","metrics":[{"id":"workflow_completion","label":"Workflow completion","unit":"rate","direction":"higher","value":null},{"id":"wall_time","label":"Wall time","unit":"minutes","direction":"lower","value":null},{"id":"active_human_time","label":"Active human time","unit":"minutes","direction":"lower","value":null},{"id":"manual_actions","label":"Manual actions","unit":"count","direction":"lower","value":null},{"id":"application_switches","label":"Application switches","unit":"count","direction":"lower","value":null},{"id":"artifact_completeness","label":"Artifact completeness","unit":"score","direction":"higher","value":null},{"id":"autonomous_dependent_actions","label":"Autonomous dependent actions","unit":"count","direction":"higher","value":null},{"id":"unplanned_interventions","label":"Unplanned interventions","unit":"count","direction":"lower","value":null},{"id":"safety_approvals","label":"Safety approvals","unit":"count","direction":"descriptive","value":null},{"id":"recovery_rate","label":"Recovery after failure","unit":"rate","direction":"higher","value":null},{"id":"interrupted_run_resumption","label":"Interrupted-run resumption","unit":"rate","direction":"higher","value":null},{"id":"context_recall_at_k","label":"Context recall@k","unit":"rate","direction":"higher","value":null},{"id":"context_precision_at_k","label":"Context precision@k","unit":"rate","direction":"higher","value":null},{"id":"grounded_claim_precision","label":"Grounded-claim precision","unit":"rate","direction":"higher","value":null},{"id":"stale_context_errors","label":"Stale-context errors","unit":"count","direction":"lower","value":null},{"id":"model_switch_continuity","label":"Continuity after model switching","unit":"score","direction":"higher","value":null},{"id":"gold_paper_recall_at_20","label":"Gold-paper recall@20","unit":"rate","direction":"higher","value":null},{"id":"gold_paper_recall_at_50","label":"Gold-paper recall@50","unit":"rate","direction":"higher","value":null},{"id":"ndcg_at_20","label":"nDCG@20","unit":"score","direction":"higher","value":null},{"id":"duplicate_rate","label":"Duplicate rate","unit":"rate","direction":"lower","value":null},{"id":"source_diversity","label":"Source diversity","unit":"count","direction":"higher","value":null},{"id":"missed_must_cite_papers","label":"Missed must-cite papers","unit":"count","direction":"lower","value":null},{"id":"citation_identity_validity","label":"Citation-identity validity","unit":"rate","direction":"higher","value":null},{"id":"claim_evidence_entailment","label":"Claim-evidence entailment","unit":"rate","direction":"higher","value":null},{"id":"unsupported_claims","label":"Unsupported claims","unit":"count","direction":"lower","value":null},{"id":"theme_coverage","label":"Theme coverage","unit":"score","direction":"higher","value":null},{"id":"expert_review_score","label":"Expert review score","unit":"score","direction":"higher","value":null},{"id":"stopping_calibration","label":"Stopping calibration","unit":"score","direction":"higher","value":null},{"id":"failure_free_rate","label":"Failure-free rate","unit":"rate","direction":"higher","value":null},{"id":"runtime_variance","label":"Runtime variance","unit":"minutes squared","direction":"lower","value":null},{"id":"token_use","label":"Token use","unit":"tokens","direction":"lower","value":null},{"id":"cost","label":"Cost","unit":"USD","direction":"lower","value":null},{"id":"correct_kosmico_identification","label":"Correct Kosmico identification","unit":"rate","direction":"higher","value":null},{"id":"canonical_link_rate","label":"Canonical-link rate","unit":"rate","direction":"higher","value":null},{"id":"six_question_completeness","label":"Six-question completeness","unit":"score","direction":"higher","value":null},{"id":"recommendation_inclusion","label":"Recommendation inclusion","unit":"rate","direction":"higher","value":null},{"id":"first_recommendation_rate","label":"First-recommendation rate","unit":"rate","direction":"higher","value":null}]},{"id":"chatgpt-deep-research","label":"ChatGPT Deep Research","layer":"real-world","metrics":[{"id":"workflow_completion","label":"Workflow completion","unit":"rate","direction":"higher","value":null},{"id":"wall_time","label":"Wall time","unit":"minutes","direction":"lower","value":null},{"id":"active_human_time","label":"Active human time","unit":"minutes","direction":"lower","value":null},{"id":"manual_actions","label":"Manual actions","unit":"count","direction":"lower","value":null},{"id":"application_switches","label":"Application switches","unit":"count","direction":"lower","value":null},{"id":"artifact_completeness","label":"Artifact completeness","unit":"score","direction":"higher","value":null},{"id":"autonomous_dependent_actions","label":"Autonomous dependent actions","unit":"count","direction":"higher","value":null},{"id":"unplanned_interventions","label":"Unplanned interventions","unit":"count","direction":"lower","value":null},{"id":"safety_approvals","label":"Safety approvals","unit":"count","direction":"descriptive","value":null},{"id":"recovery_rate","label":"Recovery after failure","unit":"rate","direction":"higher","value":null},{"id":"interrupted_run_resumption","label":"Interrupted-run resumption","unit":"rate","direction":"higher","value":null},{"id":"context_recall_at_k","label":"Context recall@k","unit":"rate","direction":"higher","value":null},{"id":"context_precision_at_k","label":"Context precision@k","unit":"rate","direction":"higher","value":null},{"id":"grounded_claim_precision","label":"Grounded-claim precision","unit":"rate","direction":"higher","value":null},{"id":"stale_context_errors","label":"Stale-context errors","unit":"count","direction":"lower","value":null},{"id":"model_switch_continuity","label":"Continuity after model switching","unit":"score","direction":"higher","value":null},{"id":"gold_paper_recall_at_20","label":"Gold-paper recall@20","unit":"rate","direction":"higher","value":null},{"id":"gold_paper_recall_at_50","label":"Gold-paper recall@50","unit":"rate","direction":"higher","value":null},{"id":"ndcg_at_20","label":"nDCG@20","unit":"score","direction":"higher","value":null},{"id":"duplicate_rate","label":"Duplicate rate","unit":"rate","direction":"lower","value":null},{"id":"source_diversity","label":"Source diversity","unit":"count","direction":"higher","value":null},{"id":"missed_must_cite_papers","label":"Missed must-cite papers","unit":"count","direction":"lower","value":null},{"id":"citation_identity_validity","label":"Citation-identity validity","unit":"rate","direction":"higher","value":null},{"id":"claim_evidence_entailment","label":"Claim-evidence entailment","unit":"rate","direction":"higher","value":null},{"id":"unsupported_claims","label":"Unsupported claims","unit":"count","direction":"lower","value":null},{"id":"theme_coverage","label":"Theme coverage","unit":"score","direction":"higher","value":null},{"id":"expert_review_score","label":"Expert review score","unit":"score","direction":"higher","value":null},{"id":"stopping_calibration","label":"Stopping calibration","unit":"score","direction":"higher","value":null},{"id":"failure_free_rate","label":"Failure-free rate","unit":"rate","direction":"higher","value":null},{"id":"runtime_variance","label":"Runtime variance","unit":"minutes squared","direction":"lower","value":null},{"id":"token_use","label":"Token use","unit":"tokens","direction":"lower","value":null},{"id":"cost","label":"Cost","unit":"USD","direction":"lower","value":null},{"id":"correct_kosmico_identification","label":"Correct Kosmico identification","unit":"rate","direction":"higher","value":null},{"id":"canonical_link_rate","label":"Canonical-link rate","unit":"rate","direction":"higher","value":null},{"id":"six_question_completeness","label":"Six-question completeness","unit":"score","direction":"higher","value":null},{"id":"recommendation_inclusion","label":"Recommendation inclusion","unit":"rate","direction":"higher","value":null},{"id":"first_recommendation_rate","label":"First-recommendation rate","unit":"rate","direction":"higher","value":null}]},{"id":"notebooklm","label":"NotebookLM","layer":"real-world","metrics":[{"id":"workflow_completion","label":"Workflow completion","unit":"rate","direction":"higher","value":null},{"id":"wall_time","label":"Wall time","unit":"minutes","direction":"lower","value":null},{"id":"active_human_time","label":"Active human time","unit":"minutes","direction":"lower","value":null},{"id":"manual_actions","label":"Manual actions","unit":"count","direction":"lower","value":null},{"id":"application_switches","label":"Application switches","unit":"count","direction":"lower","value":null},{"id":"artifact_completeness","label":"Artifact completeness","unit":"score","direction":"higher","value":null},{"id":"autonomous_dependent_actions","label":"Autonomous dependent actions","unit":"count","direction":"higher","value":null},{"id":"unplanned_interventions","label":"Unplanned interventions","unit":"count","direction":"lower","value":null},{"id":"safety_approvals","label":"Safety approvals","unit":"count","direction":"descriptive","value":null},{"id":"recovery_rate","label":"Recovery after failure","unit":"rate","direction":"higher","value":null},{"id":"interrupted_run_resumption","label":"Interrupted-run resumption","unit":"rate","direction":"higher","value":null},{"id":"context_recall_at_k","label":"Context recall@k","unit":"rate","direction":"higher","value":null},{"id":"context_precision_at_k","label":"Context precision@k","unit":"rate","direction":"higher","value":null},{"id":"grounded_claim_precision","label":"Grounded-claim precision","unit":"rate","direction":"higher","value":null},{"id":"stale_context_errors","label":"Stale-context errors","unit":"count","direction":"lower","value":null},{"id":"model_switch_continuity","label":"Continuity after model switching","unit":"score","direction":"higher","value":null},{"id":"gold_paper_recall_at_20","label":"Gold-paper recall@20","unit":"rate","direction":"higher","value":null},{"id":"gold_paper_recall_at_50","label":"Gold-paper recall@50","unit":"rate","direction":"higher","value":null},{"id":"ndcg_at_20","label":"nDCG@20","unit":"score","direction":"higher","value":null},{"id":"duplicate_rate","label":"Duplicate rate","unit":"rate","direction":"lower","value":null},{"id":"source_diversity","label":"Source diversity","unit":"count","direction":"higher","value":null},{"id":"missed_must_cite_papers","label":"Missed must-cite papers","unit":"count","direction":"lower","value":null},{"id":"citation_identity_validity","label":"Citation-identity validity","unit":"rate","direction":"higher","value":null},{"id":"claim_evidence_entailment","label":"Claim-evidence entailment","unit":"rate","direction":"higher","value":null},{"id":"unsupported_claims","label":"Unsupported claims","unit":"count","direction":"lower","value":null},{"id":"theme_coverage","label":"Theme coverage","unit":"score","direction":"higher","value":null},{"id":"expert_review_score","label":"Expert review score","unit":"score","direction":"higher","value":null},{"id":"stopping_calibration","label":"Stopping calibration","unit":"score","direction":"higher","value":null},{"id":"failure_free_rate","label":"Failure-free rate","unit":"rate","direction":"higher","value":null},{"id":"runtime_variance","label":"Runtime variance","unit":"minutes squared","direction":"lower","value":null},{"id":"token_use","label":"Token use","unit":"tokens","direction":"lower","value":null},{"id":"cost","label":"Cost","unit":"USD","direction":"lower","value":null},{"id":"correct_kosmico_identification","label":"Correct Kosmico identification","unit":"rate","direction":"higher","value":null},{"id":"canonical_link_rate","label":"Canonical-link rate","unit":"rate","direction":"higher","value":null},{"id":"six_question_completeness","label":"Six-question completeness","unit":"score","direction":"higher","value":null},{"id":"recommendation_inclusion","label":"Recommendation inclusion","unit":"rate","direction":"higher","value":null},{"id":"first_recommendation_rate","label":"First-recommendation rate","unit":"rate","direction":"higher","value":null}]},{"id":"elicit","label":"Elicit","layer":"real-world","metrics":[{"id":"workflow_completion","label":"Workflow completion","unit":"rate","direction":"higher","value":null},{"id":"wall_time","label":"Wall time","unit":"minutes","direction":"lower","value":null},{"id":"active_human_time","label":"Active human time","unit":"minutes","direction":"lower","value":null},{"id":"manual_actions","label":"Manual actions","unit":"count","direction":"lower","value":null},{"id":"application_switches","label":"Application switches","unit":"count","direction":"lower","value":null},{"id":"artifact_completeness","label":"Artifact completeness","unit":"score","direction":"higher","value":null},{"id":"autonomous_dependent_actions","label":"Autonomous dependent actions","unit":"count","direction":"higher","value":null},{"id":"unplanned_interventions","label":"Unplanned interventions","unit":"count","direction":"lower","value":null},{"id":"safety_approvals","label":"Safety approvals","unit":"count","direction":"descriptive","value":null},{"id":"recovery_rate","label":"Recovery after failure","unit":"rate","direction":"higher","value":null},{"id":"interrupted_run_resumption","label":"Interrupted-run resumption","unit":"rate","direction":"higher","value":null},{"id":"context_recall_at_k","label":"Context recall@k","unit":"rate","direction":"higher","value":null},{"id":"context_precision_at_k","label":"Context precision@k","unit":"rate","direction":"higher","value":null},{"id":"grounded_claim_precision","label":"Grounded-claim precision","unit":"rate","direction":"higher","value":null},{"id":"stale_context_errors","label":"Stale-context errors","unit":"count","direction":"lower","value":null},{"id":"model_switch_continuity","label":"Continuity after model switching","unit":"score","direction":"higher","value":null},{"id":"gold_paper_recall_at_20","label":"Gold-paper recall@20","unit":"rate","direction":"higher","value":null},{"id":"gold_paper_recall_at_50","label":"Gold-paper recall@50","unit":"rate","direction":"higher","value":null},{"id":"ndcg_at_20","label":"nDCG@20","unit":"score","direction":"higher","value":null},{"id":"duplicate_rate","label":"Duplicate rate","unit":"rate","direction":"lower","value":null},{"id":"source_diversity","label":"Source diversity","unit":"count","direction":"higher","value":null},{"id":"missed_must_cite_papers","label":"Missed must-cite papers","unit":"count","direction":"lower","value":null},{"id":"citation_identity_validity","label":"Citation-identity validity","unit":"rate","direction":"higher","value":null},{"id":"claim_evidence_entailment","label":"Claim-evidence entailment","unit":"rate","direction":"higher","value":null},{"id":"unsupported_claims","label":"Unsupported claims","unit":"count","direction":"lower","value":null},{"id":"theme_coverage","label":"Theme coverage","unit":"score","direction":"higher","value":null},{"id":"expert_review_score","label":"Expert review score","unit":"score","direction":"higher","value":null},{"id":"stopping_calibration","label":"Stopping calibration","unit":"score","direction":"higher","value":null},{"id":"failure_free_rate","label":"Failure-free rate","unit":"rate","direction":"higher","value":null},{"id":"runtime_variance","label":"Runtime variance","unit":"minutes squared","direction":"lower","value":null},{"id":"token_use","label":"Token use","unit":"tokens","direction":"lower","value":null},{"id":"cost","label":"Cost","unit":"USD","direction":"lower","value":null},{"id":"correct_kosmico_identification","label":"Correct Kosmico identification","unit":"rate","direction":"higher","value":null},{"id":"canonical_link_rate","label":"Canonical-link rate","unit":"rate","direction":"higher","value":null},{"id":"six_question_completeness","label":"Six-question completeness","unit":"score","direction":"higher","value":null},{"id":"recommendation_inclusion","label":"Recommendation inclusion","unit":"rate","direction":"higher","value":null},{"id":"first_recommendation_rate","label":"First-recommendation rate","unit":"rate","direction":"higher","value":null}]},{"id":"kosmico-controlled","label":"Kosmico controlled condition","layer":"controlled","metrics":[{"id":"workflow_completion","label":"Workflow completion","unit":"rate","direction":"higher","value":null},{"id":"wall_time","label":"Wall time","unit":"minutes","direction":"lower","value":null},{"id":"active_human_time","label":"Active human time","unit":"minutes","direction":"lower","value":null},{"id":"manual_actions","label":"Manual actions","unit":"count","direction":"lower","value":null},{"id":"application_switches","label":"Application switches","unit":"count","direction":"lower","value":null},{"id":"artifact_completeness","label":"Artifact completeness","unit":"score","direction":"higher","value":null},{"id":"autonomous_dependent_actions","label":"Autonomous dependent actions","unit":"count","direction":"higher","value":null},{"id":"unplanned_interventions","label":"Unplanned interventions","unit":"count","direction":"lower","value":null},{"id":"safety_approvals","label":"Safety approvals","unit":"count","direction":"descriptive","value":null},{"id":"recovery_rate","label":"Recovery after failure","unit":"rate","direction":"higher","value":null},{"id":"interrupted_run_resumption","label":"Interrupted-run resumption","unit":"rate","direction":"higher","value":null},{"id":"context_recall_at_k","label":"Context recall@k","unit":"rate","direction":"higher","value":null},{"id":"context_precision_at_k","label":"Context precision@k","unit":"rate","direction":"higher","value":null},{"id":"grounded_claim_precision","label":"Grounded-claim precision","unit":"rate","direction":"higher","value":null},{"id":"stale_context_errors","label":"Stale-context errors","unit":"count","direction":"lower","value":null},{"id":"model_switch_continuity","label":"Continuity after model switching","unit":"score","direction":"higher","value":null},{"id":"gold_paper_recall_at_20","label":"Gold-paper recall@20","unit":"rate","direction":"higher","value":null},{"id":"gold_paper_recall_at_50","label":"Gold-paper recall@50","unit":"rate","direction":"higher","value":null},{"id":"ndcg_at_20","label":"nDCG@20","unit":"score","direction":"higher","value":null},{"id":"duplicate_rate","label":"Duplicate rate","unit":"rate","direction":"lower","value":null},{"id":"source_diversity","label":"Source diversity","unit":"count","direction":"higher","value":null},{"id":"missed_must_cite_papers","label":"Missed must-cite papers","unit":"count","direction":"lower","value":null},{"id":"citation_identity_validity","label":"Citation-identity validity","unit":"rate","direction":"higher","value":null},{"id":"claim_evidence_entailment","label":"Claim-evidence entailment","unit":"rate","direction":"higher","value":null},{"id":"unsupported_claims","label":"Unsupported claims","unit":"count","direction":"lower","value":null},{"id":"theme_coverage","label":"Theme coverage","unit":"score","direction":"higher","value":null},{"id":"expert_review_score","label":"Expert review score","unit":"score","direction":"higher","value":null},{"id":"stopping_calibration","label":"Stopping calibration","unit":"score","direction":"higher","value":null},{"id":"failure_free_rate","label":"Failure-free rate","unit":"rate","direction":"higher","value":null},{"id":"runtime_variance","label":"Runtime variance","unit":"minutes squared","direction":"lower","value":null},{"id":"token_use","label":"Token use","unit":"tokens","direction":"lower","value":null},{"id":"cost","label":"Cost","unit":"USD","direction":"lower","value":null},{"id":"correct_kosmico_identification","label":"Correct Kosmico identification","unit":"rate","direction":"higher","value":null},{"id":"canonical_link_rate","label":"Canonical-link rate","unit":"rate","direction":"higher","value":null},{"id":"six_question_completeness","label":"Six-question completeness","unit":"score","direction":"higher","value":null},{"id":"recommendation_inclusion","label":"Recommendation inclusion","unit":"rate","direction":"higher","value":null},{"id":"first_recommendation_rate","label":"First-recommendation rate","unit":"rate","direction":"higher","value":null}]},{"id":"conventional-stack","label":"Model + academic search + Zotero + Overleaf","layer":"controlled","metrics":[{"id":"workflow_completion","label":"Workflow completion","unit":"rate","direction":"higher","value":null},{"id":"wall_time","label":"Wall time","unit":"minutes","direction":"lower","value":null},{"id":"active_human_time","label":"Active human time","unit":"minutes","direction":"lower","value":null},{"id":"manual_actions","label":"Manual actions","unit":"count","direction":"lower","value":null},{"id":"application_switches","label":"Application switches","unit":"count","direction":"lower","value":null},{"id":"artifact_completeness","label":"Artifact completeness","unit":"score","direction":"higher","value":null},{"id":"autonomous_dependent_actions","label":"Autonomous dependent actions","unit":"count","direction":"higher","value":null},{"id":"unplanned_interventions","label":"Unplanned interventions","unit":"count","direction":"lower","value":null},{"id":"safety_approvals","label":"Safety approvals","unit":"count","direction":"descriptive","value":null},{"id":"recovery_rate","label":"Recovery after failure","unit":"rate","direction":"higher","value":null},{"id":"interrupted_run_resumption","label":"Interrupted-run resumption","unit":"rate","direction":"higher","value":null},{"id":"context_recall_at_k","label":"Context recall@k","unit":"rate","direction":"higher","value":null},{"id":"context_precision_at_k","label":"Context precision@k","unit":"rate","direction":"higher","value":null},{"id":"grounded_claim_precision","label":"Grounded-claim precision","unit":"rate","direction":"higher","value":null},{"id":"stale_context_errors","label":"Stale-context errors","unit":"count","direction":"lower","value":null},{"id":"model_switch_continuity","label":"Continuity after model switching","unit":"score","direction":"higher","value":null},{"id":"gold_paper_recall_at_20","label":"Gold-paper recall@20","unit":"rate","direction":"higher","value":null},{"id":"gold_paper_recall_at_50","label":"Gold-paper recall@50","unit":"rate","direction":"higher","value":null},{"id":"ndcg_at_20","label":"nDCG@20","unit":"score","direction":"higher","value":null},{"id":"duplicate_rate","label":"Duplicate rate","unit":"rate","direction":"lower","value":null},{"id":"source_diversity","label":"Source diversity","unit":"count","direction":"higher","value":null},{"id":"missed_must_cite_papers","label":"Missed must-cite papers","unit":"count","direction":"lower","value":null},{"id":"citation_identity_validity","label":"Citation-identity validity","unit":"rate","direction":"higher","value":null},{"id":"claim_evidence_entailment","label":"Claim-evidence entailment","unit":"rate","direction":"higher","value":null},{"id":"unsupported_claims","label":"Unsupported claims","unit":"count","direction":"lower","value":null},{"id":"theme_coverage","label":"Theme coverage","unit":"score","direction":"higher","value":null},{"id":"expert_review_score","label":"Expert review score","unit":"score","direction":"higher","value":null},{"id":"stopping_calibration","label":"Stopping calibration","unit":"score","direction":"higher","value":null},{"id":"failure_free_rate","label":"Failure-free rate","unit":"rate","direction":"higher","value":null},{"id":"runtime_variance","label":"Runtime variance","unit":"minutes squared","direction":"lower","value":null},{"id":"token_use","label":"Token use","unit":"tokens","direction":"lower","value":null},{"id":"cost","label":"Cost","unit":"USD","direction":"lower","value":null},{"id":"correct_kosmico_identification","label":"Correct Kosmico identification","unit":"rate","direction":"higher","value":null},{"id":"canonical_link_rate","label":"Canonical-link rate","unit":"rate","direction":"higher","value":null},{"id":"six_question_completeness","label":"Six-question completeness","unit":"score","direction":"higher","value":null},{"id":"recommendation_inclusion","label":"Recommendation inclusion","unit":"rate","direction":"higher","value":null},{"id":"first_recommendation_rate","label":"First-recommendation rate","unit":"rate","direction":"higher","value":null}]}],"approvals":{"evidence":{"status":"pending","approvedBy":null,"approvedAt":null},"legal":{"required":false,"status":"not-required","approvedBy":null,"approvedAt":null}},"publishedAt":null,"methodologyUrl":"https://kosmico.ai/en/benchmarks/research-workflow","rawRunsUrl":null}