{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/extract-json","entry":"extract_json","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":35,"n_papers_ran":21,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":36,"n_samples_ran":22,"n_samples_fingerprinted":13,"n_places":40,"n_places_pointer_only":18,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":6,"ran_fixture":0,"ran":16,"unverified":14},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2609.00759","paper":"/paper/arxiv-2609-00759","title":"Compile, Don't Memorize: A Context Compilation Architecture (CCA) for In-Context Learning","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"TonyQJH/cca-emnlp2026","path":"code/cca_core.py","file_url":"https://github.com/TonyQJH/cca-emnlp2026/blob/HEAD/code/cca_core.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"12de2cde1bb6e417","mcp_get_code":{"code_sha256":"12de2cde1bb6e417"}},{"arxiv_id":"2608.30968","paper":"/paper/arxiv-2608-30968","title":"CogEvol: Towards Efficient and Reliable Learning Environment Generation","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"CogEvol/CogEvol-4B","path":"eval/slide_eval.py","file_url":"https://github.com/CogEvol/CogEvol-4B/blob/HEAD/eval/slide_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b2d30690ed8eef53","mcp_get_code":{"code_sha256":"b2d30690ed8eef53"}},{"arxiv_id":"2607.15766","paper":"/paper/arxiv-2607-15766","title":"Before the Action: Benchmarking LLMs on Prospective Hypothesis Discovery","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"SKYLENAGE-AI/HypoArena","path":"basics/parsing.py","file_url":"https://github.com/SKYLENAGE-AI/HypoArena/blob/HEAD/basics/parsing.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2f41dc825738de71","mcp_get_code":{"code_sha256":"2f41dc825738de71"}},{"arxiv_id":"2606.29920","paper":"/paper/arxiv-2606-29920","title":"Can LLM-as-a-Judge Reliably Verify Rubrics in Agentic Scenarios?","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"THU-KEG/RuVerBench","path":"code/run_judges/agenticcoding_runner.py","file_url":"https://github.com/THU-KEG/RuVerBench/blob/HEAD/code/run_judges/agenticcoding_runner.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"46c0327ba1728a45","mcp_get_code":{"code_sha256":"46c0327ba1728a45"}},{"arxiv_id":"2606.29920","paper":"/paper/arxiv-2606-29920","title":"Can LLM-as-a-Judge Reliably Verify Rubrics in Agentic Scenarios?","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"THU-KEG/RuVerBench","path":"code/run_judges/deepresearch_runner.py","file_url":"https://github.com/THU-KEG/RuVerBench/blob/HEAD/code/run_judges/deepresearch_runner.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"38372855661357c6","mcp_get_code":{"code_sha256":"38372855661357c6"}},{"arxiv_id":"2606.01736","paper":"/paper/arxiv-2606-01736","title":"Argument Collapse: LLMs Flatten Long-Form Public Debate","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"mungg/argument_collapse","path":"src/argument_collapse/annotate/pair_comparison_main_arg.py","file_url":"https://github.com/mungg/argument_collapse/blob/HEAD/src/argument_collapse/annotate/pair_comparison_main_arg.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b933cab935ecc2aa","mcp_get_code":{"code_sha256":"b933cab935ecc2aa"}},{"arxiv_id":"2606.01736","paper":"/paper/arxiv-2606-01736","title":"Argument Collapse: LLMs Flatten Long-Form Public Debate","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"mungg/argument_collapse","path":"src/argument_collapse/annotate/stance.py","file_url":"https://github.com/mungg/argument_collapse/blob/HEAD/src/argument_collapse/annotate/stance.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ec41fe411ec4206b","mcp_get_code":{"code_sha256":"ec41fe411ec4206b"}},{"arxiv_id":"2606.01736","paper":"/paper/arxiv-2606-01736","title":"Argument Collapse: LLMs Flatten Long-Form Public Debate","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"mungg/argument_collapse","path":"src/argument_collapse/annotate/structure.py","file_url":"https://github.com/mungg/argument_collapse/blob/HEAD/src/argument_collapse/annotate/structure.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ba2ef2aa84eacd6c","mcp_get_code":{"code_sha256":"ba2ef2aa84eacd6c"}},{"arxiv_id":"2604.26052","paper":"/paper/arxiv-2604-26052","title":"From Prompt Risk to Response Risk: Paired Analysis of Safety Behavior of Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"microsoft/PairedSafety","path":"analysis/grader_error_analysis/llm_code_errors.py","file_url":"https://github.com/microsoft/PairedSafety/blob/HEAD/analysis/grader_error_analysis/llm_code_errors.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f430bb03b2488d92","mcp_get_code":{"code_sha256":"f430bb03b2488d92"}},{"arxiv_id":"2604.21454","paper":"/paper/arxiv-2604-21454","title":"Reasoning Primitives in Hybrid and Non-Hybrid LLMs: Do Architectural Differences Yield Advantages in State-Tracking and Recall?","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"ultor1996/reasoning_primitives","path":"src/utils.py","file_url":"https://github.com/ultor1996/reasoning_primitives/blob/HEAD/src/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e4135e8bf295baaf","mcp_get_code":{"code_sha256":"e4135e8bf295baaf"}},{"arxiv_id":"2604.00986","paper":"/paper/arxiv-2604-00986","title":"Do Phone-Use Agents Respect Your Privacy?","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"FreedomIntelligence/MyPhoneBench","path":"android_world/agents/agent_utils.py","file_url":"https://github.com/FreedomIntelligence/MyPhoneBench/blob/HEAD/android_world/agents/agent_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"382a844d97026811","mcp_get_code":{"code_sha256":"382a844d97026811"}},{"arxiv_id":"2603.18743","paper":"/paper/arxiv-2603-18743","title":"Memento-Skills: Let Agents Design Agents Memento-Team","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"Memento-Teams/Memento-Skills","path":"core/memento_s/utils.py","file_url":"https://github.com/Memento-Teams/Memento-Skills/blob/HEAD/core/memento_s/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"4b48b517a30002c3","mcp_get_code":{"code_sha256":"4b48b517a30002c3"}},{"arxiv_id":"2603.05522","paper":"/paper/arxiv-2603-05522","title":"RoboLayout: Differentiable 3D Scene Generation for Embodied Agents","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"alishams21/robolayout","path":"src/layoutvlm/layoutvlm.py","file_url":"https://github.com/alishams21/robolayout/blob/HEAD/src/layoutvlm/layoutvlm.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b0987cb5d12e53ca","mcp_get_code":{"code_sha256":"b0987cb5d12e53ca"}},{"arxiv_id":"2603.01990","paper":"/paper/arxiv-2603-01990","title":"According to Me: Long-Term Personalized Referential Memory QA","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"JingbiaoMei/ATM-Bench","path":"memqa/qa_agent_baselines/A-Mem/memory_layer.py","file_url":"https://github.com/JingbiaoMei/ATM-Bench/blob/HEAD/memqa/qa_agent_baselines/A-Mem/memory_layer.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1ba9020de9ace83e","mcp_get_code":{"code_sha256":"1ba9020de9ace83e"}},{"arxiv_id":"2602.09642","paper":"/paper/arxiv-2602-09642","title":"MATA: Multi-Agent Framework for Reliable and Flexible Table Question Answering","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"AIDASLab/MATA","path":"utils/JA_extract_answer.py","file_url":"https://github.com/AIDASLab/MATA/blob/HEAD/utils/JA_extract_answer.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"36d4245c8e43ef52","mcp_get_code":{"code_sha256":"36d4245c8e43ef52"}},{"arxiv_id":"2602.05728","paper":"/paper/arxiv-2602-05728","title":"CompactRAG: Reducing LLM Calls and Token Overhead in Multi-Hop Question Answering","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"How-Young-X/CompactRAG","path":"src/core/AskCorpus.py","file_url":"https://github.com/How-Young-X/CompactRAG/blob/HEAD/src/core/AskCorpus.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9f0a5786975dd602","mcp_get_code":{"code_sha256":"9f0a5786975dd602"}},{"arxiv_id":"2601.17230","paper":"/paper/arxiv-2601-17230","title":"CaseFacts: A Benchmark for Legal Fact-Checking and Precedent Retrieval","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"idirlab/CaseFacts","path":"experiments/check_claims_contradiction.py","file_url":"https://github.com/idirlab/CaseFacts/blob/HEAD/experiments/check_claims_contradiction.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6278226e597fa860","mcp_get_code":{"code_sha256":"6278226e597fa860"}},{"arxiv_id":"2601.17230","paper":"/paper/arxiv-2601-17230","title":"CaseFacts: A Benchmark for Legal Fact-Checking and Precedent Retrieval","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"idirlab/CaseFacts","path":"experiments/check_diff_case_contradiction.py","file_url":"https://github.com/idirlab/CaseFacts/blob/HEAD/experiments/check_diff_case_contradiction.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"678ae97e3c9c6232","mcp_get_code":{"code_sha256":"678ae97e3c9c6232"}},{"arxiv_id":"2512.16279","paper":"/paper/arxiv-2512-16279","title":"QuadSentinel: Sequent Safety for Machine-Checkable Control in Multi-agent Systems","date":null,"month_inferred_from_arxiv_id":"2025-12","title_source":"syntology","repo":"yyiliu/QuadSentinel","path":"src/quadsentinel/utils/functions.py","file_url":"https://github.com/yyiliu/QuadSentinel/blob/HEAD/src/quadsentinel/utils/functions.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"402263fe04b4fc5c","mcp_get_code":{"code_sha256":"402263fe04b4fc5c"}},{"arxiv_id":"2510.22192","paper":"/paper/arxiv-2510-22192","title":"OptiTree: Hierarchical Thoughts Generation with Tree Search for LLM Optimization Modeling","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"MIRALab-USTC/OptiTree","path":"utils.py","file_url":"https://github.com/MIRALab-USTC/OptiTree/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3db55b75deebf780","mcp_get_code":{"code_sha256":"3db55b75deebf780"}},{"arxiv_id":"2506.20100","paper":"/paper/mirage-a-benchmark-for-multimodal-information","title":"MIRAGE: A Benchmark for Multimodal Information-Seeking and Reasoning in Agricultural Expert-Guided Conversations","date":"2025-06-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mirage-benchmark/mirage-benchmark","path":"MMMT/src/clarification_generator.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/clarification_generator.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9593836b0ce2c31a","mcp_get_code":{"code_sha256":"9593836b0ce2c31a"}},{"arxiv_id":"2505.17908","paper":"/paper/comfymind-toward-general-purpose-generation","title":"ComfyMind: Toward General-Purpose Generation via Tree-Based Planning and Reactive Feedback","date":"2025-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"EnVision-Research/ComfyMind","path":"utils/tools.py","file_url":"https://github.com/EnVision-Research/ComfyMind/blob/HEAD/utils/tools.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"00e58640391e7193","mcp_get_code":{"code_sha256":"00e58640391e7193"}},{"arxiv_id":"2504.21716","paper":"/paper/llm-empowered-embodied-agent-for-memory","title":"LLM-Empowered Embodied Agent for Memory-Augmented Task Planning in Household Robotics","date":"2025-04-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"marc1198/chat-hsr","path":"without-ROS/main_single_agent.py","file_url":"https://github.com/marc1198/chat-hsr/blob/HEAD/without-ROS/main_single_agent.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"02d7798ffd79f86d","mcp_get_code":{"code_sha256":"02d7798ffd79f86d"}},{"arxiv_id":"2504.21582","paper":"/paper/mf-llm-simulating-collective-decision","title":"MF-LLM: Simulating Population Decision Dynamics via a Mean-Field Large Language Model Framework","date":"2025-04-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Miracle1207/Mean-Field-LLM","path":"mf_llm/evaluate/evaluate_gpt_batch.py","file_url":"https://github.com/Miracle1207/Mean-Field-LLM/blob/HEAD/mf_llm/evaluate/evaluate_gpt_batch.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"856165cff8587e67","mcp_get_code":{"code_sha256":"856165cff8587e67"}},{"arxiv_id":"2502.18017","paper":"/paper/vidorag-visual-document-retrieval-augmented","title":"ViDoRAG: Visual Document Retrieval-Augmented Generation via Dynamic Iterative Reasoning Agents","date":"2025-02-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Alibaba-NLP/ViDoRAG","path":"vidorag_agents.py","file_url":"https://github.com/Alibaba-NLP/ViDoRAG/blob/HEAD/vidorag_agents.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"34692b533654cd8b","mcp_get_code":{"code_sha256":"34692b533654cd8b"}},{"arxiv_id":"2502.12018","paper":"/paper/atom-of-thoughts-for-markov-llm-test-time","title":"Atom of Thoughts for Markov LLM Test-Time Scaling","date":"2025-02-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"qixucen/atom","path":"experiment/module.py","file_url":"https://github.com/qixucen/atom/blob/HEAD/experiment/module.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"41aebb57ad1d527d","mcp_get_code":{"code_sha256":"41aebb57ad1d527d"}},{"arxiv_id":"2410.13185","paper":"/paper/chain-of-ideas-revolutionizing-research-in","title":"Chain of Ideas: Revolutionizing Research Via Novel Idea Development with LLM Agents","date":"2024-10-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"damo-nlp-sg/coi-agent","path":"utils.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"8da3d9ba5f65d0b8","mcp_get_code":{"code_sha256":"8da3d9ba5f65d0b8"}},{"arxiv_id":"2410.05363","paper":"/paper/towards-world-simulator-crafting-physical","title":"Towards World Simulator: Crafting Physical Commonsense-Based Benchmark for Video Generation","date":"2024-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"opengvlab/phygenbench","path":"PhyGenEval/semantic/gpt4o_sementic.py","file_url":"https://github.com/opengvlab/phygenbench/blob/HEAD/PhyGenEval/semantic/gpt4o_sementic.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f7510953d3aad7af","mcp_get_code":{"code_sha256":"f7510953d3aad7af"}},{"arxiv_id":"2407.14505","paper":"/paper/t2v-compbench-a-comprehensive-benchmark-for","title":"T2V-CompBench: A Comprehensive Benchmark for Compositional Text-to-video Generation","date":"2024-07-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"KaiyueSun98/T2V-CompBench","path":"LLaVA/llava/eval/compbench_eval_action_binding.py","file_url":"https://github.com/KaiyueSun98/T2V-CompBench/blob/HEAD/LLaVA/llava/eval/compbench_eval_action_binding.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"18e1081b616ae0e0","mcp_get_code":{"code_sha256":"18e1081b616ae0e0"}},{"arxiv_id":"2407.14505","paper":"/paper/t2v-compbench-a-comprehensive-benchmark-for","title":"T2V-CompBench: A Comprehensive Benchmark for Compositional Text-to-video Generation","date":"2024-07-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"KaiyueSun98/T2V-CompBench","path":"LLaVA/llava/eval/compbench_eval_consistent_attr.py","file_url":"https://github.com/KaiyueSun98/T2V-CompBench/blob/HEAD/LLaVA/llava/eval/compbench_eval_consistent_attr.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0b9124dcf2830cbb","mcp_get_code":{"code_sha256":"0b9124dcf2830cbb"}},{"arxiv_id":"2407.00132","paper":"/paper/shortcutsbench-a-large-scale-real-world","title":"ShortcutsBench: A Large-Scale Real-world Benchmark for API-based Agents","date":"2024-06-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"EachSheep/ShortcutsBench","path":"experiments/all_experiments.py","file_url":"https://github.com/EachSheep/ShortcutsBench/blob/HEAD/experiments/all_experiments.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"adb0f88a9d1082c2","mcp_get_code":{"code_sha256":"adb0f88a9d1082c2"}},{"arxiv_id":"2405.14573","paper":"/paper/androidworld-a-dynamic-benchmarking","title":"AndroidWorld: A Dynamic Benchmarking Environment for Autonomous Agents","date":"2024-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"google-research/android_world","path":"android_world/agents/agent_utils.py","file_url":"https://github.com/google-research/android_world/blob/HEAD/android_world/agents/agent_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"code_sha256_prefix":"382a844d97026811","mcp_get_code":{"code_sha256":"382a844d97026811"}},{"arxiv_id":"2405.05610","paper":"/paper/chain-of-attack-a-semantic-driven-contextual","title":"Chain of Attack: a Semantic-Driven Contextual Multi-Turn attacker for LLM","date":"2024-05-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yancykahn/coa","path":"common.py","file_url":"https://github.com/yancykahn/coa/blob/HEAD/common.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9f289a28c6cc9b87","mcp_get_code":{"code_sha256":"9f289a28c6cc9b87"}},{"arxiv_id":"2404.12038","paper":"/paper/uncovering-safety-risks-in-open-source-llms","title":"Uncovering Safety Risks of Large Language Models through Concept Activation Vector","date":"2024-04-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tmlr-group/DeepInception","path":"common.py","file_url":"https://github.com/tmlr-group/DeepInception/blob/HEAD/common.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d7a1e728e5b74d3f","mcp_get_code":{"code_sha256":"d7a1e728e5b74d3f"}},{"arxiv_id":"2404.02151","paper":"/paper/jailbreaking-leading-safety-aligned-llms-with","title":"Jailbreaking Leading Safety-Aligned LLMs with Simple Adaptive Attacks","date":"2024-04-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tml-epfl/llm-adaptive-attacks","path":"common.py","file_url":"https://github.com/tml-epfl/llm-adaptive-attacks/blob/HEAD/common.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d7a1e728e5b74d3f","mcp_get_code":{"code_sha256":"d7a1e728e5b74d3f"}},{"arxiv_id":"2403.00867","paper":"/paper/gradient-cuff-detecting-jailbreak-attacks-on","title":"Gradient Cuff: Detecting Jailbreak Attacks on Large Language Models by Exploring Refusal Loss Landscapes","date":"2024-03-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"RICommunity/TAP","path":"common.py","file_url":"https://github.com/RICommunity/TAP/blob/HEAD/common.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7337ffebcca7ec8c","mcp_get_code":{"code_sha256":"7337ffebcca7ec8c"}},{"arxiv_id":"2402.16459","paper":"/paper/defending-llms-against-jailbreaking-attacks","title":"Defending LLMs against Jailbreaking Attacks via Backtranslation","date":"2024-02-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yihanwang617/llm-jailbreaking-defense-backtranslation","path":"PAIR/common.py","file_url":"https://github.com/yihanwang617/llm-jailbreaking-defense-backtranslation/blob/HEAD/PAIR/common.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"code_sha256_prefix":"d7a1e728e5b74d3f","mcp_get_code":{"code_sha256":"d7a1e728e5b74d3f"}},{"arxiv_id":"2311.03191","paper":"/paper/deepinception-hypnotize-large-language-model","title":"DeepInception: Hypnotize Large Language Model to Be Jailbreaker","date":"2023-11-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tmlr-group/deepinception","path":"common.py","file_url":"https://github.com/tmlr-group/deepinception/blob/HEAD/common.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d7a1e728e5b74d3f","mcp_get_code":{"code_sha256":"d7a1e728e5b74d3f"}},{"arxiv_id":"2310.08419","paper":"/paper/jailbreaking-black-box-large-language-models","title":"Jailbreaking Black Box Large Language Models in Twenty Queries","date":"2023-10-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"patrickrchao/jailbreakingllms","path":"conversers.py","file_url":"https://github.com/patrickrchao/jailbreakingllms/blob/HEAD/conversers.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7f22816d570dc592","mcp_get_code":{"code_sha256":"7f22816d570dc592"}},{"arxiv_id":"2308.10032","paper":"/paper/gameeval-evaluating-llms-on-conversational","title":"GameEval: Evaluating LLMs on Conversational Games","date":"2023-08-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gameeval/gameeval","path":"chat/text003_chat.py","file_url":"https://github.com/gameeval/gameeval/blob/HEAD/chat/text003_chat.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e8aa7c0e2fd87cd4","mcp_get_code":{"code_sha256":"e8aa7c0e2fd87cd4"}}]}