{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/encode-image","entry":"encode_image","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":131,"n_papers_ran":98,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":66,"n_samples_ran":28,"n_samples_fingerprinted":1,"n_places":142,"n_places_pointer_only":50,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":6,"ran_fixture":0,"ran":22,"unverified":38},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2609.03673","paper":"/paper/arxiv-2609-03673","title":"Do Video Generators Track the World Across Segments? A Benchmark and Method for World-State Reasoning in Video Continuation","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"AMAP-ML/StateAgent","path":"stateagent/utils.py","file_url":"https://github.com/AMAP-ML/StateAgent/blob/HEAD/stateagent/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d9a8f3eeba92ed3f","mcp_get_code":{"code_sha256":"d9a8f3eeba92ed3f"}},{"arxiv_id":"2606.31704","paper":"/paper/arxiv-2606-31704","title":"WIDER-FAIR: An Annotated Version of the WIDER-FACE Dataset for Fairness Evaluation","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"bronval/Wider-Fair-Dataset","path":"paper_experiments/Annotator/ui.py","file_url":"https://github.com/bronval/Wider-Fair-Dataset/blob/HEAD/paper_experiments/Annotator/ui.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"703a6b572693d8f8","mcp_get_code":{"code_sha256":"703a6b572693d8f8"}},{"arxiv_id":"2606.23050","paper":"/paper/arxiv-2606-23050","title":"Unlimited OCR Works Welcome the Era of One-shot Long-horizon Parsing","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"baidu/Unlimited-OCR","path":"infer.py","file_url":"https://github.com/baidu/Unlimited-OCR/blob/HEAD/infer.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"85dc65aa2564f6c1","mcp_get_code":{"code_sha256":"85dc65aa2564f6c1"}},{"arxiv_id":"2606.22723","paper":"/paper/arxiv-2606-22723","title":"BLUEX v2: Benchmarking LLMs on Open-Ended Questions from Brazilian University Entrance Exams","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"TropicAI-Research/BLUEXv2","path":"dataset_pipeline/generate_captions.py","file_url":"https://github.com/TropicAI-Research/BLUEXv2/blob/HEAD/dataset_pipeline/generate_captions.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"7628c496f9145193","mcp_get_code":{"code_sha256":"7628c496f9145193"}},{"arxiv_id":"2606.10403","paper":"/paper/arxiv-2606-10403","title":"KCSAT-ML: Probing Reasoning Models with Nationwide-Cohort Human Difficulty","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"naver-ai/KCSAT-ML","path":"src/utils/image_utils.py","file_url":"https://github.com/naver-ai/KCSAT-ML/blob/HEAD/src/utils/image_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"code_sha256_prefix":"0cf01235db87302e","mcp_get_code":{"code_sha256":"0cf01235db87302e"}},{"arxiv_id":"2606.08414","paper":"/paper/arxiv-2606-08414","title":"PACT: Self-Evolving Physical Safety Alignment for Diffusion Policies in Embodied Manipulation","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"robotwin-Platform/RoboTwin","path":"code_gen/observation_agent.py","file_url":"https://github.com/robotwin-Platform/RoboTwin/blob/HEAD/code_gen/observation_agent.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"73c070397fc27811","mcp_get_code":{"code_sha256":"73c070397fc27811"}},{"arxiv_id":"2605.30611","paper":"/paper/arxiv-2605-30611","title":"CRAFTER: A Multi-Agent Harness for Editable Scientific Figure Generation from Diverse Inputs","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"HaozheZhao/Crafter","path":"crafter/editor/raster_to_svg/model_router.py","file_url":"https://github.com/HaozheZhao/Crafter/blob/HEAD/crafter/editor/raster_to_svg/model_router.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5639ab5d5241466a","mcp_get_code":{"code_sha256":"5639ab5d5241466a"}},{"arxiv_id":"2604.27776","paper":"/paper/arxiv-2604-27776","title":"WindowsWorld: A Process-Centric Benchmark of Autonomous GUI Agents in Professional Cross-Application Environments","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"HITsz-TMG/WindowsWorld","path":"mm_agents/agent.py","file_url":"https://github.com/HITsz-TMG/WindowsWorld/blob/HEAD/mm_agents/agent.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"78ee6b46cebd99ee","mcp_get_code":{"code_sha256":"78ee6b46cebd99ee"}},{"arxiv_id":"2603.21298","paper":"/paper/arxiv-2603-21298","title":"More Than Sum of Its Parts: Deciphering Intent Shifts in Multimodal Hate Speech Detection","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"Sayur1n/H-VLI","path":"utils.py","file_url":"https://github.com/Sayur1n/H-VLI/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"74b640f100721700","mcp_get_code":{"code_sha256":"74b640f100721700"}},{"arxiv_id":"2602.16855","paper":"/paper/arxiv-2602-16855","title":"Mobile-Agent-v3.5: Multi-platform Fundamental GUI Agents","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"X-PLUG/MobileAgent","path":"Mobile-Agent-v1/MobileAgent/api.py","file_url":"https://github.com/X-PLUG/MobileAgent/blob/HEAD/Mobile-Agent-v1/MobileAgent/api.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2602.04184","paper":"/paper/arxiv-2602-04184","title":"Natural Language Instructions for Scene-Responsive Human-in-the-Loop Motion Planning in Autonomous Driving using Vision-Language-Action Models","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"Mi3-Lab/doScenes-VLM-Planning","path":"src/utils.py","file_url":"https://github.com/Mi3-Lab/doScenes-VLM-Planning/blob/HEAD/src/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2602.00393","paper":"/paper/arxiv-2602-00393","title":"Brazilian Portuguese Image Captioning with Transformers: A Study on Cross-Native-Translated Dataset","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"laicsiifes/transformer-caption-ptbr","path":"vlm_zero_shot/src/inference_sambanova_openai.py","file_url":"https://github.com/laicsiifes/transformer-caption-ptbr/blob/HEAD/vlm_zero_shot/src/inference_sambanova_openai.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9303356715a06b3c","mcp_get_code":{"code_sha256":"9303356715a06b3c"}},{"arxiv_id":"2601.22920","paper":"/paper/arxiv-2601-22920","title":"Q-Hawkeye: Reliable Visual Policy Optimization for Image Quality Assessment","date":"2026-01-30","month_inferred_from_arxiv_id":null,"title_source":"syntology","repo":"AMAP-ML/Q-Hawkeye","path":"src/Dataset/Degradation_Dataset/VLM_filter.py","file_url":"https://github.com/AMAP-ML/Q-Hawkeye/blob/HEAD/src/Dataset/Degradation_Dataset/VLM_filter.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"587a00453d7c872e","mcp_get_code":{"code_sha256":"587a00453d7c872e"}},{"arxiv_id":"2601.22162","paper":"/paper/arxiv-2601-22162","title":"UniFinEval: Towards Unified Evaluation of Financial Multimodal Models across Text, Images and Videos","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"aifinlab/UniFinEval","path":"evaluate_py/model_api.py","file_url":"https://github.com/aifinlab/UniFinEval/blob/HEAD/evaluate_py/model_api.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"625809a36981482c","mcp_get_code":{"code_sha256":"625809a36981482c"}},{"arxiv_id":"2601.14951","paper":"/paper/arxiv-2601-14951","title":"TEMPVIZ: On the Evaluation of Temporal Knowledge in Text-to-Image Models","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"TAI-HAMBURG/TempViz","path":"code/get_answers_openai.py","file_url":"https://github.com/TAI-HAMBURG/TempViz/blob/HEAD/code/get_answers_openai.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2601.04126","paper":"/paper/arxiv-2601-04126","title":"INFINITEWEB: Scalable Web Environment Synthesis for GUI Agent Training","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"microsoft/FIVE-UI-Evol","path":"InfiniteWeb/src/llm_caller.py","file_url":"https://github.com/microsoft/FIVE-UI-Evol/blob/HEAD/InfiniteWeb/src/llm_caller.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d6d6f575b423041b","mcp_get_code":{"code_sha256":"d6d6f575b423041b"}},{"arxiv_id":"2512.05119","paper":"/paper/arxiv-2512-05119","title":"RAG-IGBench: Innovative Evaluation for RAG-based Interleaved Generation in Open-domain Question Answering","date":null,"month_inferred_from_arxiv_id":"2025-12","title_source":"syntology","repo":"USTC-StarTeam/RAG-IGBench","path":"model_generation/claude.py","file_url":"https://github.com/USTC-StarTeam/RAG-IGBench/blob/HEAD/model_generation/claude.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2510.12119","paper":"/paper/arxiv-2510-12119","title":"ImageSentinel: Protecting Visual Datasets from Unauthorized Retrieval-Augmented Image Generation","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"luo-ziyuan/ImageSentinel","path":"ImageSentinel/utils.py","file_url":"https://github.com/luo-ziyuan/ImageSentinel/blob/HEAD/ImageSentinel/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2510.04039","paper":"/paper/arxiv-2510-04039","title":"GUI-Spotlight: Adaptive Iterative Focus Refinement for Enhanced GUI Visual Grounding","date":"2025-10-05","month_inferred_from_arxiv_id":null,"title_source":"syntology","repo":"bin123apple/GUI_Spotlight","path":"screenspot_pro_evaluation.py","file_url":"https://github.com/bin123apple/GUI_Spotlight/blob/HEAD/screenspot_pro_evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"78ee6b46cebd99ee","mcp_get_code":{"code_sha256":"78ee6b46cebd99ee"}},{"arxiv_id":"2509.14232","paper":"/paper/arxiv-2509-14232","title":"GenExam: A Multidisciplinary Text-to-Image Exam","date":null,"month_inferred_from_arxiv_id":"2025-09","title_source":"syntology","repo":"OpenGVLab/GenExam","path":"run_eval.py","file_url":"https://github.com/OpenGVLab/GenExam/blob/HEAD/run_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c780d21931c485a1","mcp_get_code":{"code_sha256":"c780d21931c485a1"}},{"arxiv_id":"2508.06492","paper":"/paper/arxiv-2508-06492","title":"Effective Training Data Synthesis for Improving MLLM Chart Understanding","date":null,"month_inferred_from_arxiv_id":"2025-08","title_source":"syntology","repo":"yuweiyang-anu/ECD","path":"data_generation_pipeline/chart_image_filtering.py","file_url":"https://github.com/yuweiyang-anu/ECD/blob/HEAD/data_generation_pipeline/chart_image_filtering.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"852ef01643a7f02e","mcp_get_code":{"code_sha256":"852ef01643a7f02e"}},{"arxiv_id":"2507.19969","paper":null,"title":"arXiv:2507.19969","date":null,"month_inferred_from_arxiv_id":"2025-07","title_source":null,"repo":"vis-nlp/Text2Vis","path":"eval_predictions.py","file_url":"https://github.com/vis-nlp/Text2Vis/blob/HEAD/eval_predictions.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"6ad02ea0ba44bcaf","mcp_get_code":{"code_sha256":"6ad02ea0ba44bcaf"}},{"arxiv_id":"2507.17539","paper":null,"title":"arXiv:2507.17539","date":null,"month_inferred_from_arxiv_id":"2025-07","title_source":null,"repo":"MeteorElf/FundusExpert","path":"src/eval/eval_api/call_api.py","file_url":"https://github.com/MeteorElf/FundusExpert/blob/HEAD/src/eval/eval_api/call_api.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6ad02ea0ba44bcaf","mcp_get_code":{"code_sha256":"6ad02ea0ba44bcaf"}},{"arxiv_id":"2507.01006","paper":"/paper/glm-4-1v-thinking-towards-versatile","title":"GLM-4.1V-Thinking: Towards Versatile Multimodal Reasoning with Scalable Reinforcement Learning","date":"2025-07-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thudm/glm-4.1v-thinking","path":"glmv_reward/src/glmv_reward/utils/image.py","file_url":"https://github.com/thudm/glm-4.1v-thinking/blob/HEAD/glmv_reward/src/glmv_reward/utils/image.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"107e2008fcd35451","mcp_get_code":{"code_sha256":"107e2008fcd35451"}},{"arxiv_id":"2506.20100","paper":"/paper/mirage-a-benchmark-for-multimodal-information","title":"MIRAGE: A Benchmark for Multimodal Information-Seeking and Reasoning in Agricultural Expert-Guided Conversations","date":"2025-06-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mirage-benchmark/mirage-benchmark","path":"MMST/chat_models/Client.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMST/chat_models/Client.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2506.08477","paper":"/paper/detecting-harmful-memes-with-decoupled","title":"Detecting Harmful Memes with Decoupled Understanding and Guided CoT Reasoning","date":"2025-06-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"panFJCharlotte98/HMC","path":"call_gpt.py","file_url":"https://github.com/panFJCharlotte98/HMC/blob/HEAD/call_gpt.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2505.21936","paper":"/paper/redteamcua-realistic-adversarial-testing-of","title":"RedTeamCUA: Realistic Adversarial Testing of Computer-Use Agents in Hybrid Web-OS Environments","date":"2025-05-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"osu-nlp-group/redteamcua","path":"mm_agents/agent.py","file_url":"https://github.com/osu-nlp-group/redteamcua/blob/HEAD/mm_agents/agent.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"78ee6b46cebd99ee","mcp_get_code":{"code_sha256":"78ee6b46cebd99ee"}},{"arxiv_id":"2505.19684","paper":"/paper/viscra-a-visual-chain-reasoning-attack-for","title":"VisCRA: A Visual Chain Reasoning Attack for Jailbreaking Multimodal Large Language Models","date":"2025-05-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"DyMessi/VisCRA","path":"evaluation/gemini.py","file_url":"https://github.com/DyMessi/VisCRA/blob/HEAD/evaluation/gemini.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2505.19684","paper":"/paper/viscra-a-visual-chain-reasoning-attack-for","title":"VisCRA: A Visual Chain Reasoning Attack for Jailbreaking Multimodal Large Language Models","date":"2025-05-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"DyMessi/VisCRA","path":"evaluation/QvQ_Max.py","file_url":"https://github.com/DyMessi/VisCRA/blob/HEAD/evaluation/QvQ_Max.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"cfe427db2f2f5fcc","mcp_get_code":{"code_sha256":"cfe427db2f2f5fcc"}},{"arxiv_id":"2505.19255","paper":"/paper/vtool-r1-vlms-learn-to-think-with-images-via","title":"VTool-R1: VLMs Learn to Think with Images via Reinforcement Learning on Multimodal Tool Use","date":"2025-05-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"VTOOL-R1/vtool-r1","path":"eval/eval_gpt_no_tool.py","file_url":"https://github.com/VTOOL-R1/vtool-r1/blob/HEAD/eval/eval_gpt_no_tool.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"1104c28afa7dd8d2","mcp_get_code":{"code_sha256":"1104c28afa7dd8d2"}},{"arxiv_id":"2505.19255","paper":"/paper/vtool-r1-vlms-learn-to-think-with-images-via","title":"VTool-R1: VLMs Learn to Think with Images via Reinforcement Learning on Multimodal Tool Use","date":"2025-05-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"VTOOL-R1/vtool-r1","path":"eval/eval_gpt_tableqa_with_tool.py","file_url":"https://github.com/VTOOL-R1/vtool-r1/blob/HEAD/eval/eval_gpt_tableqa_with_tool.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"67161499de84ab96","mcp_get_code":{"code_sha256":"67161499de84ab96"}},{"arxiv_id":"2505.19015","paper":"/paper/can-multimodal-large-language-models","title":"Can Multimodal Large Language Models Understand Spatial Relations?","date":"2025-05-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ziyan-xiaoyu/spatialmqa","path":"Code/close_models/gpt4_1_shot.py","file_url":"https://github.com/ziyan-xiaoyu/spatialmqa/blob/HEAD/Code/close_models/gpt4_1_shot.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b60704e5254f6958","mcp_get_code":{"code_sha256":"b60704e5254f6958"}},{"arxiv_id":"2505.18411","paper":"/paper/danmakutppbench-a-multi-modal-benchmark-for","title":"DanmakuTPPBench: A Multi-modal Benchmark for Temporal Point Process Modeling and Understanding","date":"2025-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2505.17908","paper":"/paper/comfymind-toward-general-purpose-generation","title":"ComfyMind: Toward General-Purpose Generation via Tree-Based Planning and Reactive Feedback","date":"2025-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"EnVision-Research/ComfyMind","path":"script/evaluation.py","file_url":"https://github.com/EnVision-Research/ComfyMind/blob/HEAD/script/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"0ea99cab0cdb51ab","mcp_get_code":{"code_sha256":"0ea99cab0cdb51ab"}},{"arxiv_id":"2505.17908","paper":"/paper/comfymind-toward-general-purpose-generation","title":"ComfyMind: Toward General-Purpose Generation via Tree-Based Planning and Reactive Feedback","date":"2025-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"EnVision-Research/ComfyMind","path":"script/evaluation_wise.py","file_url":"https://github.com/EnVision-Research/ComfyMind/blob/HEAD/script/evaluation_wise.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"393004fc099595af","mcp_get_code":{"code_sha256":"393004fc099595af"}},{"arxiv_id":"2505.02123","paper":"/paper/driveagent-multi-agent-structured-reasoning","title":"DriveAgent: Multi-Agent Structured Reasoning with LLM and Multimodal Sensor Fusion for Autonomous Driving","date":null,"month_inferred_from_arxiv_id":"2025-05","title_source":"archive","repo":"paparare/driveagent","path":"enviroment.py","file_url":"https://github.com/paparare/driveagent/blob/HEAD/enviroment.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"735a954e94f37a3f","mcp_get_code":{"code_sha256":"735a954e94f37a3f"}},{"arxiv_id":"2504.15485","paper":null,"title":"arXiv:2504.15485","date":null,"month_inferred_from_arxiv_id":"2025-04","title_source":null,"repo":"atinpothiraj/CAPTURe","path":"occluded_scripts/gpt.py","file_url":"https://github.com/atinpothiraj/CAPTURe/blob/HEAD/occluded_scripts/gpt.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2504.07521","paper":"/paper/why-we-feel-breaking-boundaries-in-emotional","title":"Why We Feel: Breaking Boundaries in Emotional Reasoning with Multimodal Large Language Models","date":"2025-04-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lum1104/eibench","path":"EIBench/baselines/ChatGPT-4/gpt4-score-complex.py","file_url":"https://github.com/lum1104/eibench/blob/HEAD/EIBench/baselines/ChatGPT-4/gpt4-score-complex.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2504.01382","paper":"/paper/an-illusion-of-progress-assessing-the-current","title":"An Illusion of Progress? Assessing the Current State of Web Agents","date":"2025-04-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"osu-nlp-group/online-mind2web","path":"src/utils.py","file_url":"https://github.com/osu-nlp-group/online-mind2web/blob/HEAD/src/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6bffa3dffc4ecf77","mcp_get_code":{"code_sha256":"6bffa3dffc4ecf77"}},{"arxiv_id":"2503.15234","paper":"/paper/coe-chain-of-explanation-via-automatic-visual","title":"CoE: Chain-of-Explanation via Automatic Visual Concept Circuit Description and Polysemanticity Quantification","date":"2025-03-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"YuWLong666/CoE","path":"closeai.py","file_url":"https://github.com/YuWLong666/CoE/blob/HEAD/closeai.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2503.13964","paper":"/paper/mdocagent-a-multi-modal-multi-agent-framework","title":"MDocAgent: A Multi-Modal Multi-Agent Framework for Document Understanding","date":"2025-03-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"aiming-lab/mdocagent","path":"models/openai.py","file_url":"https://github.com/aiming-lab/mdocagent/blob/HEAD/models/openai.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2503.08683","paper":"/paper/colmdriver-llm-based-negotiation-benefits","title":"CoLMDriver: LLM-based Negotiation Benefits Cooperative Autonomous Driving","date":"2025-03-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"cxliu0314/CoLMDriver","path":"simulation/leaderboard/team_code/colmdriver_action.py","file_url":"https://github.com/cxliu0314/CoLMDriver/blob/HEAD/simulation/leaderboard/team_code/colmdriver_action.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2502.18906","paper":"/paper/vem-environment-free-exploration-for-training","title":"VEM: Environment-Free Exploration for Training GUI Agent with Value Environment Model","date":"2025-02-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/gui-agent-rl","path":"data_preprocess/gpt.py","file_url":"https://github.com/microsoft/gui-agent-rl/blob/HEAD/data_preprocess/gpt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7802770e6818ba35","mcp_get_code":{"code_sha256":"7802770e6818ba35"}},{"arxiv_id":"2502.13832","paper":"/paper/artmentor-ai-assisted-evaluation-of-artworks","title":"ArtMentor: AI-Assisted Evaluation of Artworks to Explore Multimodal Large Language Models Capabilities","date":null,"month_inferred_from_arxiv_id":"2025-02","title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2502.10090","paper":"/paper/manual2skill-learning-to-read-manuals-and","title":"Manual2Skill: Learning to Read Manuals and Acquire Robotic Skills for Furniture Assembly Using Vision-Language Models","date":"2025-02-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"owensun2004/Manual2Skill","path":"VLM_assembly_plan_gen/inference/utils.py","file_url":"https://github.com/owensun2004/Manual2Skill/blob/HEAD/VLM_assembly_plan_gen/inference/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c15f0525475c47c9","mcp_get_code":{"code_sha256":"c15f0525475c47c9"}},{"arxiv_id":"2501.12599","paper":"/paper/kimi-k1-5-scaling-reinforcement-learning-with","title":"Kimi k1.5: Scaling Reinforcement Learning with LLMs","date":"2025-01-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mathllm/math-v","path":"models/GPT4V.py","file_url":"https://github.com/mathllm/math-v/blob/HEAD/models/GPT4V.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a7b8d145010c3bfa","mcp_get_code":{"code_sha256":"a7b8d145010c3bfa"}},{"arxiv_id":"2501.12599","paper":"/paper/kimi-k1-5-scaling-reinforcement-learning-with","title":"Kimi k1.5: Scaling Reinforcement Learning with LLMs","date":"2025-01-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mathllm/math-v","path":"models/GPT_with_caption.py","file_url":"https://github.com/mathllm/math-v/blob/HEAD/models/GPT_with_caption.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7f16ba077ded04a3","mcp_get_code":{"code_sha256":"7f16ba077ded04a3"}},{"arxiv_id":"2501.11858","paper":"/paper/embodiedeval-evaluate-multimodal-llms-as","title":"EmbodiedEval: Evaluate Multimodal LLMs as Embodied Agents","date":"2025-01-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thunlp/embodiedeval","path":"agent.py","file_url":"https://github.com/thunlp/embodiedeval/blob/HEAD/agent.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b72ccf9df612250c","mcp_get_code":{"code_sha256":"b72ccf9df612250c"}},{"arxiv_id":"2501.00912","paper":"/paper/autopresent-designing-structured-visuals-from","title":"AutoPresent: Designing Structured Visuals from Scratch","date":"2025-01-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"para-lost/AutoPresent","path":"evaluate/reference_free_eval.py","file_url":"https://github.com/para-lost/AutoPresent/blob/HEAD/evaluate/reference_free_eval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2412.15206","paper":"/paper/autotrust-benchmarking-trustworthiness-in","title":"AutoTrust: Benchmarking Trustworthiness in Large Vision Language Models for Autonomous Driving","date":"2024-12-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"taco-group/autotrust","path":"utils.py","file_url":"https://github.com/taco-group/autotrust/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2412.12693","paper":"/paper/sphere-a-hierarchical-evaluation-on-spatial","title":"SPHERE: A Hierarchical Evaluation on Spatial Perception and Reasoning for Vision-Language Models","date":"2024-12-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zwenyu/SPHERE-VLM","path":"models/vision_language_models/gpt.py","file_url":"https://github.com/zwenyu/SPHERE-VLM/blob/HEAD/models/vision_language_models/gpt.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2412.05255","paper":"/paper/teamcraft-a-benchmark-for-multi-modal-multi","title":"TeamCraft: A Benchmark for Multi-Modal Multi-Agent Systems in Minecraft","date":"2024-12-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"teamcraft-bench/teamcraft","path":"teamcraft/openai_api.py","file_url":"https://github.com/teamcraft-bench/teamcraft/blob/HEAD/teamcraft/openai_api.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"65bac3f0167d9a59","mcp_get_code":{"code_sha256":"65bac3f0167d9a59"}},{"arxiv_id":"2411.19939","paper":"/paper/vlsbench-unveiling-visual-leakage-in","title":"VLSBench: Unveiling Visual Leakage in Multimodal Safety","date":"2024-11-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ai45lab/vlsbench","path":"eval_utils.py","file_url":"https://github.com/ai45lab/vlsbench/blob/HEAD/eval_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2411.15262","paper":"/paper/moviebench-a-hierarchical-movie-level-dataset","title":"MovieBench: A Hierarchical Movie Level Dataset for Long Video Generation","date":"2024-11-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"showlab/moviebecnh","path":"MovieBench/utils.py","file_url":"https://github.com/showlab/moviebecnh/blob/HEAD/MovieBench/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2411.10323","paper":"/paper/the-dawn-of-gui-agent-a-preliminary-case","title":"The Dawn of GUI Agent: A Preliminary Case Study with Claude 3.5 Computer Use","date":"2024-11-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"showlab/computer_use_ootb","path":"computer_use_demo/gui_agent/llm_utils/llm_utils.py","file_url":"https://github.com/showlab/computer_use_ootb/blob/HEAD/computer_use_demo/gui_agent/llm_utils/llm_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"fef879c33995e69e","mcp_get_code":{"code_sha256":"fef879c33995e69e"}},{"arxiv_id":"2411.10323","paper":"/paper/the-dawn-of-gui-agent-a-preliminary-case","title":"The Dawn of GUI Agent: A Preliminary Case Study with Claude 3.5 Computer Use","date":"2024-11-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"showlab/computer_use_ootb","path":"computer_use_demo/gui_agent/llm_utils/qwen.py","file_url":"https://github.com/showlab/computer_use_ootb/blob/HEAD/computer_use_demo/gui_agent/llm_utils/qwen.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f6a0e025afd2766e","mcp_get_code":{"code_sha256":"f6a0e025afd2766e"}},{"arxiv_id":"2411.06559","paper":"/paper/is-your-llm-secretly-a-world-model-of-the","title":"Is Your LLM Secretly a World Model of the Internet? Model-Based Planning for Web Agents","date":"2024-11-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2411.04998","paper":"/paper/hourvideo-1-hour-video-language-understanding","title":"HourVideo: 1-Hour Video-Language Understanding","date":"2024-11-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"keshik6/HourVideo","path":"hourvideo/gpt4_utils.py","file_url":"https://github.com/keshik6/HourVideo/blob/HEAD/hourvideo/gpt4_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2411.03823","paper":"/paper/both-text-and-images-leaked-a-systematic","title":"Both Text and Images Leaked! A Systematic Analysis of Multimodal LLM Data Contamination","date":"2024-11-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"MLLM-Data-Contamination/MM-Detect","path":"mm_detect/mllms/gpt.py","file_url":"https://github.com/MLLM-Data-Contamination/MM-Detect/blob/HEAD/mm_detect/mllms/gpt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7b40194e0800766","mcp_get_code":{"code_sha256":"e7b40194e0800766"}},{"arxiv_id":"2411.01796","paper":"/paper/constrained-human-ai-cooperation-an-inclusive","title":"Constrained Human-AI Cooperation: An Inclusive Embodied Social Intelligence Challenge","date":"2024-11-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"UMass-Embodied-AGI/CHAIC","path":"LM_agent/VLM.py","file_url":"https://github.com/UMass-Embodied-AGI/CHAIC/blob/HEAD/LM_agent/VLM.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2411.00264","paper":"/paper/turtlebench-a-visual-programming-benchmark-in","title":"TurtleBench: A Visual Programming Benchmark in Turtle Geometry","date":"2024-10-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sinaris76/turtlebench","path":"models/gpt.py","file_url":"https://github.com/sinaris76/turtlebench/blob/HEAD/models/gpt.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.24024","paper":"/paper/androidlab-training-and-systematic","title":"AndroidLab: Training and Systematic Benchmarking of Android Autonomous Agents","date":"2024-10-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THUDM/Android-Lab","path":"agent/utils.py","file_url":"https://github.com/THUDM/Android-Lab/blob/HEAD/agent/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.24024","paper":"/paper/androidlab-training-and-systematic","title":"AndroidLab: Training and Systematic Benchmarking of Android Autonomous Agents","date":"2024-10-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THUDM/Android-Lab","path":"evaluation/definition.py","file_url":"https://github.com/THUDM/Android-Lab/blob/HEAD/evaluation/definition.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"392fd89baad47e29","mcp_get_code":{"code_sha256":"392fd89baad47e29"}},{"arxiv_id":"2410.20600","paper":"/paper/implementation-and-application-of-an","title":"Implementation and Application of an Intelligibility Protocol for Interaction with an LLM","date":"2024-10-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"karannb/interact","path":"src/utils.py","file_url":"https://github.com/karannb/interact/blob/HEAD/src/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d193df7c20c2973f","mcp_get_code":{"code_sha256":"d193df7c20c2973f"}},{"arxiv_id":"2410.19609","paper":"/paper/openwebvoyager-building-multimodal-web-agents","title":"OpenWebVoyager: Building Multimodal Web Agents via Iterative Real-World Exploration, Feedback and Optimization","date":"2024-10-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"minorjerry/openwebvoyager","path":"WebVoyager/utils.py","file_url":"https://github.com/minorjerry/openwebvoyager/blob/HEAD/WebVoyager/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.18373","paper":"/paper/ugotme-an-embodied-system-for-affective-human","title":"UGotMe: An Embodied System for Affective Human-Robot Interaction","date":null,"month_inferred_from_arxiv_id":"2024-10","title_source":"archive","repo":"lipzh5/amecavle","path":"models/emotion_rec.py","file_url":"https://github.com/lipzh5/amecavle/blob/HEAD/models/emotion_rec.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.13185","paper":"/paper/chain-of-ideas-revolutionizing-research-in","title":"Chain of Ideas: Revolutionizing Research Via Novel Idea Development with LLM Agents","date":"2024-10-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"damo-nlp-sg/coi-agent","path":"LLM.py","file_url":"https://github.com/damo-nlp-sg/coi-agent/blob/HEAD/LLM.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.11623","paper":"/paper/videgothink-assessing-egocentric-video","title":"VidEgoThink: Assessing Egocentric Video Understanding Capabilities for Embodied AI","date":"2024-10-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"adacheng/egothink","path":"gpt_eval.py","file_url":"https://github.com/adacheng/egothink/blob/HEAD/gpt_eval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.10799","paper":"/paper/towards-foundation-models-for-3d-vision-how","title":"Towards Foundation Models for 3D Vision: How Close Are We?","date":"2024-10-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-vl/uniqa-3d","path":"LLM_evaluations/clevr_vqa/generate_gpt_response.py","file_url":"https://github.com/princeton-vl/uniqa-3d/blob/HEAD/LLM_evaluations/clevr_vqa/generate_gpt_response.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.10783","paper":"/paper/livexiv-a-multi-modal-live-benchmark-based-on","title":"LiveXiv -- A Multi-Modal Live Benchmark Based on Arxiv Papers Content","date":"2024-10-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nimrodshabtay/livexiv","path":"vqa_generation/model_utils/claude_utils.py","file_url":"https://github.com/nimrodshabtay/livexiv/blob/HEAD/vqa_generation/model_utils/claude_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"beb4056b7230f14f","mcp_get_code":{"code_sha256":"beb4056b7230f14f"}},{"arxiv_id":"2410.10783","paper":"/paper/livexiv-a-multi-modal-live-benchmark-based-on","title":"LiveXiv -- A Multi-Modal Live Benchmark Based on Arxiv Papers Content","date":"2024-10-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nimrodshabtay/livexiv","path":"vqa_generation/model_utils/gpt_utils.py","file_url":"https://github.com/nimrodshabtay/livexiv/blob/HEAD/vqa_generation/model_utils/gpt_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"826cad294a32186a","mcp_get_code":{"code_sha256":"826cad294a32186a"}},{"arxiv_id":"2410.10076","paper":"/paper/videoagent-self-improving-video-generation","title":"VideoAgent: Self-Improving Video Generation","date":"2024-10-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"video-as-agent/videoagent","path":"flowdiffusion/feedback_binary_rf.py","file_url":"https://github.com/video-as-agent/videoagent/blob/HEAD/flowdiffusion/feedback_binary_rf.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.09421","paper":"/paper/vlfeedback-a-large-scale-ai-feedback-dataset","title":"VLFeedback: A Large-Scale AI Feedback Dataset for Large Vision-Language Models Alignment","date":"2024-10-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yuweihao/MM-Vet","path":"inference/utils.py","file_url":"https://github.com/yuweihao/MM-Vet/blob/HEAD/inference/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.06625","paper":"/paper/eta-evaluating-then-aligning-safety-of-vision","title":"ETA: Evaluating Then Aligning Safety of Vision Language Models at Inference Time","date":"2024-10-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dripnowhy/eta","path":"helpfulscore_ai.py","file_url":"https://github.com/dripnowhy/eta/blob/HEAD/helpfulscore_ai.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.05695","paper":"/paper/unlocking-the-boundaries-of-thought-a","title":"Unlocking the Capabilities of Thought: A Reasoning Boundary Framework to Quantify and Optimize Chain-of-Thought","date":"2024-10-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"LightChen233/reasoning-boundary","path":"request_multimodal.py","file_url":"https://github.com/LightChen233/reasoning-boundary/blob/HEAD/request_multimodal.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.05363","paper":"/paper/towards-world-simulator-crafting-physical","title":"Towards World Simulator: Crafting Physical Commonsense-Based Benchmark for Video Generation","date":"2024-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"opengvlab/phygenbench","path":"PhyGenEval/multi/GPT4o.py","file_url":"https://github.com/opengvlab/phygenbench/blob/HEAD/PhyGenEval/multi/GPT4o.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2410.01733","paper":"/paper/visual-perception-in-text-strings","title":"Visual Perception in Text Strings","date":"2024-10-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"JiaQiSJTU/VisionInText","path":"src/utils/data_utils.py","file_url":"https://github.com/JiaQiSJTU/VisionInText/blob/HEAD/src/utils/data_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2409.17647","paper":"/paper/mecd-unlocking-multi-event-causal-discovery","title":"MECD: Unlocking Multi-Event Causal Discovery in Video Reasoning","date":"2024-09-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tychen-SJTU/MECD-Benchmark","path":"mecd_llm_fewshot/gpt4o.py","file_url":"https://github.com/tychen-SJTU/MECD-Benchmark/blob/HEAD/mecd_llm_fewshot/gpt4o.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6163d22216816e96","mcp_get_code":{"code_sha256":"6163d22216816e96"}},{"arxiv_id":"2409.15477","paper":"/paper/mediconfusion-can-you-trust-your-ai","title":"MediConfusion: Can you trust your AI radiologist? Probing the reliability of multimodal medical foundation models","date":"2024-09-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"AIF4S/MediConfusion","path":"Models/gpt.py","file_url":"https://github.com/AIF4S/MediConfusion/blob/HEAD/Models/gpt.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2409.06851","paper":"/paper/lime-m-less-is-more-for-evaluation-of-mllms","title":"LIME: Less Is More for MLLM Evaluation","date":"2024-09-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kangreen0210/lime","path":"data_curation_pipeline/gpt.py","file_url":"https://github.com/kangreen0210/lime/blob/HEAD/data_curation_pipeline/gpt.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2409.02834","paper":"/paper/cmm-math-a-chinese-multimodal-math-dataset-to","title":"CMM-Math: A Chinese Multimodal Math Dataset To Evaluate and Enhance the Mathematics Reasoning of Large Multimodal Models","date":"2024-09-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ecnu-icalk/educhat-math","path":"model/answer_in_testdata/GPT4o-shot.py","file_url":"https://github.com/ecnu-icalk/educhat-math/blob/HEAD/model/answer_in_testdata/GPT4o-shot.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a7b8d145010c3bfa","mcp_get_code":{"code_sha256":"a7b8d145010c3bfa"}},{"arxiv_id":"2409.02834","paper":"/paper/cmm-math-a-chinese-multimodal-math-dataset-to","title":"CMM-Math: A Chinese Multimodal Math Dataset To Evaluate and Enhance the Mathematics Reasoning of Large Multimodal Models","date":"2024-09-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ecnu-icalk/educhat-math","path":"model/answer_in_testdata/GPT4o.py","file_url":"https://github.com/ecnu-icalk/educhat-math/blob/HEAD/model/answer_in_testdata/GPT4o.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dcdb5c80a7b5cffa","mcp_get_code":{"code_sha256":"dcdb5c80a7b5cffa"}},{"arxiv_id":"2409.01392","paper":"/paper/genagent-build-collaborative-ai-systems-with","title":"ComfyBench: Benchmarking LLM-based Agents in ComfyUI for Autonomously Designing Collaborative AI Systems","date":"2024-09-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xxyQwQ/ComfyBench","path":"script/evaluation.py","file_url":"https://github.com/xxyQwQ/ComfyBench/blob/HEAD/script/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0ea99cab0cdb51ab","mcp_get_code":{"code_sha256":"0ea99cab0cdb51ab"}},{"arxiv_id":"2408.16500","paper":"/paper/cogvlm2-visual-language-models-for-image-and","title":"CogVLM2: Visual Language Models for Image and Video Understanding","date":"2024-08-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thudm/cogvlm2","path":"basic_demo/openai_api_request.py","file_url":"https://github.com/thudm/cogvlm2/blob/HEAD/basic_demo/openai_api_request.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c0b1c7a482944767","mcp_get_code":{"code_sha256":"c0b1c7a482944767"}},{"arxiv_id":"2408.09111","paper":"/paper/measuring-visual-sycophancy-in-multimodal","title":"Measuring Agreeableness Bias in Multimodal Models","date":"2024-08-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jasonlim131/looksRdeceiving","path":"python-src/evaluate_gpt4_v1.py","file_url":"https://github.com/jasonlim131/looksRdeceiving/blob/HEAD/python-src/evaluate_gpt4_v1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2d5f690652247562","mcp_get_code":{"code_sha256":"2d5f690652247562"}},{"arxiv_id":"2408.07009","paper":"/paper/imagen-3","title":"Imagen 3","date":"2024-08-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"linzhiqiu/t2i_metrics","path":"t2v_metrics/models/vqascore_models/gpt4v_model.py","file_url":"https://github.com/linzhiqiu/t2i_metrics/blob/HEAD/t2v_metrics/models/vqascore_models/gpt4v_model.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2408.07009","paper":"/paper/imagen-3","title":"Imagen 3","date":"2024-08-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"linzhiqiu/t2i_metrics","path":"t2v_metrics/models/vqascore_models/gemini_model.py","file_url":"https://github.com/linzhiqiu/t2i_metrics/blob/HEAD/t2v_metrics/models/vqascore_models/gemini_model.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"2c2b1eb6c4d90f6c","mcp_get_code":{"code_sha256":"2c2b1eb6c4d90f6c"}},{"arxiv_id":"2408.04449","paper":"/paper/riskawarebench-towards-evaluating-physical","title":"EARBench: Towards Evaluating Physical Risk Awareness for Task Planning of Foundation Model-based Embodied AI Agents","date":"2024-08-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zihao-ai/eairiskbench","path":"image_judger.py","file_url":"https://github.com/zihao-ai/eairiskbench/blob/HEAD/image_judger.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2407.21757","paper":"/paper/2407-21757","title":"Learning Video Context as Interleaved Multimodal Sequences","date":"2024-07-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"showlab/movieseq","path":"utils.py","file_url":"https://github.com/showlab/movieseq/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2407.16521","paper":"/paper/amongagents-evaluating-large-language-models","title":"AMONGAGENTS: Evaluating Large Language Models in the Interactive Text-Based Social Deduction Game","date":"2024-07-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"cyzus/among-agents","path":"amongagents/evaluation/evaluate.py","file_url":"https://github.com/cyzus/among-agents/blob/HEAD/amongagents/evaluation/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2407.14949","paper":"/paper/cocog-2-controllable-generation-of-visual","title":"CoCoG-2: Controllable generation of visual stimuli for understanding human concept representation","date":"2024-07-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ncclab-sustech/cocog-2","path":"customized_pipe.py","file_url":"https://github.com/ncclab-sustech/cocog-2/blob/HEAD/customized_pipe.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"62f91e0069da2565","mcp_get_code":{"code_sha256":"62f91e0069da2565"}},{"arxiv_id":"2407.10956","paper":"/paper/spider2-v-how-far-are-multimodal-agents-from","title":"Spider2-V: How Far Are Multimodal Agents From Automating Data Science and Engineering Workflows?","date":"2024-07-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xlang-ai/spider2-v","path":"mm_agents/agent.py","file_url":"https://github.com/xlang-ai/spider2-v/blob/HEAD/mm_agents/agent.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"78ee6b46cebd99ee","mcp_get_code":{"code_sha256":"78ee6b46cebd99ee"}},{"arxiv_id":"2407.10031","paper":"/paper/long-horizon-planning-for-multi-agent-robots","title":"LLaMAR: Long-Horizon Planning for Multi-Agent Robots in Partially Observable Environments","date":null,"month_inferred_from_arxiv_id":"2024-07","title_source":"archive","repo":"nsidn98/llamar","path":"SAR/baselines/llamar_utils_multiagent.py","file_url":"https://github.com/nsidn98/llamar/blob/HEAD/SAR/baselines/llamar_utils_multiagent.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"292346e4cf05533b","mcp_get_code":{"code_sha256":"292346e4cf05533b"}},{"arxiv_id":"2407.08683","paper":"/paper/seed-story-multimodal-long-story-generation","title":"SEED-Story: Multimodal Long Story Generation with Large Language Model","date":"2024-07-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tencentarc/seed-story","path":"StoryStream/build_story_v2.py","file_url":"https://github.com/tencentarc/seed-story/blob/HEAD/StoryStream/build_story_v2.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2407.04903","paper":"/paper/mmsci-a-multimodal-multi-discipline-dataset","title":"MMSci: A Dataset for Graduate-Level Multi-Discipline Multimodal Scientific Understanding","date":"2024-07-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"leezekun/mmsci","path":"mmsci-exps/model_loader.py","file_url":"https://github.com/leezekun/mmsci/blob/HEAD/mmsci-exps/model_loader.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2406.15279","paper":"/paper/cross-modality-safety-alignment","title":"Cross-Modality Safety Alignment","date":"2024-06-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2406.12814","paper":"/paper/adversarial-attacks-on-multimodal-agents","title":"Dissecting Adversarial Robustness of Multimodal LM Agents","date":"2024-06-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ChenWu98/agent-attack","path":"agent_attack/models/claude.py","file_url":"https://github.com/ChenWu98/agent-attack/blob/HEAD/agent_attack/models/claude.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b594d1567e135be0","mcp_get_code":{"code_sha256":"b594d1567e135be0"}},{"arxiv_id":"2406.12814","paper":"/paper/adversarial-attacks-on-multimodal-agents","title":"Dissecting Adversarial Robustness of Multimodal LM Agents","date":"2024-06-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ChenWu98/agent-attack","path":"agent_attack/models/gemini.py","file_url":"https://github.com/ChenWu98/agent-attack/blob/HEAD/agent_attack/models/gemini.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"72cb0591b722709a","mcp_get_code":{"code_sha256":"72cb0591b722709a"}},{"arxiv_id":"2406.12793","paper":"/paper/chatglm-a-family-of-large-language-models","title":"ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools","date":"2024-06-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thudm/chatglm4","path":"inference/glm4v_api_request.py","file_url":"https://github.com/thudm/chatglm4/blob/HEAD/inference/glm4v_api_request.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c0b1c7a482944767","mcp_get_code":{"code_sha256":"c0b1c7a482944767"}},{"arxiv_id":"2406.11230","paper":"/paper/multimodal-needle-in-a-haystack-benchmarking","title":"Multimodal Needle in a Haystack: Benchmarking Long-Context Capability of Multimodal Large Language Models","date":"2024-06-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"wang-ml-lab/multimodal-needle-in-a-haystack","path":"utils.py","file_url":"https://github.com/wang-ml-lab/multimodal-needle-in-a-haystack/blob/HEAD/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"09468b7dfba57e92","mcp_get_code":{"code_sha256":"09468b7dfba57e92"}},{"arxiv_id":"2406.10900","paper":"/paper/autohallusion-automatic-generation-of","title":"AutoHallusion: Automatic Generation of Hallucination Benchmarks for Vision-Language Models","date":"2024-06-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2406.10638","paper":"/paper/seeing-clearly-answering-incorrectly-a","title":"Unveiling the Ignorance of MLLMs: Seeing Clearly, Answering Incorrectly","date":"2024-06-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"baai-dcai/multimodal-robustness-benchmark","path":"dataset/data_generation.py","file_url":"https://github.com/baai-dcai/multimodal-robustness-benchmark/blob/HEAD/dataset/data_generation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2406.09988","paper":"/paper/details-make-a-difference-object-state","title":"Details Make a Difference: Object State-Sensitive Neurorobotic Task Planning","date":"2024-06-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xiao-wen-sun/ossa","path":"utils.py","file_url":"https://github.com/xiao-wen-sun/ossa/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"1e90c2ddcf6921e2","mcp_get_code":{"code_sha256":"1e90c2ddcf6921e2"}},{"arxiv_id":"2406.09396","paper":"/paper/too-many-frames-not-all-useful-efficient","title":"Too Many Frames, Not All Useful: Efficient Strategies for Long-Form Video QA","date":"2024-06-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jongwoopark7978/LVNet","path":"fineKeyframeDetector.py","file_url":"https://github.com/jongwoopark7978/LVNet/blob/HEAD/fineKeyframeDetector.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2406.08702","paper":"/paper/vlind-bench-measuring-language-priors-in","title":"VLind-Bench: Measuring Language Priors in Large Vision-Language Models","date":"2024-06-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"klee972/vlind-bench","path":"eval/gpt4o_eval.py","file_url":"https://github.com/klee972/vlind-bench/blob/HEAD/eval/gpt4o_eval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2406.08656","paper":"/paper/tc-bench-benchmarking-temporal","title":"TC-Bench: Benchmarking Temporal Compositionality in Text-to-Video and Image-to-Video Generation","date":"2024-06-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"weixi-feng/tc-bench","path":"vlm_eval.py","file_url":"https://github.com/weixi-feng/tc-bench/blob/HEAD/vlm_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3d39217a41c77766","mcp_get_code":{"code_sha256":"3d39217a41c77766"}},{"arxiv_id":"2406.01014","paper":"/paper/mobile-agent-v2-mobile-device-operation","title":"Mobile-Agent-v2: Mobile Device Operation Assistant with Effective Navigation via Multi-Agent Collaboration","date":"2024-06-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2405.14702","paper":"/paper/g3-an-effective-and-adaptive-framework-for","title":"G3: An Effective and Adaptive Framework for Worldwide Geolocalization Using Large Multi-Modality Models","date":"2024-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"applied-machine-learning-lab/g3","path":"llm_predict.py","file_url":"https://github.com/applied-machine-learning-lab/g3/blob/HEAD/llm_predict.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2405.14156","paper":"/paper/unveiling-the-tapestry-of-consistency-in","title":"Unveiling the Tapestry of Consistency in Large Vision-Language Models","date":"2024-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"foundation-multimodal-models/conbench","path":"eval/GPT-4o.py","file_url":"https://github.com/foundation-multimodal-models/conbench/blob/HEAD/eval/GPT-4o.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2404.07972","paper":"/paper/osworld-benchmarking-multimodal-agents-for","title":"OSWorld: Benchmarking Multimodal Agents for Open-Ended Tasks in Real Computer Environments","date":"2024-04-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xlang-ai/OSWorld","path":"mm_agents/agent.py","file_url":"https://github.com/xlang-ai/OSWorld/blob/HEAD/mm_agents/agent.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"78ee6b46cebd99ee","mcp_get_code":{"code_sha256":"78ee6b46cebd99ee"}},{"arxiv_id":"2404.01197","paper":"/paper/getting-it-right-improving-spatial","title":"Getting it Right: Improving Spatial Consistency in Text-to-Image Models","date":"2024-04-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"SPRIGHT-T2I/SPRIGHT","path":"eval/gpt4/eval_with_gpt4.py","file_url":"https://github.com/SPRIGHT-T2I/SPRIGHT/blob/HEAD/eval/gpt4/eval_with_gpt4.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"57bfaec2f02fdac2","mcp_get_code":{"code_sha256":"57bfaec2f02fdac2"}},{"arxiv_id":"2403.18346","paper":"/paper/quantifying-and-mitigating-unimodal-biases-in","title":"Quantifying and Mitigating Unimodal Biases in Multimodal Large Language Models: A Causal Perspective","date":"2024-03-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"opencausalab/more","path":"utils/utils.py","file_url":"https://github.com/opencausalab/more/blob/HEAD/utils/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"5366edd7a38c7f65","mcp_get_code":{"code_sha256":"5366edd7a38c7f65"}},{"arxiv_id":"2403.17465","paper":"/paper/lare-2-latent-reconstruction-error-based","title":"LaRE^2: Latent Reconstruction Error Based Method for Diffusion-Generated Image Detection","date":"2024-03-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"luo3300612/lare","path":"model.py","file_url":"https://github.com/luo3300612/lare/blob/HEAD/model.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b59cbe843294ab85","mcp_get_code":{"code_sha256":"b59cbe843294ab85"}},{"arxiv_id":"2403.11083","paper":"/paper/customizing-visual-language-foundation-models","title":"Customizing Visual-Language Foundation Models for Multi-modal Anomaly Detection and Reasoning","date":"2024-03-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2402.18695","paper":"/paper/grounding-language-models-for-visual-entity","title":"Grounding Language Models for Visual Entity Recognition","date":"2024-02-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mrzilinxiao/autover","path":"baselines/submit_gpt4v.py","file_url":"https://github.com/mrzilinxiao/autover/blob/HEAD/baselines/submit_gpt4v.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2402.13254","paper":"/paper/countercurate-enhancing-physical-and-semantic","title":"CounterCurate: Enhancing Physical and Semantic Visio-Linguistic Compositional Reasoning via Counterfactual Examples","date":"2024-02-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hansolo9682/countercurate","path":"datasets/gpt4v_prompt_gen.py","file_url":"https://github.com/hansolo9682/countercurate/blob/HEAD/datasets/gpt4v_prompt_gen.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2402.12058","paper":"/paper/scaffolding-coordinates-to-promote-vision","title":"Scaffolding Coordinates to Promote Vision-Language Coordination in Large Multi-Modal Models","date":"2024-02-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"leixy20/scaffold","path":"call-api.py","file_url":"https://github.com/leixy20/scaffold/blob/HEAD/call-api.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"181b183e1782d76e","mcp_get_code":{"code_sha256":"181b183e1782d76e"}},{"arxiv_id":"2402.11453","paper":"/paper/matplotagent-method-and-evaluation-for-llm","title":"MatPlotAgent: Method and Evaluation for LLM-Based Agentic Scientific Data Visualization","date":"2024-02-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thunlp/matplotagent","path":"evaluation/api_eval.py","file_url":"https://github.com/thunlp/matplotagent/blob/HEAD/evaluation/api_eval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2402.04178","paper":"/paper/shield-an-evaluation-benchmark-for-face","title":"SHIELD : An Evaluation Benchmark for Face Spoofing and Forgery Detection with Multimodal Large Language Models","date":"2024-02-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"laiyingxin2/shield","path":"Script/GPT-4V_single_image.py","file_url":"https://github.com/laiyingxin2/shield/blob/HEAD/Script/GPT-4V_single_image.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"de8e5401032cdf3a","mcp_get_code":{"code_sha256":"de8e5401032cdf3a"}},{"arxiv_id":"2402.02207","paper":"/paper/safety-fine-tuning-at-almost-no-cost-a","title":"Safety Fine-Tuning at (Almost) No Cost: A Baseline for Vision Large Language Models","date":"2024-02-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ys-zong/vlguard","path":"gpt4_evaluator.py","file_url":"https://github.com/ys-zong/vlguard/blob/HEAD/gpt4_evaluator.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"d18629042893a7ec","mcp_get_code":{"code_sha256":"d18629042893a7ec"}},{"arxiv_id":"2402.00626","paper":"/paper/vision-llms-can-fool-themselves-with-self","title":"Vision-LLMs Can Fool Themselves with Self-Generated Typographic Attacks","date":"2024-02-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mqraitem/self-gen-typo-attack","path":"utils_models/utils_gpt4.py","file_url":"https://github.com/mqraitem/self-gen-typo-attack/blob/HEAD/utils_models/utils_gpt4.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2401.17600","paper":"/paper/good-at-captioning-bad-at-counting","title":"Good at captioning, bad at counting: Benchmarking GPT-4V on Earth observation data","date":"2024-01-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2401.04092","paper":"/paper/gpt-4v-ision-is-a-human-aligned-evaluator-for","title":"GPT-4V(ision) is a Human-Aligned Evaluator for Text-to-3D Generation","date":"2024-01-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2401.01614","paper":"/paper/gpt-4v-ision-is-a-generalist-web-agent-if","title":"GPT-4V(ision) is a Generalist Web Agent, if Grounded","date":"2024-01-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"osu-nlp-group/seeact","path":"seeact_package/seeact/demo_utils/inference_engine.py","file_url":"https://github.com/osu-nlp-group/seeact/blob/HEAD/seeact_package/seeact/demo_utils/inference_engine.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2312.14867","paper":"/paper/viescore-towards-explainable-metrics-for","title":"VIEScore: Towards Explainable Metrics for Conditional Image Synthesis Evaluation","date":"2023-12-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"TIGER-AI-Lab/VIEScore","path":"viescore/mllm_tools/openai.py","file_url":"https://github.com/TIGER-AI-Lab/VIEScore/blob/HEAD/viescore/mllm_tools/openai.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2312.08914","paper":"/paper/cogagent-a-visual-language-model-for-gui","title":"CogAgent: A Visual Language Model for GUI Agents","date":"2023-12-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THUDM/CogAgent","path":"app/client.py","file_url":"https://github.com/THUDM/CogAgent/blob/HEAD/app/client.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"1f0bc3f4321973e5","mcp_get_code":{"code_sha256":"1f0bc3f4321973e5"}},{"arxiv_id":"2311.15732","paper":"/paper/gpt4vis-what-can-gpt-4-do-for-zero-shot","title":"GPT4Vis: What Can GPT-4 Do for Zero-shot Visual Recognition?","date":"2023-11-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"whwu95/GPT4Vis","path":"GPT4V_ZS.py","file_url":"https://github.com/whwu95/GPT4Vis/blob/HEAD/GPT4V_ZS.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2310.01218","paper":"/paper/making-llama-see-and-draw-with-seed-tokenizer","title":"Making LLaMA SEE and Draw with SEED Tokenizer","date":"2023-10-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ailab-cvc/seed","path":"gradio_demo/seed_llama_flask.py","file_url":"https://github.com/ailab-cvc/seed/blob/HEAD/gradio_demo/seed_llama_flask.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"65dbe48364c80cba","mcp_get_code":{"code_sha256":"65dbe48364c80cba"}},{"arxiv_id":"2310.01218","paper":"/paper/making-llama-see-and-draw-with-seed-tokenizer","title":"Making LLaMA SEE and Draw with SEED Tokenizer","date":"2023-10-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ailab-cvc/seed","path":"gradio_demo/seed_llama_gradio.py","file_url":"https://github.com/ailab-cvc/seed/blob/HEAD/gradio_demo/seed_llama_gradio.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"d25aa46a7e2f707f","mcp_get_code":{"code_sha256":"d25aa46a7e2f707f"}},{"arxiv_id":"2309.16211","paper":"/paper/vdc-versatile-data-cleanser-for-detecting","title":"VDC: Versatile Data Cleanser based on Visual-Linguistic Inconsistency by Multimodal Large Language Models","date":"2023-09-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zihao-ai/vdc","path":"vdc/cleanser.py","file_url":"https://github.com/zihao-ai/vdc/blob/HEAD/vdc/cleanser.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b5fa7736e0dfb6df","mcp_get_code":{"code_sha256":"b5fa7736e0dfb6df"}},{"arxiv_id":"2309.05950","paper":"/paper/language-models-as-black-box-optimizers-for","title":"Language Models as Black-Box Optimizers for Vision-Language Models","date":"2023-09-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shihongl1998/llm-as-a-blackbox-optimizer","path":"auto_prompt_image_optimization.py","file_url":"https://github.com/shihongl1998/llm-as-a-blackbox-optimizer/blob/HEAD/auto_prompt_image_optimization.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2110.13601","paper":"/paper/dag-card-is-the-new-model-card","title":"DAG Card is the new Model Card","date":"2021-10-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jacopotagliabue/recs-at-resonable-scale","path":"src/app_utils.py","file_url":"https://github.com/jacopotagliabue/recs-at-resonable-scale/blob/HEAD/src/app_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9e8d4792ee55f8f8","mcp_get_code":{"code_sha256":"9e8d4792ee55f8f8"}},{"arxiv_id":"2107.07651","paper":"/paper/align-before-fuse-vision-and-language","title":"Align before Fuse: Vision and Language Representation Learning with Momentum Distillation","date":"2021-07-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"facebookresearch/multimodal","path":"examples/albef/finetune_retrieval.py","file_url":"https://github.com/facebookresearch/multimodal/blob/HEAD/examples/albef/finetune_retrieval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"code_sha256_prefix":"0ebf6e3d4677547a","mcp_get_code":{"code_sha256":"0ebf6e3d4677547a"}},{"arxiv_id":"2010.11929","paper":"/paper/an-image-is-worth-16x16-words-transformers-1","title":"An Image is Worth 16x16 Words: Transformers for Image Recognition at Scale","date":"2020-10-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lukas-blecher/LaTeX-OCR","path":"pix2tex/api/streamlit.py","file_url":"https://github.com/lukas-blecher/LaTeX-OCR/blob/HEAD/pix2tex/api/streamlit.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b2cb3afa961b359c","mcp_get_code":{"code_sha256":"b2cb3afa961b359c"}},{"arxiv_id":"openreview_lbaBsu0CaY","paper":null,"title":"arXiv:openreview_lbaBsu0CaY","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"Tunanzzz/Meerkat-VL","path":"eval/cal_result_by_gpt.py","file_url":"https://github.com/Tunanzzz/Meerkat-VL/blob/HEAD/eval/cal_result_by_gpt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"54b054cabd98d878","mcp_get_code":{"code_sha256":"54b054cabd98d878"}},{"arxiv_id":"ijcai2025_1122","paper":null,"title":"arXiv:ijcai2025_1122","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"mao-code/SmartSpatial","path":"SmartSpatialEval/utils.py","file_url":"https://github.com/mao-code/SmartSpatial/blob/HEAD/SmartSpatialEval/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"27705a4d9e84fccc","mcp_get_code":{"code_sha256":"27705a4d9e84fccc"}},{"arxiv_id":"Zhang_Critic-V_VLM_Critics_Help_Catch_VLM_Errors_in_Multimodal_Reasoning_CVPR_2025_paper","paper":null,"title":"arXiv:Zhang_Critic-V_VLM_Critics_Help_Catch_VLM_Errors_in_Multimodal_Reasoning_CVPR_2025_paper","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"kyrieLei/Critic-V","path":"data_utils/utils/get_reponse.py","file_url":"https://github.com/kyrieLei/Critic-V/blob/HEAD/data_utils/utils/get_reponse.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"Cao_MAPLM_A_Real-World_Large-Scale_Vision-Language_Benchmark_for_Map_and_Traffic_CVPR_2024_paper","paper":null,"title":"arXiv:Cao_MAPLM_A_Real-World_Large-Scale_Vision-Language_Benchmark_for_Map_and_Traffic_CVPR_2024_paper","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"LLVM-AD/MAPLM","path":"baseline/gpt4.py","file_url":"https://github.com/LLVM-AD/MAPLM/blob/HEAD/baseline/gpt4.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2025.naacl-long.166","paper":null,"title":"arXiv:2025.naacl-long.166","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"Wang-ML-Lab/multimodal-needle-in-a-haystack","path":"utils.py","file_url":"https://github.com/Wang-ML-Lab/multimodal-needle-in-a-haystack/blob/HEAD/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"09468b7dfba57e92","mcp_get_code":{"code_sha256":"09468b7dfba57e92"}},{"arxiv_id":"2025.findings-acl.979","paper":null,"title":"arXiv:2025.findings-acl.979","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"Mrshenshen/FRUIT","path":"models/gpt/video_caption.py","file_url":"https://github.com/Mrshenshen/FRUIT/blob/HEAD/models/gpt/video_caption.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2025.acl-long.663","paper":null,"title":"arXiv:2025.acl-long.663","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"BlueZeros/ReflecTool","path":"reflectool/models/openai_model.py","file_url":"https://github.com/BlueZeros/ReflecTool/blob/HEAD/reflectool/models/openai_model.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f41cb1a19b154297","mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"arxiv_id":"2024.findings-emnlp.960","paper":null,"title":"arXiv:2024.findings-emnlp.960","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"OpenCausaLab/MORE","path":"utils/utils.py","file_url":"https://github.com/OpenCausaLab/MORE/blob/HEAD/utils/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"5366edd7a38c7f65","mcp_get_code":{"code_sha256":"5366edd7a38c7f65"}}]}