{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/get-prompt","entry":"get_prompt","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":99,"n_papers_ran":61,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":98,"n_samples_ran":56,"n_samples_fingerprinted":18,"n_places":106,"n_places_pointer_only":43,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":22,"ran_fixture":1,"ran":33,"unverified":42},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2608.24952","paper":"/paper/arxiv-2608-24952","title":"The Dialect Tax: Dialectal Biases Persist throughout the Language Modeling Pipeline","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"socialnlp/dialecttax","path":"src/dialecttax/prompts.py","file_url":"https://github.com/socialnlp/dialecttax/blob/HEAD/src/dialecttax/prompts.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"efdb63cf50def444","mcp_get_code":{"code_sha256":"efdb63cf50def444"}},{"arxiv_id":"2606.00869","paper":"/paper/arxiv-2606-00869","title":"Enhancing LLM Metacognition via Cognitive Pairwise Training","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"Tsinghua-dhy/CPT","path":"eval/utils/judge_math_answer_gpt.py","file_url":"https://github.com/Tsinghua-dhy/CPT/blob/HEAD/eval/utils/judge_math_answer_gpt.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"04002d844af70af7","mcp_get_code":{"code_sha256":"04002d844af70af7"}},{"arxiv_id":"2606.00869","paper":"/paper/arxiv-2606-00869","title":"Enhancing LLM Metacognition via Cognitive Pairwise Training","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"Tsinghua-dhy/CPT","path":"eval/utils/math_equal.py","file_url":"https://github.com/Tsinghua-dhy/CPT/blob/HEAD/eval/utils/math_equal.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"a8eed4c1f2f2831b","mcp_get_code":{"code_sha256":"a8eed4c1f2f2831b"}},{"arxiv_id":"2605.12813","paper":"/paper/arxiv-2605-12813","title":"REALISTA: Realistic Latent Adversarial Attacks that Elicit LLM Hallucinations","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"Buyun-Liang/REALISTA","path":"src/realista.py","file_url":"https://github.com/Buyun-Liang/REALISTA/blob/HEAD/src/realista.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4c2b9df401898003","mcp_get_code":{"code_sha256":"4c2b9df401898003"}},{"arxiv_id":"2605.10076","paper":"/paper/arxiv-2605-10076","title":"A Stability Benchmark of Generative Regularizers for Inverse Problems","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"alexdenker/GenRegBench","path":"main_flow.py","file_url":"https://github.com/alexdenker/GenRegBench/blob/HEAD/main_flow.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"cab1fb2b4d059325","mcp_get_code":{"code_sha256":"cab1fb2b4d059325"}},{"arxiv_id":"2604.22758","paper":"/paper/arxiv-2604-22758","title":"REDPARROT: Accelerating NL-to-DSL for Business Analytics via Query Semantic Caching","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"TommyIsNotHere/RedParrot","path":"hybrid_rewrite/prompt.py","file_url":"https://github.com/TommyIsNotHere/RedParrot/blob/HEAD/hybrid_rewrite/prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1bd873161388cd83","mcp_get_code":{"code_sha256":"1bd873161388cd83"}},{"arxiv_id":"2604.20051","paper":"/paper/arxiv-2604-20051","title":"Bootstrapping Post-training Signals for Open-ended Tasks via Rubric-based Self-play on Pre-training Text","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"HCY123902/POP","path":"synthesis/sample.py","file_url":"https://github.com/HCY123902/POP/blob/HEAD/synthesis/sample.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"fb0f7009b88df9e4","mcp_get_code":{"code_sha256":"fb0f7009b88df9e4"}},{"arxiv_id":"2604.16665","paper":"/paper/arxiv-2604-16665","title":"CBRS: Cognitive Blood Request System with Bilingual Dataset and Dual-Layer Filtering for Multi-Platform Social Streams","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"aaniksahaa/CBRS","path":"binary-classifier/dual-layer-filtering/eval-v1.py","file_url":"https://github.com/aaniksahaa/CBRS/blob/HEAD/binary-classifier/dual-layer-filtering/eval-v1.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"62cf1ea9a6a67f98","mcp_get_code":{"code_sha256":"62cf1ea9a6a67f98"}},{"arxiv_id":"2603.10624","paper":"/paper/arxiv-2603-10624","title":"Reinforcement Learning with Conditional Expectation Reward","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"changyi7231/CER","path":"recipe/cer/src/data_preparation.py","file_url":"https://github.com/changyi7231/CER/blob/HEAD/recipe/cer/src/data_preparation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"a6f637988f82514b","mcp_get_code":{"code_sha256":"a6f637988f82514b"}},{"arxiv_id":"2603.01778","paper":"/paper/arxiv-2603-01778","title":"LLM-as-an-Annotator: Training Lightweight Models with LLM-Annotated Examples for Aspect Sentiment Tuple Prediction","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"NilsHellwig/LA-ABSA","path":"01_create_annotated_examples/03_llm_eda_few_shot_augmenter.py","file_url":"https://github.com/NilsHellwig/LA-ABSA/blob/HEAD/01_create_annotated_examples/03_llm_eda_few_shot_augmenter.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"eb03a5076b44c920","mcp_get_code":{"code_sha256":"eb03a5076b44c920"}},{"arxiv_id":"2602.07892","paper":"/paper/arxiv-2602-07892","title":"Safety Alignment as Continual Learning: Mitigating the Alignment Tax via Orthogonal Gradient Projection","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"SunGL001/OGPSA","path":"eval/AlpacaEval_pre.py","file_url":"https://github.com/SunGL001/OGPSA/blob/HEAD/eval/AlpacaEval_pre.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"fa528f9eb5d2a2af","mcp_get_code":{"code_sha256":"fa528f9eb5d2a2af"}},{"arxiv_id":"2602.05134","paper":"/paper/arxiv-2602-05134","title":"SEMPIPES -Optimizable Semantic Data Operators for Tabular Machine Learning Pipelines","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"noahho/CAAFE","path":"caafe/caafe.py","file_url":"https://github.com/noahho/CAAFE/blob/HEAD/caafe/caafe.py","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1288c06136735dd7","mcp_get_code":{"code_sha256":"1288c06136735dd7"}},{"arxiv_id":"2601.22925","paper":"/paper/arxiv-2601-22925","title":"BEAR: Towards Beam-Search-Aware Optimization for Recommendation with Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"Tiny-Snow/BEAR-SIGIR-2026","path":"utils.py","file_url":"https://github.com/Tiny-Snow/BEAR-SIGIR-2026/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"369f30df2ac7e4ca","mcp_get_code":{"code_sha256":"369f30df2ac7e4ca"}},{"arxiv_id":"2601.22162","paper":"/paper/arxiv-2601-22162","title":"UniFinEval: Towards Unified Evaluation of Financial Multimodal Models across Text, Images and Videos","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"aifinlab/UniFinEval","path":"evaluate_py/prompts.py","file_url":"https://github.com/aifinlab/UniFinEval/blob/HEAD/evaluate_py/prompts.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"eb48ceb1095ac012","mcp_get_code":{"code_sha256":"eb48ceb1095ac012"}},{"arxiv_id":"2510.17671","paper":"/paper/arxiv-2510-17671","title":"LILO: Bayesian Optimization with Natural Language Feedback","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"facebookresearch/lilo","path":"lilo/human_feedback_simulator.py","file_url":"https://github.com/facebookresearch/lilo/blob/HEAD/lilo/human_feedback_simulator.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"af08af9d44db5a5c","mcp_get_code":{"code_sha256":"af08af9d44db5a5c"}},{"arxiv_id":"2510.04398","paper":"/paper/arxiv-2510-04398","title":"SECA: Semantically Equivalent and Coherent Attacks for Eliciting LLM Hallucinations","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"Buyun-Liang/SECA","path":"src/seca.py","file_url":"https://github.com/Buyun-Liang/SECA/blob/HEAD/src/seca.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"cf0a660abde18864","mcp_get_code":{"code_sha256":"cf0a660abde18864"}},{"arxiv_id":"2508.00890","paper":"/paper/arxiv-2508-00890","title":"AgentTTS: Large Language Model Agent for Test-time Compute-optimal Scaling Strategy in Complex Tasks","date":null,"month_inferred_from_arxiv_id":"2025-08","title_source":"syntology","repo":"FairyFali/AgentTTS","path":"code_others/archon_taskbench.py","file_url":"https://github.com/FairyFali/AgentTTS/blob/HEAD/code_others/archon_taskbench.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"043fc4b1c4483d0c","mcp_get_code":{"code_sha256":"043fc4b1c4483d0c"}},{"arxiv_id":"2507.10302","paper":null,"title":"arXiv:2507.10302","date":null,"month_inferred_from_arxiv_id":"2025-07","title_source":null,"repo":"ZJHTerry18/DisCo","path":"evaluation/eval_egoschema.py","file_url":"https://github.com/ZJHTerry18/DisCo/blob/HEAD/evaluation/eval_egoschema.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"7321ecad6597e9ff","mcp_get_code":{"code_sha256":"7321ecad6597e9ff"}},{"arxiv_id":"2506.08607","paper":"/paper/sample-efficient-demonstration-selection-for","title":"Sample Efficient Demonstration Selection for In-Context Learning","date":"2025-06-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kiranpurohit/case","path":"Code/LLM_experiments/CASE_Gsm8K_selection.py","file_url":"https://github.com/kiranpurohit/case/blob/HEAD/Code/LLM_experiments/CASE_Gsm8K_selection.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"f4cfbae1ebeb837e","mcp_get_code":{"code_sha256":"f4cfbae1ebeb837e"}},{"arxiv_id":"2505.24875","paper":"/paper/reasongen-r1-cot-for-autoregressive-image","title":"ReasonGen-R1: CoT for Autoregressive Image generation models through SFT and RL","date":"2025-05-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Franklin-Zhang0/ReasonGen-R1","path":"benchmark/generate_inference_dpg.py","file_url":"https://github.com/Franklin-Zhang0/ReasonGen-R1/blob/HEAD/benchmark/generate_inference_dpg.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"0b420baae60fbb33","mcp_get_code":{"code_sha256":"0b420baae60fbb33"}},{"arxiv_id":"2505.23368","paper":"/paper/threading-the-needle-reweaving-chain-of","title":"Threading the Needle: Reweaving Chain-of-Thought Reasoning to Explain Human Label Variation","date":"2025-05-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mainlp/CoT2EL","path":"Pipeline/generator.py","file_url":"https://github.com/mainlp/CoT2EL/blob/HEAD/Pipeline/generator.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"556513f9f88952ef","mcp_get_code":{"code_sha256":"556513f9f88952ef"}},{"arxiv_id":"2505.14362","paper":"/paper/deepeyes-incentivizing-thinking-with-images","title":"DeepEyes: Incentivizing \"Thinking with Images\" via Reinforcement Learning","date":"2025-05-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"visual-agent/deepeyes","path":"eval/judge_result.py","file_url":"https://github.com/visual-agent/deepeyes/blob/HEAD/eval/judge_result.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"1b4e5e1a7c7a8eee","mcp_get_code":{"code_sha256":"1b4e5e1a7c7a8eee"}},{"arxiv_id":"2505.02123","paper":"/paper/driveagent-multi-agent-structured-reasoning","title":"DriveAgent: Multi-Agent Structured Reasoning with LLM and Multimodal Sensor Fusion for Autonomous Driving","date":null,"month_inferred_from_arxiv_id":"2025-05","title_source":"archive","repo":"paparare/driveagent","path":"enviroment.py","file_url":"https://github.com/paparare/driveagent/blob/HEAD/enviroment.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8dc3ad180e974ab6","mcp_get_code":{"code_sha256":"8dc3ad180e974ab6"}},{"arxiv_id":"2504.09772","paper":"/paper/two-heads-are-better-than-one-test-time-1","title":"Two Heads are Better Than One: Test-time Scaling of Multi-agent Collaborative Reasoning","date":"2025-04-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jincan333/MAS-TTS","path":"evaluate_dataset.py","file_url":"https://github.com/jincan333/MAS-TTS/blob/HEAD/evaluate_dataset.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"3cbafc44dba51a7d","mcp_get_code":{"code_sha256":"3cbafc44dba51a7d"}},{"arxiv_id":"2504.02953","paper":"/paper/cultural-learning-based-culture-adaptation-of","title":"Cultural Learning-Based Culture Adaptation of Language Models","date":"2025-04-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ukplab/arxiv2025-clca","path":"CLCA/llm_roleplay/common/model_tuning.py","file_url":"https://github.com/ukplab/arxiv2025-clca/blob/HEAD/CLCA/llm_roleplay/common/model_tuning.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"ebd71a3edf5b5421","mcp_get_code":{"code_sha256":"ebd71a3edf5b5421"}},{"arxiv_id":"2503.21457","paper":"/paper/facebench-a-multi-view-multi-level-facial","title":"FaceBench: A Multi-View Multi-Level Facial Attribute VQA Dataset for Benchmarking Face Perception MLLMs","date":"2025-03-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"CVI-SZU/FaceBench","path":"evaluation/evaluation.py","file_url":"https://github.com/CVI-SZU/FaceBench/blob/HEAD/evaluation/evaluation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"7af521bbe681ec46","mcp_get_code":{"code_sha256":"7af521bbe681ec46"}},{"arxiv_id":"2503.17352","paper":"/paper/openvlthinker-an-early-exploration-to-complex","title":"OpenVLThinker: An Early Exploration to Complex Vision-Language Reasoning via Iterative Self-Improvement","date":"2025-03-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yihedeng9/openvlthinker","path":"v1/evaluation/verify_mathverse_gpt4.py","file_url":"https://github.com/yihedeng9/openvlthinker/blob/HEAD/v1/evaluation/verify_mathverse_gpt4.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"ccfc28a818189693","mcp_get_code":{"code_sha256":"ccfc28a818189693"}},{"arxiv_id":"2503.14681","paper":"/paper/dpimagebench-a-unified-benchmark-for","title":"DPImageBench: A Unified Benchmark for Differentially Private Image Synthesis","date":"2025-03-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"2019chengong/dpimagebench","path":"evaluation/evaluator.py","file_url":"https://github.com/2019chengong/dpimagebench/blob/HEAD/evaluation/evaluator.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"092ceca055502762","mcp_get_code":{"code_sha256":"092ceca055502762"}},{"arxiv_id":"2502.18418","paper":"/paper/rank1-test-time-compute-for-reranking-in","title":"Rank1: Test-Time Compute for Reranking in Information Retrieval","date":"2025-02-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"orionw/rank1","path":"prompts.py","file_url":"https://github.com/orionw/rank1/blob/HEAD/prompts.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b52aabdc316d82de","mcp_get_code":{"code_sha256":"b52aabdc316d82de"}},{"arxiv_id":"2502.18001","paper":"/paper/unveiling-the-key-factors-for-distilling","title":"Unveiling the Key Factors for Distilling Chain-of-Thought Reasoning","date":"2025-02-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"eit-nlp/distilling-cot-reasoning","path":"Evaluation/reasoning_eval/prompt_utils.py","file_url":"https://github.com/eit-nlp/distilling-cot-reasoning/blob/HEAD/Evaluation/reasoning_eval/prompt_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"9dcdd19fa07a9553","mcp_get_code":{"code_sha256":"9dcdd19fa07a9553"}},{"arxiv_id":"2502.09565","paper":"/paper/mdcrow-automating-molecular-dynamics","title":"MDCrow: Automating Molecular Dynamics Workflows with Large Language Models","date":"2025-02-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ur-whitelab/MDCrow","path":"notebooks/experiments/Robustness/robustness_prompts.py","file_url":"https://github.com/ur-whitelab/MDCrow/blob/HEAD/notebooks/experiments/Robustness/robustness_prompts.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5a0080520de90215","mcp_get_code":{"code_sha256":"5a0080520de90215"}},{"arxiv_id":"2502.04923","paper":"/paper/cached-multi-lora-composition-for-multi","title":"Cached Multi-Lora Composition for Multi-Concept Image Generation","date":"2025-02-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Yqcca/CMLoRA","path":"utils.py","file_url":"https://github.com/Yqcca/CMLoRA/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"bb2bba270e29d9ee","mcp_get_code":{"code_sha256":"bb2bba270e29d9ee"}},{"arxiv_id":"2502.00212","paper":"/paper/stp-self-play-llm-theorem-provers-with","title":"STP: Self-play LLM Theorem Provers with Iterative Conjecturing and Proving","date":"2025-01-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kfdong/STP","path":"RL/utils/model_utils.py","file_url":"https://github.com/kfdong/STP/blob/HEAD/RL/utils/model_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ba7c25429714f2b5","mcp_get_code":{"code_sha256":"ba7c25429714f2b5"}},{"arxiv_id":"2501.15087","paper":"/paper/patchrec-multi-grained-patching-for-efficient","title":"Multi-Grained Patch Training for Efficient LLM-based Recommendation","date":"2025-01-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ljy0ustc/patchrec","path":"utils.py","file_url":"https://github.com/ljy0ustc/patchrec/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7fcc27faf959d335","mcp_get_code":{"code_sha256":"7fcc27faf959d335"}},{"arxiv_id":"2412.20299","paper":"/paper/no-preference-left-behind-group","title":"No Preference Left Behind: Group Distributional Preference Optimization","date":"2024-12-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"BigBinnie/GDPO","path":"evaluate_BPC.py","file_url":"https://github.com/BigBinnie/GDPO/blob/HEAD/evaluate_BPC.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"d420ed106490686d","mcp_get_code":{"code_sha256":"d420ed106490686d"}},{"arxiv_id":"2410.22239","paper":"/paper/discern-decoding-systematic-errors-in-natural","title":"DISCERN: Decoding Systematic Errors in Natural Language for Text Classifiers","date":"2024-10-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rrmenon10/DISCERN","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"478e1775f2ce81ae","mcp_get_code":{"code_sha256":"478e1775f2ce81ae"}},{"arxiv_id":"2410.21759","paper":"/paper/intlora-integral-low-rank-adaptation-of","title":"IntLoRA: Integral Low-rank Adaptation of Quantized Diffusion Models","date":"2024-10-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"csguoh/IntLoRA","path":"evaluation.py","file_url":"https://github.com/csguoh/IntLoRA/blob/HEAD/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ed99491f78bd49e1","mcp_get_code":{"code_sha256":"ed99491f78bd49e1"}},{"arxiv_id":"2410.19054","paper":"/paper/infogent-an-agent-based-framework-for-web","title":"Infogent: An Agent-Based Framework for Web Information Aggregation","date":"2024-10-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gangiswag/infogent","path":"direct-api-driven/fanoutqa_answer.py","file_url":"https://github.com/gangiswag/infogent/blob/HEAD/direct-api-driven/fanoutqa_answer.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"05e25bd96ace7a5c","mcp_get_code":{"code_sha256":"05e25bd96ace7a5c"}},{"arxiv_id":"2410.13886","paper":"/paper/refusal-trained-llms-are-easily-jailbroken-as","title":"Refusal-Trained LLMs Are Easily Jailbroken As Browser Agents","date":"2024-10-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"scaleapi/browser-art","path":"src/agents/OpenDevin/agenthub/browsing_agent/browsing_agent.py","file_url":"https://github.com/scaleapi/browser-art/blob/HEAD/src/agents/OpenDevin/agenthub/browsing_agent/browsing_agent.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"dd9d65d0d8999bb0","mcp_get_code":{"code_sha256":"dd9d65d0d8999bb0"}},{"arxiv_id":"2410.12327","paper":"/paper/neuron-based-personality-trait-induction-in","title":"Neuron-based Personality Trait Induction in Large Language Models","date":"2024-10-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rucaibox/npti","path":"NPTI/code/gpt4_score.py","file_url":"https://github.com/rucaibox/npti/blob/HEAD/NPTI/code/gpt4_score.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6340f1429608e7df","mcp_get_code":{"code_sha256":"6340f1429608e7df"}},{"arxiv_id":"2410.10139","paper":"/paper/mmie-massive-multimodal-interleaved","title":"MMIE: Massive Multimodal Interleaved Comprehension Benchmark for Large Vision-Language Models","date":"2024-10-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Lillianwei-h/MMIE","path":"prompts.py","file_url":"https://github.com/Lillianwei-h/MMIE/blob/HEAD/prompts.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3007270c1b69f9a2","mcp_get_code":{"code_sha256":"3007270c1b69f9a2"}},{"arxiv_id":"2410.04612","paper":"/paper/regressing-the-relative-future-efficient","title":"Regressing the Relative Future: Efficient Policy Optimization for Multi-turn RLHF","date":"2024-10-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhaolingao/refuel","path":"setting_one/user_generator.py","file_url":"https://github.com/zhaolingao/refuel/blob/HEAD/setting_one/user_generator.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"fc0417f911166c83","mcp_get_code":{"code_sha256":"fc0417f911166c83"}},{"arxiv_id":"2410.04521","paper":"/paper/mc-cot-a-modular-collaborative-cot-framework","title":"MC-CoT: A Modular Collaborative CoT Framework for Zero-shot Medical-VQA with LLM and MLLM Integration","date":"2024-10-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thomaswei-cn/MC-CoT","path":"method/VisualOnly.py","file_url":"https://github.com/thomaswei-cn/MC-CoT/blob/HEAD/method/VisualOnly.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9c7f5d246b874c50","mcp_get_code":{"code_sha256":"9c7f5d246b874c50"}},{"arxiv_id":"2410.04139","paper":"/paper/from-reading-to-compressing-exploring-the","title":"From Reading to Compressing: Exploring the Multi-document Reader for Prompt Compression","date":"2024-10-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"eunseongc/r2c","path":"src_comp/compress_utils.py","file_url":"https://github.com/eunseongc/r2c/blob/HEAD/src_comp/compress_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"edc9d2c346eeeec0","mcp_get_code":{"code_sha256":"edc9d2c346eeeec0"}},{"arxiv_id":"2410.02062","paper":"/paper/tpp-llm-modeling-temporal-point-processes-by","title":"TPP-LLM: Modeling Temporal Point Processes by Efficiently Fine-Tuning Large Language Models","date":"2024-10-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zefang-liu/TPP-LLM","path":"src/tpp_llm/utils.py","file_url":"https://github.com/zefang-liu/TPP-LLM/blob/HEAD/src/tpp_llm/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dd21bdbaacebe685","mcp_get_code":{"code_sha256":"dd21bdbaacebe685"}},{"arxiv_id":"2409.17647","paper":"/paper/mecd-unlocking-multi-event-causal-discovery","title":"MECD: Unlocking Multi-Event Causal Discovery in Video Reasoning","date":"2024-09-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tychen-sjtu/mecd-benchmark","path":"mecd_vllm_fewshot/VideoChat2/multi_event.py","file_url":"https://github.com/tychen-sjtu/mecd-benchmark/blob/HEAD/mecd_vllm_fewshot/VideoChat2/multi_event.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"748d5a2586ca3178","mcp_get_code":{"code_sha256":"748d5a2586ca3178"}},{"arxiv_id":"2409.01055","paper":"/paper/follow-your-canvas-higher-resolution-video","title":"Follow-Your-Canvas: Higher-Resolution Video Outpainting with Extensive Content Generation","date":"2024-09-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mayuelala/followyourcanvas","path":"inference_outpainting-dir-with-prompt.py","file_url":"https://github.com/mayuelala/followyourcanvas/blob/HEAD/inference_outpainting-dir-with-prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"f67980fe15864660","mcp_get_code":{"code_sha256":"f67980fe15864660"}},{"arxiv_id":"2408.17443","paper":"/paper/bridging-episodes-and-semantics-a-novel","title":"HERMES: temporal-coHERent long-forM understanding with Episodes and Semantics","date":"2024-08-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"joslefaure/HERMES","path":"lavis/tasks/moviecore_eval_prompts.py","file_url":"https://github.com/joslefaure/HERMES/blob/HEAD/lavis/tasks/moviecore_eval_prompts.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"50228589040ec9ac","mcp_get_code":{"code_sha256":"50228589040ec9ac"}},{"arxiv_id":"2408.08435","paper":"/paper/automated-design-of-agentic-systems","title":"Automated Design of Agentic Systems","date":"2024-08-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shengranhu/adas","path":"_arc/arc_prompt.py","file_url":"https://github.com/shengranhu/adas/blob/HEAD/_arc/arc_prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e303009857c65536","mcp_get_code":{"code_sha256":"e303009857c65536"}},{"arxiv_id":"2408.08435","paper":"/paper/automated-design-of-agentic-systems","title":"Automated Design of Agentic Systems","date":"2024-08-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shengranhu/adas","path":"_drop/drop_prompt.py","file_url":"https://github.com/shengranhu/adas/blob/HEAD/_drop/drop_prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7c697edb0437e5f5","mcp_get_code":{"code_sha256":"7c697edb0437e5f5"}},{"arxiv_id":"2408.08435","paper":"/paper/automated-design-of-agentic-systems","title":"Automated Design of Agentic Systems","date":"2024-08-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shengranhu/adas","path":"_gpqa/gpqa_prompt.py","file_url":"https://github.com/shengranhu/adas/blob/HEAD/_gpqa/gpqa_prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"505fd0fb8ba0fca7","mcp_get_code":{"code_sha256":"505fd0fb8ba0fca7"}},{"arxiv_id":"2407.04108","paper":"/paper/future-events-as-backdoor-triggers","title":"Future Events as Backdoor Triggers: Investigating Temporal Vulnerabilities in LLMs","date":"2024-07-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sbp354/future_triggered_backdoors","path":"future_probing/headline_prompting/prompting_utils.py","file_url":"https://github.com/sbp354/future_triggered_backdoors/blob/HEAD/future_probing/headline_prompting/prompting_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2a75fb471a0321a6","mcp_get_code":{"code_sha256":"2a75fb471a0321a6"}},{"arxiv_id":"2407.00782","paper":"/paper/step-controlled-dpo-leveraging-stepwise-error","title":"Step-Controlled DPO: Leveraging Stepwise Error for Enhanced Mathematical Reasoning","date":"2024-06-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mathllm/Step-Controlled_DPO","path":"src/step_controled_dpo_lce/lce_solution_gen_different_negative_gsm8k.py","file_url":"https://github.com/mathllm/Step-Controlled_DPO/blob/HEAD/src/step_controled_dpo_lce/lce_solution_gen_different_negative_gsm8k.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"587a8be3d359400c","mcp_get_code":{"code_sha256":"587a8be3d359400c"}},{"arxiv_id":"2407.00782","paper":"/paper/step-controlled-dpo-leveraging-stepwise-error","title":"Step-Controlled DPO: Leveraging Stepwise Error for Enhanced Mathematical Reasoning","date":"2024-06-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mathllm/Step-Controlled_DPO","path":"src/step_controled_dpo_lce/lce_solution_gen_different_negative_gsm8k_divided.py","file_url":"https://github.com/mathllm/Step-Controlled_DPO/blob/HEAD/src/step_controled_dpo_lce/lce_solution_gen_different_negative_gsm8k_divided.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a681b908de92655d","mcp_get_code":{"code_sha256":"a681b908de92655d"}},{"arxiv_id":"2406.13375","paper":"/paper/aliice-evaluating-positional-fine-grained","title":"ALiiCE: Evaluating Positional Fine-grained Citation Generation","date":"2024-06-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ylXuu/ALiiCE","path":"generate.py","file_url":"https://github.com/ylXuu/ALiiCE/blob/HEAD/generate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a9aab662c8ac6543","mcp_get_code":{"code_sha256":"a9aab662c8ac6543"}},{"arxiv_id":"2406.11687","paper":"/paper/tokenization-falling-short-the-curse-of","title":"Tokenization Falling Short: On Subword Robustness in Large Language Models","date":"2024-06-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"floatai/tkeval","path":"src/utils.py","file_url":"https://github.com/floatai/tkeval/blob/HEAD/src/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3d2dcc489e259a67","mcp_get_code":{"code_sha256":"3d2dcc489e259a67"}},{"arxiv_id":"2406.07971","paper":"/paper/it-takes-two-on-the-seamlessness-between","title":"It Takes Two: On the Seamlessness between Reward and Policy Model in RLHF","date":"2024-06-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"taiminglu/seamless","path":"code/retrival/gpt/retrival.py","file_url":"https://github.com/taiminglu/seamless/blob/HEAD/code/retrival/gpt/retrival.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"845ccc17a73fe529","mcp_get_code":{"code_sha256":"845ccc17a73fe529"}},{"arxiv_id":"2406.07145","paper":"/paper/failures-are-fated-but-can-be-faded","title":"Failures Are Fated, But Can Be Faded: Characterizing and Mitigating Unwanted Behaviors in Large-Scale Vision and Language Models","date":"2024-06-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"somsagar07/FailureShiftRL","path":"Baselines/Generation/config.py","file_url":"https://github.com/somsagar07/FailureShiftRL/blob/HEAD/Baselines/Generation/config.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9f09587c18b2d305","mcp_get_code":{"code_sha256":"9f09587c18b2d305"}},{"arxiv_id":"2406.06040","paper":"/paper/vript-a-video-is-worth-thousands-of-words","title":"Vript: A Video Is Worth Thousands of Words","date":"2024-06-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mutonix/Vript","path":"vript-hard/models/videochat2/videochat2_vriptCAP.py","file_url":"https://github.com/mutonix/Vript/blob/HEAD/vript-hard/models/videochat2/videochat2_vriptCAP.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"748d5a2586ca3178","mcp_get_code":{"code_sha256":"748d5a2586ca3178"}},{"arxiv_id":"2406.04264","paper":"/paper/mlvu-a-comprehensive-benchmark-for-multi-task","title":"MLVU: Benchmarking Multi-task Long Video Understanding","date":"2024-06-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"748d5a2586ca3178","mcp_get_code":{"code_sha256":"748d5a2586ca3178"}},{"arxiv_id":"2405.20363","paper":"/paper/llmgeo-benchmarking-large-language-models-on","title":"LLMGeo: Benchmarking Large Language Models on Image Geolocation In-the-wild","date":"2024-05-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yeyimilk/llmgeo","path":"src/fschat.py","file_url":"https://github.com/yeyimilk/llmgeo/blob/HEAD/src/fschat.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b691056ddf790e1d","mcp_get_code":{"code_sha256":"b691056ddf790e1d"}},{"arxiv_id":"2405.18111","paper":"/paper/atm-adversarial-tuning-multi-agent-system","title":"ATM: Adversarial Tuning Multi-agent System Makes a Robust Retrieval-Augmented Generator","date":"2024-05-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"chuhac/ATM-RAG","path":"atm_train/attacker_build_data/prompting_for_rag.py","file_url":"https://github.com/chuhac/ATM-RAG/blob/HEAD/atm_train/attacker_build_data/prompting_for_rag.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"3df45990990f233a","mcp_get_code":{"code_sha256":"3df45990990f233a"}},{"arxiv_id":"2405.15984","paper":"/paper/evaluating-the-adversarial-robustness-of-1","title":"Evaluating and Safeguarding the Adversarial Robustness of Retrieval-Based In-Context Learning","date":"2024-05-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"simonucl/adv-retreival-icl","path":"src/quick_exp.py","file_url":"https://github.com/simonucl/adv-retreival-icl/blob/HEAD/src/quick_exp.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"8eddb6f291aed485","mcp_get_code":{"code_sha256":"8eddb6f291aed485"}},{"arxiv_id":"2405.15984","paper":"/paper/evaluating-the-adversarial-robustness-of-1","title":"Evaluating and Safeguarding the Adversarial Robustness of Retrieval-Based In-Context Learning","date":"2024-05-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"simonucl/adv-retreival-icl","path":"src/transfer_attack.py","file_url":"https://github.com/simonucl/adv-retreival-icl/blob/HEAD/src/transfer_attack.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"d47716bfc331903f","mcp_get_code":{"code_sha256":"d47716bfc331903f"}},{"arxiv_id":"2405.00218","paper":"/paper/constrained-decoding-for-secure-code","title":"Constrained Decoding for Secure Code Generation","date":"2024-04-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dynamite321/codeguardplus","path":"correctness_eval.py","file_url":"https://github.com/dynamite321/codeguardplus/blob/HEAD/correctness_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"718990946e149fe1","mcp_get_code":{"code_sha256":"718990946e149fe1"}},{"arxiv_id":"2404.05221","paper":"/paper/llm-reasoners-new-evaluation-library-and","title":"LLM Reasoners: New Evaluation, Library, and Analysis of Step-by-Step Reasoning with Large Language Models","date":"2024-04-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"maitrix-org/llm-reasoners","path":"examples/ReasonerAgent-Web/baseline/openhands_browsing_agent.py","file_url":"https://github.com/maitrix-org/llm-reasoners/blob/HEAD/examples/ReasonerAgent-Web/baseline/openhands_browsing_agent.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"d59734e9fc209dc9","mcp_get_code":{"code_sha256":"d59734e9fc209dc9"}},{"arxiv_id":"2404.02823","paper":"/paper/conifer-improving-complex-constrained","title":"Conifer: Improving Complex Constrained Instruction-Following Ability of Large Language Models","date":"2024-04-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"coniferlm/conifer","path":"utils.py","file_url":"https://github.com/coniferlm/conifer/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"82360028338c7371","mcp_get_code":{"code_sha256":"82360028338c7371"}},{"arxiv_id":"2403.20127","paper":"/paper/the-impact-of-prompts-on-zero-shot-detection","title":"The Impact of Prompts on Zero-Shot Detection of AI-Generated Text","date":"2024-03-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kaito25atugich/detector","path":"tmp/generate_estimation_prompts.py","file_url":"https://github.com/kaito25atugich/detector/blob/HEAD/tmp/generate_estimation_prompts.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9a9e58cbb8b7370b","mcp_get_code":{"code_sha256":"9a9e58cbb8b7370b"}},{"arxiv_id":"2403.11807","paper":"/paper/how-far-are-we-on-the-decision-making-of-llms","title":"How Far Are We on the Decision-Making of LLMs? Evaluating LLMs' Gaming Ability in Multi-Agent Environments","date":"2024-03-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"cuhk-arise/gamabench","path":"server.py","file_url":"https://github.com/cuhk-arise/gamabench/blob/HEAD/server.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"73776d01a9a3cb5a","mcp_get_code":{"code_sha256":"73776d01a9a3cb5a"}},{"arxiv_id":"2403.10854","paper":"/paper/a-comprehensive-study-of-multimodal-large","title":"A Comprehensive Study of Multimodal Large Language Models for Image Quality Assessment","date":"2024-03-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tianhewu/mllms-for-iqa","path":"prompts/gpt4v_prompt.py","file_url":"https://github.com/tianhewu/mllms-for-iqa/blob/HEAD/prompts/gpt4v_prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0ed0bc77cad17572","mcp_get_code":{"code_sha256":"0ed0bc77cad17572"}},{"arxiv_id":"2403.05266","paper":"/paper/erbench-an-entity-relationship-based","title":"ERBench: An Entity-Relationship based Automatically Verifiable Hallucination Benchmark for Large Language Models","date":"2024-03-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dilab-kaist/erbench","path":"binary/run_qa.py","file_url":"https://github.com/dilab-kaist/erbench/blob/HEAD/binary/run_qa.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"cd65beff1b6f34f4","mcp_get_code":{"code_sha256":"cd65beff1b6f34f4"}},{"arxiv_id":"2402.17453","paper":"/paper/ds-agent-automated-data-science-by-empowering","title":"DS-Agent: Automated Data Science by Empowering Large Language Models with Case-Based Reasoning","date":"2024-02-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"guosyjlu/DS-Agent","path":"deployment/prompt.py","file_url":"https://github.com/guosyjlu/DS-Agent/blob/HEAD/deployment/prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"15c1e503e527a5bf","mcp_get_code":{"code_sha256":"15c1e503e527a5bf"}},{"arxiv_id":"2402.16040","paper":"/paper/ehrnoteqa-a-patient-specific-question","title":"EHRNoteQA: An LLM Benchmark for Real-World Clinical Practice Using Discharge Summaries","date":"2024-02-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ji-youn-kim/ehrnoteqa","path":"src/evaluation/utils.py","file_url":"https://github.com/ji-youn-kim/ehrnoteqa/blob/HEAD/src/evaluation/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e80d57247e8567c0","mcp_get_code":{"code_sha256":"e80d57247e8567c0"}},{"arxiv_id":"2402.16040","paper":"/paper/ehrnoteqa-a-patient-specific-question","title":"EHRNoteQA: An LLM Benchmark for Real-World Clinical Practice Using Discharge Summaries","date":"2024-02-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ji-youn-kim/ehrnoteqa","path":"src/generation/utils.py","file_url":"https://github.com/ji-youn-kim/ehrnoteqa/blob/HEAD/src/generation/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"574a354a5ba5d4c1","mcp_get_code":{"code_sha256":"574a354a5ba5d4c1"}},{"arxiv_id":"2402.11457","paper":"/paper/when-do-llms-need-retrieval-augmentation","title":"When Do LLMs Need Retrieval Augmentation? Mitigating LLMs' Overconfidence Helps Retrieval Augmentation","date":"2024-02-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shiyunee/when-to-retrieve","path":"utils/prompt.py","file_url":"https://github.com/shiyunee/when-to-retrieve/blob/HEAD/utils/prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"277b2d82e854b570","mcp_get_code":{"code_sha256":"277b2d82e854b570"}},{"arxiv_id":"2402.09910","paper":"/paper/de-cop-detecting-copyrighted-content-in","title":"DE-COP: Detecting Copyrighted Content in Language Models Training Data","date":"2024-02-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"avduarte333/de-cop_method","path":"2_decop_hf.py","file_url":"https://github.com/avduarte333/de-cop_method/blob/HEAD/2_decop_hf.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"abeb23f8b2eef76f","mcp_get_code":{"code_sha256":"abeb23f8b2eef76f"}},{"arxiv_id":"2402.07876","paper":"/paper/policy-improvement-using-language-feedback","title":"Policy Improvement using Language Feedback Models","date":"2024-02-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"vzhong/language_feedback_models","path":"experiments/prompt_utils.py","file_url":"https://github.com/vzhong/language_feedback_models/blob/HEAD/experiments/prompt_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"fe361276102577d5","mcp_get_code":{"code_sha256":"fe361276102577d5"}},{"arxiv_id":"2401.13311","paper":"/paper/contextual-evaluating-context-sensitive-text","title":"ConTextual: Evaluating Context-Sensitive Text-Rich Visual Reasoning in Large Multimodal Models","date":"2024-01-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rohan598/contextual","path":"eval/response_eval_gpt4.py","file_url":"https://github.com/rohan598/contextual/blob/HEAD/eval/response_eval_gpt4.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"bf992be8225b649f","mcp_get_code":{"code_sha256":"bf992be8225b649f"}},{"arxiv_id":"2401.06081","paper":"/paper/improving-large-language-models-via-fine","title":"Improving Large Language Models via Fine-grained Reinforcement Learning with Minimum Editing Constraint","date":"2024-01-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rucaibox/rlmec","path":"evaluate/Math/prompt_utils.py","file_url":"https://github.com/rucaibox/rlmec/blob/HEAD/evaluate/Math/prompt_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ad26ca57156a1cf3","mcp_get_code":{"code_sha256":"ad26ca57156a1cf3"}},{"arxiv_id":"2401.03428","paper":"/paper/exploring-large-language-model-based","title":"Exploring Large Language Model based Intelligent Agents: Definitions, Methods, and Prospects","date":"2024-01-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"melih-unsal/demogpt","path":"demogpt/prompt.py","file_url":"https://github.com/melih-unsal/demogpt/blob/HEAD/demogpt/prompt.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"938213d47dbb3323","mcp_get_code":{"code_sha256":"938213d47dbb3323"}},{"arxiv_id":"2312.15918","paper":"/paper/supervised-knowledge-makes-large-language","title":"Supervised Knowledge Makes Large Language Models Better In-context Learners","date":"2023-12-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yanglinyi/supervised-knowledge-makes-large-language-models-better-in-context-learners","path":"NLG/model.py","file_url":"https://github.com/yanglinyi/supervised-knowledge-makes-large-language-models-better-in-context-learners/blob/HEAD/NLG/model.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0dd5a42ce6af7976","mcp_get_code":{"code_sha256":"0dd5a42ce6af7976"}},{"arxiv_id":"2312.15166","paper":"/paper/solar-10-7b-scaling-large-language-models","title":"SOLAR 10.7B: Scaling Large Language Models with Simple yet Effective Depth Up-Scaling","date":"2023-12-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jquesnelle/yarn","path":"eval/quality.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/quality.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8d8ff31fdde07297","mcp_get_code":{"code_sha256":"8d8ff31fdde07297"}},{"arxiv_id":"2312.06924","paper":"/paper/safety-alignment-in-nlp-tasks-weakly-aligned","title":"Safety Alignment in NLP Tasks: Weakly Aligned Summarization as an In-Context Attack","date":"2023-12-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"fyyfu/safetyalignnlp","path":"multi_prompt_generation.py","file_url":"https://github.com/fyyfu/safetyalignnlp/blob/HEAD/multi_prompt_generation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"873e7b82a3f92f78","mcp_get_code":{"code_sha256":"873e7b82a3f92f78"}},{"arxiv_id":"2311.17330","paper":"/paper/biomedical-knowledge-graph-enhanced-prompt","title":"Biomedical knowledge graph-optimized prompt generation for large language models","date":"2023-11-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"BaranziniLab/KG_RAG","path":"kg_rag/utility.py","file_url":"https://github.com/BaranziniLab/KG_RAG/blob/HEAD/kg_rag/utility.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"a7b09ec8d815a408","mcp_get_code":{"code_sha256":"a7b09ec8d815a408"}},{"arxiv_id":"2311.17005","paper":"/paper/mvbench-a-comprehensive-multi-modal-video","title":"MVBench: A Comprehensive Multi-modal Video Understanding Benchmark","date":"2023-11-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"opengvlab/ask-anything","path":"video_chat/conversation.py","file_url":"https://github.com/opengvlab/ask-anything/blob/HEAD/video_chat/conversation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"748d5a2586ca3178","mcp_get_code":{"code_sha256":"748d5a2586ca3178"}},{"arxiv_id":"2311.04901","paper":"/paper/genome-generative-neuro-symbolic-visual","title":"GENOME: GenerativE Neuro-symbOlic visual reasoning by growing and reusing ModulEs","date":"2023-11-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"umass-foundation-model/genome","path":"engine/prompt.py","file_url":"https://github.com/umass-foundation-model/genome/blob/HEAD/engine/prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"04269c1b860b83ab","mcp_get_code":{"code_sha256":"04269c1b860b83ab"}},{"arxiv_id":"2311.00738","paper":"/paper/can-foundation-models-watch-talk-and-guide","title":"Can Foundation Models Watch, Talk and Guide You Step by Step to Make a Cake?","date":"2023-11-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sled-group/watch-talk-and-guide","path":"src/util.py","file_url":"https://github.com/sled-group/watch-talk-and-guide/blob/HEAD/src/util.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"508259e833b104dd","mcp_get_code":{"code_sha256":"508259e833b104dd"}},{"arxiv_id":"2310.05492","paper":"/paper/how-abilities-in-large-language-models-are","title":"How Abilities in Large Language Models are Affected by Supervised Fine-tuning Data Composition","date":"2023-10-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"wangrongsheng/caregpt","path":"Gradio/model.py","file_url":"https://github.com/wangrongsheng/caregpt/blob/HEAD/Gradio/model.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"code_sha256_prefix":"0dd5a42ce6af7976","mcp_get_code":{"code_sha256":"0dd5a42ce6af7976"}},{"arxiv_id":"2310.04928","paper":"/paper/large-language-models-only-pass-primary","title":"Large Language Models Only Pass Primary School Exams in Indonesia: A Comprehensive Test on IndoMMLU","date":"2023-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"fajri91/indommlu","path":"evaluate.py","file_url":"https://github.com/fajri91/indommlu/blob/HEAD/evaluate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3565fb4a9a36a3c8","mcp_get_code":{"code_sha256":"3565fb4a9a36a3c8"}},{"arxiv_id":"2310.04793","paper":"/paper/fingpt-instruction-tuning-benchmark-for-open","title":"FinGPT: Instruction Tuning Benchmark for Open-Source Large Language Models in Financial Datasets","date":"2023-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"AI4Finance-Foundation/FinGPT","path":"fingpt/FinGPT_Benchmark/utils.py","file_url":"https://github.com/AI4Finance-Foundation/FinGPT/blob/HEAD/fingpt/FinGPT_Benchmark/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"130ab5832e26fbdd","mcp_get_code":{"code_sha256":"130ab5832e26fbdd"}},{"arxiv_id":"2310.03368","paper":"/paper/evaluating-hallucinations-in-chinese-large","title":"Evaluating Hallucinations in Chinese Large Language Models","date":"2023-10-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xiami2019/halluqa","path":"calculate_metrics.py","file_url":"https://github.com/xiami2019/halluqa/blob/HEAD/calculate_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"38fc10020d01d898","mcp_get_code":{"code_sha256":"38fc10020d01d898"}},{"arxiv_id":"2310.00647","paper":"/paper/beyond-task-performance-evaluating-and","title":"Beyond Task Performance: Evaluating and Reducing the Flaws of Large Multimodal Models with In-Context Learning","date":"2023-10-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mshukor/EvALign-ICL","path":"open_flamingo/eval/caption_utils.py","file_url":"https://github.com/mshukor/EvALign-ICL/blob/HEAD/open_flamingo/eval/caption_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"48d71ff25acc020d","mcp_get_code":{"code_sha256":"48d71ff25acc020d"}},{"arxiv_id":"2307.02390","paper":"/paper/causal-discovery-with-language-models-as","title":"Causal Discovery with Language Models as Imperfect Experts","date":"2023-07-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"StephLong614/Causal-disco","path":"utils/language_models.py","file_url":"https://github.com/StephLong614/Causal-disco/blob/HEAD/utils/language_models.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"835e7af85a1f4c96","mcp_get_code":{"code_sha256":"835e7af85a1f4c96"}},{"arxiv_id":"2305.18459","paper":"/paper/diffusion-model-is-an-effective-planner-and-1","title":"Diffusion Model is an Effective Planner and Data Synthesizer for Multi-Task Reinforcement Learning","date":"2023-05-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tinnerhrhe/MTDiff","path":"diffuser/models/temporal.py","file_url":"https://github.com/tinnerhrhe/MTDiff/blob/HEAD/diffuser/models/temporal.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5321d9d86ff019c1","mcp_get_code":{"code_sha256":"5321d9d86ff019c1"}},{"arxiv_id":"2305.18404","paper":"/paper/conformal-prediction-with-large-language","title":"Conformal Prediction with Large Language Models for Multi-Choice Question Answering","date":"2023-05-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bhaweshiitk/conformalllm","path":"conformal_llm_scores.py","file_url":"https://github.com/bhaweshiitk/conformalllm/blob/HEAD/conformal_llm_scores.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"62b2e921ddd66bd8","mcp_get_code":{"code_sha256":"62b2e921ddd66bd8"}},{"arxiv_id":"2305.14314","paper":"/paper/qlora-efficient-finetuning-of-quantized-llms","title":"QLoRA: Efficient Finetuning of Quantized LLMs","date":"2023-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"georgesung/llm_qlora","path":"inference.py","file_url":"https://github.com/georgesung/llm_qlora/blob/HEAD/inference.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8bdd290bac0f9c85","mcp_get_code":{"code_sha256":"8bdd290bac0f9c85"}},{"arxiv_id":"2305.13627","paper":"/paper/instruct-align-teaching-novel-languages-with","title":"InstructAlign: High-and-Low Resource Language Alignment via Continual Crosslingual Instruction Tuning","date":"2023-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hltchkust/instructalign","path":"nlu_prompt.py","file_url":"https://github.com/hltchkust/instructalign/blob/HEAD/nlu_prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"9239c0977078e7fe","mcp_get_code":{"code_sha256":"9239c0977078e7fe"}},{"arxiv_id":"2305.13300","paper":"/paper/adaptive-chameleon-or-stubborn-sloth","title":"Adaptive Chameleon or Stubborn Sloth: Revealing the Behavior of Large Language Models in Knowledge Conflicts","date":"2023-05-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"copenlu/context-utilisation-for-rag","path":"src/get_model_predictions/get_model_predictions.py","file_url":"https://github.com/copenlu/context-utilisation-for-rag/blob/HEAD/src/get_model_predictions/get_model_predictions.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"525dbdd48bc6f54c","mcp_get_code":{"code_sha256":"525dbdd48bc6f54c"}},{"arxiv_id":"2305.04091","paper":"/paper/plan-and-solve-prompting-improving-zero-shot","title":"Plan-and-Solve Prompting: Improving Zero-Shot Chain-of-Thought Reasoning by Large Language Models","date":"2023-05-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"AGI-Edgerunners/Plan-and-Solve-Prompting","path":"prompt.py","file_url":"https://github.com/AGI-Edgerunners/Plan-and-Solve-Prompting/blob/HEAD/prompt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c90b87435028c310","mcp_get_code":{"code_sha256":"c90b87435028c310"}},{"arxiv_id":"2305.03403","paper":"/paper/large-language-models-for-automated-data-1","title":"Large Language Models for Automated Data Science: Introducing CAAFE for Context-Aware Automated Feature Engineering","date":"2023-05-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"noahho/caafe","path":"caafe/caafe.py","file_url":"https://github.com/noahho/caafe/blob/HEAD/caafe/caafe.py","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"1288c06136735dd7","mcp_get_code":{"code_sha256":"1288c06136735dd7"}},{"arxiv_id":"2210.16536","paper":"/paper/differentiable-data-augmentation-for","title":"Differentiable Data Augmentation for Contrastive Sentence Representation Learning","date":"2022-10-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"TianduoWang/DiffAug","path":"diffaug/models.py","file_url":"https://github.com/TianduoWang/DiffAug/blob/HEAD/diffaug/models.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1694baf70cd8e233","mcp_get_code":{"code_sha256":"1694baf70cd8e233"}},{"arxiv_id":"2208.12242","paper":"/paper/dreambooth-fine-tuning-text-to-image","title":"DreamBooth: Fine Tuning Text-to-Image Diffusion Models for Subject-Driven Generation","date":"2022-08-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"ed99491f78bd49e1","mcp_get_code":{"code_sha256":"ed99491f78bd49e1"}},{"arxiv_id":"aaai_34547","paper":null,"title":"arXiv:aaai_34547","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"Aatrox103/CrAM","path":"utils/re_weighting.py","file_url":"https://github.com/Aatrox103/CrAM/blob/HEAD/utils/re_weighting.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1472b8313235999a","mcp_get_code":{"code_sha256":"1472b8313235999a"}},{"arxiv_id":"2024.findings-emnlp.86","paper":null,"title":"arXiv:2024.findings-emnlp.86","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"FloatAI/TKEval","path":"src/utils.py","file_url":"https://github.com/FloatAI/TKEval/blob/HEAD/src/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3d2dcc489e259a67","mcp_get_code":{"code_sha256":"3d2dcc489e259a67"}},{"arxiv_id":"2024.findings-acl.326","paper":null,"title":"arXiv:2024.findings-acl.326","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"dscc-admin-ch/statbot.swiss","path":"src/utility.py","file_url":"https://github.com/dscc-admin-ch/statbot.swiss/blob/HEAD/src/utility.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"code_sha256_prefix":"0dd5a42ce6af7976","mcp_get_code":{"code_sha256":"0dd5a42ce6af7976"}},{"arxiv_id":"2024.findings-acl.326","paper":null,"title":"arXiv:2024.findings-acl.326","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"dscc-admin-ch/statbot.swiss","path":"src/main-mixtral.py","file_url":"https://github.com/dscc-admin-ch/statbot.swiss/blob/HEAD/src/main-mixtral.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"code_sha256_prefix":"0f6f984fa9637589","mcp_get_code":{"code_sha256":"0f6f984fa9637589"}}]}