{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/normalize-answer","entry":"normalize_answer","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":304,"n_papers_ran":250,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":113,"n_samples_ran":64,"n_samples_fingerprinted":63,"n_places":323,"n_places_pointer_only":131,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":26,"ran_fixture":0,"ran":38,"unverified":49},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2609.12243","paper":"/paper/arxiv-2609-12243","title":"Chopthin-Consensus Power Sampling: A Diversity-Preserving Approach to LLM Decoding","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"MinooAhmadii/chopthin-consensus-power-sampling","path":"ccps/graders/math_normalize.py","file_url":"https://github.com/MinooAhmadii/chopthin-consensus-power-sampling/blob/HEAD/ccps/graders/math_normalize.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"d51cc295307937e6","mcp_get_code":{"code_sha256":"d51cc295307937e6"}},{"arxiv_id":"2609.01224","paper":"/paper/arxiv-2609-01224","title":"S$^2$Prune: Spatially Structured Visual Token Pruning for Multimodal Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"yuanyuanjia71-spec/S2Prune","path":"s2prune/vizwiz_eval.py","file_url":"https://github.com/yuanyuanjia71-spec/S2Prune/blob/HEAD/s2prune/vizwiz_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"3bea898e987fab1e","mcp_get_code":{"code_sha256":"3bea898e987fab1e"}},{"arxiv_id":"2609.00513","paper":"/paper/arxiv-2609-00513","title":"ISO-RAG: Isoperimetric Noise Control for Retrieval-Augmented Generation","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"ZaiizaiZHANG/ISO-RAG","path":"analyze_predictions.py","file_url":"https://github.com/ZaiizaiZHANG/ISO-RAG/blob/HEAD/analyze_predictions.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0d2dc85735d18403","mcp_get_code":{"code_sha256":"0d2dc85735d18403"}},{"arxiv_id":"2609.00513","paper":"/paper/arxiv-2609-00513","title":"ISO-RAG: Isoperimetric Noise Control for Retrieval-Augmented Generation","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"ZaiizaiZHANG/ISO-RAG","path":"analyze_retrieval_qa_transfer.py","file_url":"https://github.com/ZaiizaiZHANG/ISO-RAG/blob/HEAD/analyze_retrieval_qa_transfer.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1691191d75c4338c","mcp_get_code":{"code_sha256":"1691191d75c4338c"}},{"arxiv_id":"2608.19029","paper":"/paper/arxiv-2608-19029","title":"Adaptive Memory and Reflection Multi-Agent System for Medical Question Answering","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"mm-air/AMR-Agent","path":"evaluate.py","file_url":"https://github.com/mm-air/AMR-Agent/blob/HEAD/evaluate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9fc3dafdabb9a23a","mcp_get_code":{"code_sha256":"9fc3dafdabb9a23a"}},{"arxiv_id":"2608.13006","paper":"/paper/arxiv-2608-13006","title":"EviReform: Evidence-Guided Query Reformulation for Multi-Hop Graph Retrieval","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"XrazyMee/EviReform","path":"src/evireform/metrics.py","file_url":"https://github.com/XrazyMee/EviReform/blob/HEAD/src/evireform/metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"77114e88a1e17b9b","mcp_get_code":{"code_sha256":"77114e88a1e17b9b"}},{"arxiv_id":"2608.12875","paper":"/paper/arxiv-2608-12875","title":"The Embedder's Dilemma: LLMs Are Better, but at What Cost?","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"embeddings-benchmark/embedders-dilemma","path":"llm_judge/evaluators/llm_rag_evaluator.py","file_url":"https://github.com/embeddings-benchmark/embedders-dilemma/blob/HEAD/llm_judge/evaluators/llm_rag_evaluator.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"97012cf96aa79266","mcp_get_code":{"code_sha256":"97012cf96aa79266"}},{"arxiv_id":"2608.08557","paper":"/paper/arxiv-2608-08557","title":"OpenVisTool: An Open Recipe for Synthesizing Instructive Visual Tool-Use Trajectories","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"Changhao-Xiang/OpenVisTool","path":"distill/filter/evaluate_answer_rule.py","file_url":"https://github.com/Changhao-Xiang/OpenVisTool/blob/HEAD/distill/filter/evaluate_answer_rule.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"24e4999eff337610","mcp_get_code":{"code_sha256":"24e4999eff337610"}},{"arxiv_id":"2608.08024","paper":"/paper/arxiv-2608-08024","title":"Prompt Embedding Probes (PEP): Hallucination Detection in LLMs from Hidden States","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"zazamrykh/internal_probing","path":"src/dataset/squad_eval.py","file_url":"https://github.com/zazamrykh/internal_probing/blob/HEAD/src/dataset/squad_eval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2608.04355","paper":"/paper/arxiv-2608-04355","title":"The Calibration Floor: Format Repair Can Masquerade as Self-Correction at Small-to-Mid Scale","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"MBZUAI-CLeaR/IoE-Prompting","path":"run_Hotpot_IoE.py","file_url":"https://github.com/MBZUAI-CLeaR/IoE-Prompting/blob/HEAD/run_Hotpot_IoE.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b1a674018a8b1ccf","mcp_get_code":{"code_sha256":"b1a674018a8b1ccf"}},{"arxiv_id":"2608.04355","paper":"/paper/arxiv-2608-04355","title":"The Calibration Floor: Format Repair Can Masquerade as Self-Correction at Small-to-Mid Scale","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"MBZUAI-CLeaR/IoE-Prompting","path":"run_math_IoE.py","file_url":"https://github.com/MBZUAI-CLeaR/IoE-Prompting/blob/HEAD/run_math_IoE.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"5d6dae5b935c9916","mcp_get_code":{"code_sha256":"5d6dae5b935c9916"}},{"arxiv_id":"2608.03467","paper":"/paper/arxiv-2608-03467","title":"When Correct Solutions Repeat: Rarity-Aware Credit Redistribution for GRPO","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"CzZ12/When-Correct-Solutions-Repeat-Rarity-Aware-Credit-Redistribution-for-GRPO","path":"domain_config.py","file_url":"https://github.com/CzZ12/When-Correct-Solutions-Repeat-Rarity-Aware-Credit-Redistribution-for-GRPO/blob/HEAD/domain_config.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6151d15713a7392f","mcp_get_code":{"code_sha256":"6151d15713a7392f"}},{"arxiv_id":"2608.01904","paper":"/paper/arxiv-2608-01904","title":"CoEvoKG: Co-Evolving Knowledge Graphs with Self-Evolving Search Agents","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"lazzy1225/CoEvoKG","path":"coevokg/reward/score/coevokg_score.py","file_url":"https://github.com/lazzy1225/CoEvoKG/blob/HEAD/coevokg/reward/score/coevokg_score.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2607.20833","paper":"/paper/arxiv-2607-20833","title":"ReFact: Adaptive Fact Restatement for Compact and Faithful Chain-of-Thought Reasoning","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"NEUIR/REFACT","path":"verl/verl/utils/reward_score/evdience_reward.py","file_url":"https://github.com/NEUIR/REFACT/blob/HEAD/verl/verl/utils/reward_score/evdience_reward.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"445de90902805074","mcp_get_code":{"code_sha256":"445de90902805074"}},{"arxiv_id":"2607.16431","paper":"/paper/arxiv-2607-16431","title":"RIMS: Preference Optimization via Smoothed Multi-pair Aggregation for Small-Scale LLM Retrieval-Augmented Generation","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"tptrix29/RIMS","path":"baseline/baseline_instructrag.py","file_url":"https://github.com/tptrix29/RIMS/blob/HEAD/baseline/baseline_instructrag.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2606.08397","paper":"/paper/arxiv-2606-08397","title":"TrustMargin: Training-Free Arbitration between Parametric Memory and Retrieved Evidence in Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"mojixu/TrustMargin","path":"src/trustmargin.py","file_url":"https://github.com/mojixu/TrustMargin/blob/HEAD/src/trustmargin.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"3dc410d21f1c3741","mcp_get_code":{"code_sha256":"3dc410d21f1c3741"}},{"arxiv_id":"2606.06036","paper":"/paper/arxiv-2606-06036","title":"Memory is Reconstructed, Not Retrieved: Graph Memory for LLM Agents","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"Ji-shuo/MRAgent","path":"eval/evaluation.py","file_url":"https://github.com/Ji-shuo/MRAgent/blob/HEAD/eval/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e096279b3e0d7bd9","mcp_get_code":{"code_sha256":"e096279b3e0d7bd9"}},{"arxiv_id":"2606.04244","paper":"/paper/arxiv-2606-04244","title":"VAMPS: Visual-Assisted Mathematical Problem Solving Benchmark","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"vampsbenchmark/VAMPS","path":"extract_answers.py","file_url":"https://github.com/vampsbenchmark/VAMPS/blob/HEAD/extract_answers.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5c64b982057e8460","mcp_get_code":{"code_sha256":"5c64b982057e8460"}},{"arxiv_id":"2606.02578","paper":"/paper/arxiv-2606-02578","title":"Mitigating Perceptual Judgment Bias in Multimodal LLM-as-a-Judge via Perceptual Perturbation and Reward Modeling","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"kaist-cvml/perception-judge","path":"prepare-datasets/utils.py","file_url":"https://github.com/kaist-cvml/perception-judge/blob/HEAD/prepare-datasets/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2b85173eac27fd50","mcp_get_code":{"code_sha256":"2b85173eac27fd50"}},{"arxiv_id":"2605.29250","paper":"/paper/arxiv-2605-29250","title":"OmniRetrieval: Unified Retrieval across Heterogeneous Knowledge Sources","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"JinheonBaek/OmniRetrieval","path":"src/evaluation/metrics.py","file_url":"https://github.com/JinheonBaek/OmniRetrieval/blob/HEAD/src/evaluation/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2605.26952","paper":"/paper/arxiv-2605-26952","title":"Efficient Agentic Reinforcement Learning with On-Policy Intrinsic Knowledge Boundary Enhancement","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"CuSO4-Chen/AKBE","path":"AKBE/verl_akbe/verl/utils/reward_score/reward_em_betagrpo.py","file_url":"https://github.com/CuSO4-Chen/AKBE/blob/HEAD/AKBE/verl_akbe/verl/utils/reward_score/reward_em_betagrpo.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6fcc2c2f4fe3ad67","mcp_get_code":{"code_sha256":"6fcc2c2f4fe3ad67"}},{"arxiv_id":"2605.21856","paper":"/paper/arxiv-2605-21856","title":"The Illusion of Reasoning: Exposing Evasive Data Contamination in LLMs via Zero-CoT Truncation","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"Yifan-Lan/zero-cot-probe","path":"math_normalize.py","file_url":"https://github.com/Yifan-Lan/zero-cot-probe/blob/HEAD/math_normalize.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"644c1663d2f9533f","mcp_get_code":{"code_sha256":"644c1663d2f9533f"}},{"arxiv_id":"2605.21177","paper":"/paper/arxiv-2605-21177","title":"ChunkFT: Byte-Streamed Optimization for Memory-Efficient Full Fine-Tuning","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"misonsky/chunk","path":"metrics/cuad/compute_score.py","file_url":"https://github.com/misonsky/chunk/blob/HEAD/metrics/cuad/compute_score.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2605.18565","paper":"/paper/arxiv-2605-18565","title":"MINTEVAL: Evaluating Memory under Multi-Target Interference in Long-Horizon Agent Systems","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"amy-hyunji/MINTEval","path":"src/mem_alpha/memalpha/llm_agent/metrics.py","file_url":"https://github.com/amy-hyunji/MINTEval/blob/HEAD/src/mem_alpha/memalpha/llm_agent/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"3b3fe5d04eaa0d33","mcp_get_code":{"code_sha256":"3b3fe5d04eaa0d33"}},{"arxiv_id":"2605.10195","paper":"/paper/arxiv-2605-10195","title":"Breaking the Reward Barrier: Accelerating Tree-of-Thought Reasoning via Speculative Exploration","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"PKU-SEC-Lab/SPEX","path":"evaluate/evaluate_utils/math_normalize.py","file_url":"https://github.com/PKU-SEC-Lab/SPEX/blob/HEAD/evaluate/evaluate_utils/math_normalize.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"101b1e54fcdadf90","mcp_get_code":{"code_sha256":"101b1e54fcdadf90"}},{"arxiv_id":"2605.00199","paper":"/paper/arxiv-2605-00199","title":"RSAT: Structured Attribution Makes Small Language Models Faithful Table Reasoners","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"JugalGajjar/RSAT","path":"src/rewards/answer_reward.py","file_url":"https://github.com/JugalGajjar/RSAT/blob/HEAD/src/rewards/answer_reward.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ee8c43ee40891b07","mcp_get_code":{"code_sha256":"ee8c43ee40891b07"}},{"arxiv_id":"2604.23626","paper":"/paper/arxiv-2604-23626","title":"GraphPlanner: Graph Memory-Augmented Agentic Routing for Multi-Agent LLMs","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"ulab-uiuc/GraphPlanner","path":"router_planner/shared/utils.py","file_url":"https://github.com/ulab-uiuc/GraphPlanner/blob/HEAD/router_planner/shared/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c2e95e64eec380a8","mcp_get_code":{"code_sha256":"c2e95e64eec380a8"}},{"arxiv_id":"2604.21454","paper":"/paper/arxiv-2604-21454","title":"Reasoning Primitives in Hybrid and Non-Hybrid LLMs: Do Architectural Differences Yield Advantages in State-Tracking and Recall?","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"ultor1996/reasoning_primitives","path":"src/utils.py","file_url":"https://github.com/ultor1996/reasoning_primitives/blob/HEAD/src/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"85ea1889d0093c0d","mcp_get_code":{"code_sha256":"85ea1889d0093c0d"}},{"arxiv_id":"2604.20806","paper":"/paper/arxiv-2604-20806","title":"OMIBench: Benchmarking Olympiad-Level Multi-Image Reasoning in Large Vision-Language Models","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"LightChen233/OMIBench","path":"omi_bench.py","file_url":"https://github.com/LightChen233/OMIBench/blob/HEAD/omi_bench.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"9ea845c01a954867","mcp_get_code":{"code_sha256":"9ea845c01a954867"}},{"arxiv_id":"2604.19459","paper":"/paper/arxiv-2604-19459","title":"Do LLMs Game Formalization? Evaluating Faithfulness in Logical Reasoning","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"koreankiwi99/formalization-gaming","path":"src/utils/answer_parsing.py","file_url":"https://github.com/koreankiwi99/formalization-gaming/blob/HEAD/src/utils/answer_parsing.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"651b0f7470e1a3e9","mcp_get_code":{"code_sha256":"651b0f7470e1a3e9"}},{"arxiv_id":"2604.11510","paper":"/paper/arxiv-2604-11510","title":"Policy Split: Incentivizing Dual-Mode Exploration in LLM Reinforcement with Dual-Mode Entropy Regularization","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"BITHLP/PolicySplit","path":"verl/utils/reward_score/openai_math_grade.py","file_url":"https://github.com/BITHLP/PolicySplit/blob/HEAD/verl/utils/reward_score/openai_math_grade.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"101b1e54fcdadf90","mcp_get_code":{"code_sha256":"101b1e54fcdadf90"}},{"arxiv_id":"2604.11048","paper":"/paper/arxiv-2604-11048","title":"A Systematic Analysis of the Impact of Persona Steering on LLM Capabilities","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"cjia7/DPR","path":"src/npti/eval/eval_bbh.py","file_url":"https://github.com/cjia7/DPR/blob/HEAD/src/npti/eval/eval_bbh.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"caa05d10c796877f","mcp_get_code":{"code_sha256":"caa05d10c796877f"}},{"arxiv_id":"2604.10900","paper":"/paper/arxiv-2604-10900","title":"CASK: Core-Aware Selective KV Compression for Reasoning Traces","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"THUDM/LongBench","path":"LongBench/metrics.py","file_url":"https://github.com/THUDM/LongBench/blob/HEAD/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2604.05818","paper":"/paper/arxiv-2604-05818","title":"WikiSeeker: Rethinking the Role of Vision-Language Models in Knowledge-Based Visual Question Answering","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"zhuyjan/WikiSeeker","path":"utils/infoseek_evaluation_utils.py","file_url":"https://github.com/zhuyjan/WikiSeeker/blob/HEAD/utils/infoseek_evaluation_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"4925192ddc8dfa9a","mcp_get_code":{"code_sha256":"4925192ddc8dfa9a"}},{"arxiv_id":"2604.04356","paper":"/paper/arxiv-2604-04356","title":"REAM: Merging Improves Pruning of Experts in LLMs","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"zai-org/glm-simple-evals","path":"evals/grading/math_normalize.py","file_url":"https://github.com/zai-org/glm-simple-evals/blob/HEAD/evals/grading/math_normalize.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"101b1e54fcdadf90","mcp_get_code":{"code_sha256":"101b1e54fcdadf90"}},{"arxiv_id":"2603.26791","paper":"/paper/arxiv-2603-26791","title":"Crystal: Characterizing Relative Impact of Scholarly Publications","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"allenai/multicite","path":"qa/qasper_evaluator.py","file_url":"https://github.com/allenai/multicite/blob/HEAD/qa/qasper_evaluator.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"d6eb7fc276a790c2","mcp_get_code":{"code_sha256":"d6eb7fc276a790c2"}},{"arxiv_id":"2603.16654","paper":"/paper/arxiv-2603-16654","title":"Omanic: Towards Step-wise Evaluation of Multi-hop Reasoning in Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"XiaojieGu/Omanic","path":"eval/open_eval.py","file_url":"https://github.com/XiaojieGu/Omanic/blob/HEAD/eval/open_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"5cdd5f07b69b4933","mcp_get_code":{"code_sha256":"5cdd5f07b69b4933"}},{"arxiv_id":"2603.15892","paper":"/paper/arxiv-2603-15892","title":"Temporal Conflicts in LLMs: Reproducibility Insights from Unifying DYNAMICQA and MULAN","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"terrierteam/temporal_conflicts","path":"fact_mutability/analysis/f1_score.py","file_url":"https://github.com/terrierteam/temporal_conflicts/blob/HEAD/fact_mutability/analysis/f1_score.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2603.11327","paper":"/paper/arxiv-2603-11327","title":"Meta-Reinforcement Learning with Self-Reflection for Agentic Search","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"tengxiao1/MR-Search","path":"verl/utils/reward_score/search_r1_like_qa_em.py","file_url":"https://github.com/tengxiao1/MR-Search/blob/HEAD/verl/utils/reward_score/search_r1_like_qa_em.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2603.04759","paper":"/paper/arxiv-2603-04759","title":"Stacked from One: Multi-Scale Self-Injection for Context Window Extension","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"Clement25/SharedLLM","path":"LongBenchTest/metrics.py","file_url":"https://github.com/Clement25/SharedLLM/blob/HEAD/LongBenchTest/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2603.04759","paper":"/paper/arxiv-2603-04759","title":"Stacked from One: Multi-Scale Self-Injection for Context Window Extension","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"Clement25/SharedLLM","path":"utils.py","file_url":"https://github.com/Clement25/SharedLLM/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6c668580324cfd97","mcp_get_code":{"code_sha256":"6c668580324cfd97"}},{"arxiv_id":"2603.01281","paper":"/paper/arxiv-2603-01281","title":"Spectral Attention Steering for Prompt Highlighting","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"waylonli/SEKA","path":"pastalib/profiler/synthetic_profiler.py","file_url":"https://github.com/waylonli/SEKA/blob/HEAD/pastalib/profiler/synthetic_profiler.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"76b695cd06f395b4","mcp_get_code":{"code_sha256":"76b695cd06f395b4"}},{"arxiv_id":"2602.19672","paper":"/paper/arxiv-2602-19672","title":"SkillOrchestra: Learning to Route Agents via Skill Transfer","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"jiayuww/SkillOrchestra","path":"skillorchestra/eval/metrics.py","file_url":"https://github.com/jiayuww/SkillOrchestra/blob/HEAD/skillorchestra/eval/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f23e31c1141ee0fe","mcp_get_code":{"code_sha256":"f23e31c1141ee0fe"}},{"arxiv_id":"2602.17155","paper":"/paper/arxiv-2602-17155","title":"Powering Up Zeroth-Order Training via Subspace Gradient Orthogonalization","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"OPTML-Group/ZO-Muon","path":"llm/metrics.py","file_url":"https://github.com/OPTML-Group/ZO-Muon/blob/HEAD/llm/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2602.13551","paper":"/paper/arxiv-2602-13551","title":"Small Reward Models via Backward Inference","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"yikee/FLIP","path":"metrics.py","file_url":"https://github.com/yikee/FLIP/blob/HEAD/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"5a9d2662f435748f","mcp_get_code":{"code_sha256":"5a9d2662f435748f"}},{"arxiv_id":"2602.05728","paper":"/paper/arxiv-2602-05728","title":"CompactRAG: Reducing LLM Calls and Token Overhead in Multi-Hop Question Answering","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"How-Young-X/CompactRAG","path":"src/metrics/F1Eval.py","file_url":"https://github.com/How-Young-X/CompactRAG/blob/HEAD/src/metrics/F1Eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"26a63699755bd04e","mcp_get_code":{"code_sha256":"26a63699755bd04e"}},{"arxiv_id":"2602.03689","paper":"/paper/arxiv-2602-03689","title":"Rethinking the Reranker: Boundary-Aware Evidence Selection for Robust Retrieval-Augmented Generation","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"GasolSun36/BAR-RAG","path":"filter.py","file_url":"https://github.com/GasolSun36/BAR-RAG/blob/HEAD/filter.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"85fc7d4bd5e8711a","mcp_get_code":{"code_sha256":"85fc7d4bd5e8711a"}},{"arxiv_id":"2602.03645","paper":"/paper/arxiv-2602-03645","title":"Reinforcement Fine-Tuning for History-Aware Dense Retriever in RAG","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"zyc140345/HARR","path":"metric/qa_em.py","file_url":"https://github.com/zyc140345/HARR/blob/HEAD/metric/qa_em.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2602.02518","paper":"/paper/arxiv-2602-02518","title":"GRAPHDANCER: Training LLMs to Explore and Reason over Graphs via Two-Stage Curriculum Post-Training","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"leopoldwhite/GraphDancer","path":"verl/utils/reward_score/graphdancer_qa_em.py","file_url":"https://github.com/leopoldwhite/GraphDancer/blob/HEAD/verl/utils/reward_score/graphdancer_qa_em.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2602.02258","paper":"/paper/arxiv-2602-02258","title":"Alignment-Aware Model Adaptation via Feedback-Guided Optimization","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"facebookresearch/TruthRL","path":"evaluation/evaluate.py","file_url":"https://github.com/facebookresearch/TruthRL/blob/HEAD/evaluation/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2602.00723","paper":"/paper/arxiv-2602-00723","title":"Rethinking Hallucinations: Correctness, Consistency, and Prompt Multiplicity","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"AI21Labs/in-context-ralm","path":"eval_qa.py","file_url":"https://github.com/AI21Labs/in-context-ralm/blob/HEAD/eval_qa.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2601.22047","paper":"/paper/arxiv-2601-22047","title":"On the Paradoxical Interference between Instruction-Following and Task Solving","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"kijlk/IF-Interference","path":"src/math_and_qa/qa_utils.py","file_url":"https://github.com/kijlk/IF-Interference/blob/HEAD/src/math_and_qa/qa_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2601.21064","paper":"/paper/arxiv-2601-21064","title":"Textual Equilibrium Propagation for Deep Compound AI Systems","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"MinghuiChen43/TEP","path":"src/tep/metrics/f1.py","file_url":"https://github.com/MinghuiChen43/TEP/blob/HEAD/src/tep/metrics/f1.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"03171efd2aab7e80","mcp_get_code":{"code_sha256":"03171efd2aab7e80"}},{"arxiv_id":"2601.09402","paper":"/paper/arxiv-2601-09402","title":"SEEK: Steering LLM Reasoning for RAG via Internal Reasoning Sketches","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"OpenBMB/PAGER","path":"src/evaluate_infer.py","file_url":"https://github.com/OpenBMB/PAGER/blob/HEAD/src/evaluate_infer.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d22718d531542ab7","mcp_get_code":{"code_sha256":"d22718d531542ab7"}},{"arxiv_id":"2601.09028","paper":"/paper/arxiv-2601-09028","title":"OpenDecoder: Open Large Language Model Decoding to Incorporate Document Quality in RAG","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"fengranMark/OpenDecoder","path":"src/evaluation.py","file_url":"https://github.com/fengranMark/OpenDecoder/blob/HEAD/src/evaluation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"24a31436a8dbd623","mcp_get_code":{"code_sha256":"24a31436a8dbd623"}},{"arxiv_id":"2601.07894","paper":"/paper/arxiv-2601-07894","title":"Revealing the Attention Floating Mechanism in Masked Diffusion Models","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"NEUIR/Attention-Floating","path":"evaluate/evaluate_dream.py","file_url":"https://github.com/NEUIR/Attention-Floating/blob/HEAD/evaluate/evaluate_dream.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ff3b542378f5d19c","mcp_get_code":{"code_sha256":"ff3b542378f5d19c"}},{"arxiv_id":"2601.05903","paper":"/paper/arxiv-2601-05903","title":"HAPS: Hierarchical LLM Routing with Joint Architecture and Parameter Search","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"zihangtian/HAPS","path":"HotpotQA/joint_rl/rl_batch.py","file_url":"https://github.com/zihangtian/HAPS/blob/HEAD/HotpotQA/joint_rl/rl_batch.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6c668580324cfd97","mcp_get_code":{"code_sha256":"6c668580324cfd97"}},{"arxiv_id":"2601.03908","paper":"/paper/arxiv-2601-03908","title":"Decide Then Retrieve: A Training-Free Framework with Uncertainty-Guided Triggering and Dual-Path Retrieval","date":"2026-01-07","month_inferred_from_arxiv_id":null,"title_source":"syntology","repo":"ChenWangHKU/DTR","path":"evaluation/metrics.py","file_url":"https://github.com/ChenWangHKU/DTR/blob/HEAD/evaluation/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b92b07bb3574992a","mcp_get_code":{"code_sha256":"b92b07bb3574992a"}},{"arxiv_id":"2601.03042","paper":"/paper/arxiv-2601-03042","title":"BaseCal: Unsupervised Confidence Calibration via Base Model Signals","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"Tan-Hexiang/BaseCal","path":"gen_metric/em.py","file_url":"https://github.com/Tan-Hexiang/BaseCal/blob/HEAD/gen_metric/em.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9891ffc5ad4f4684","mcp_get_code":{"code_sha256":"9891ffc5ad4f4684"}},{"arxiv_id":"2511.19900","paper":"/paper/arxiv-2511-19900","title":"Agent0-VL: Exploring Self-Evolving Agent for Tool-Integrated Vision-Language Reasoning","date":null,"month_inferred_from_arxiv_id":"2025-11","title_source":"syntology","repo":"aiming-lab/Agent0","path":"Agent0-VL/verl/evaluation/metrics.py","file_url":"https://github.com/aiming-lab/Agent0/blob/HEAD/Agent0-VL/verl/evaluation/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e92c6c13f949ae88","mcp_get_code":{"code_sha256":"e92c6c13f949ae88"}},{"arxiv_id":"2511.07328","paper":"/paper/arxiv-2511-07328","title":"Q-RAG: Long Context Multi-step Retrieval via Value-based Embedder Training","date":null,"month_inferred_from_arxiv_id":"2025-11","title_source":"syntology","repo":"griver/Q-RAG","path":"eval_llm_openqa.py","file_url":"https://github.com/griver/Q-RAG/blob/HEAD/eval_llm_openqa.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b51d54e1d87d8bd9","mcp_get_code":{"code_sha256":"b51d54e1d87d8bd9"}},{"arxiv_id":"2510.13494","paper":"/paper/arxiv-2510-13494","title":"LiteraryQA: Towards Effective Evaluation of Long-document Narrative QA","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"SapienzaNLP/LiteraryQA","path":"literaryqa/ngram_metrics.py","file_url":"https://github.com/SapienzaNLP/LiteraryQA/blob/HEAD/literaryqa/ngram_metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dfcf1201010b7d31","mcp_get_code":{"code_sha256":"dfcf1201010b7d31"}},{"arxiv_id":"2510.07312","paper":"/paper/arxiv-2510-07312","title":"BOOTSTRAPPING LLMS TO REASON OVER LONGER HORIZONS VIA REINFORCEMENT LEARNING","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"AlesyaIvanova/h1","path":"math_utils.py","file_url":"https://github.com/AlesyaIvanova/h1/blob/HEAD/math_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"101b1e54fcdadf90","mcp_get_code":{"code_sha256":"101b1e54fcdadf90"}},{"arxiv_id":"2510.00568","paper":"/paper/arxiv-2510-00568","title":"ReSeek: A Self-Correcting Framework for Search Agents with Instructive Rewards","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2509.19633","paper":"/paper/arxiv-2509-19633","title":"Mamba Modulation On the Length Generalization of Mamba","date":null,"month_inferred_from_arxiv_id":"2025-09","title_source":"syntology","repo":"gnepul-ace/mamba_modulation","path":"Mamba/LongBench/metrics.py","file_url":"https://github.com/gnepul-ace/mamba_modulation/blob/HEAD/Mamba/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2509.14671","paper":"/paper/arxiv-2509-14671","title":"TableDART: Dynamic Adaptive Multi-Modal Routing for Table Understanding","date":null,"month_inferred_from_arxiv_id":"2025-09","title_source":"syntology","repo":"xiaobo-xing/TableDART","path":"evaluate.py","file_url":"https://github.com/xiaobo-xing/TableDART/blob/HEAD/evaluate.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"90ce0a2b27b15e27","mcp_get_code":{"code_sha256":"90ce0a2b27b15e27"}},{"arxiv_id":"2508.18847","paper":"/paper/arxiv-2508-18847","title":"ConfTuner: Training Large Language Models to Express Their Confidence Verbally","date":null,"month_inferred_from_arxiv_id":"2025-08","title_source":"syntology","repo":"liushiliushi/ConfTuner","path":"src/llama_recipes/datasets2/hotpot_qa.py","file_url":"https://github.com/liushiliushi/ConfTuner/blob/HEAD/src/llama_recipes/datasets2/hotpot_qa.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2507.21892","paper":null,"title":"arXiv:2507.21892","date":null,"month_inferred_from_arxiv_id":"2025-07","title_source":null,"repo":"LHRLAB/Graph-R1","path":"evaluation/eval_r.py","file_url":"https://github.com/LHRLAB/Graph-R1/blob/HEAD/evaluation/eval_r.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"78b0c6179667e101","mcp_get_code":{"code_sha256":"78b0c6179667e101"}},{"arxiv_id":"2505.24630","paper":"/paper/the-hallucination-dilemma-factuality-aware","title":"The Hallucination Dilemma: Factuality-Aware Reinforcement Learning for Large Reasoning Models","date":"2025-05-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nusnlp/fspo","path":"evaluate/inference.py","file_url":"https://github.com/nusnlp/fspo/blob/HEAD/evaluate/inference.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"d22718d531542ab7","mcp_get_code":{"code_sha256":"d22718d531542ab7"}},{"arxiv_id":"2505.21297","paper":"/paper/rstar-coder-scaling-competitive-code","title":"rStar-Coder: Scaling Competitive Code Reasoning with a Large-Scale Verified Dataset","date":"2025-05-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/rstar","path":"fused_compute_score/prime_math/math_normalize.py","file_url":"https://github.com/microsoft/rstar/blob/HEAD/fused_compute_score/prime_math/math_normalize.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"31113f93a6e0e98a","mcp_get_code":{"code_sha256":"31113f93a6e0e98a"}},{"arxiv_id":"2505.20128","paper":"/paper/iterative-self-incentivization-empowers-large","title":"Iterative Self-Incentivization Empowers Large Language Models as Agentic Searchers","date":"2025-05-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mangopy/searchlm","path":"src/utilize/metrics.py","file_url":"https://github.com/mangopy/searchlm/blob/HEAD/src/utilize/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"93730b62e07a68f0","mcp_get_code":{"code_sha256":"93730b62e07a68f0"}},{"arxiv_id":"2505.17005","paper":"/paper/r1-searcher-incentivizing-the-dynamic","title":"R1-Searcher++: Incentivizing the Dynamic Knowledge Acquisition of LLMs via Reinforcement Learning","date":"2025-05-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"RUCAIBox/R1-Searcher","path":"evaluation/metric_calc_rule.py","file_url":"https://github.com/RUCAIBox/R1-Searcher/blob/HEAD/evaluation/metric_calc_rule.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d22718d531542ab7","mcp_get_code":{"code_sha256":"d22718d531542ab7"}},{"arxiv_id":"2505.13430","paper":"/paper/fine-tuning-quantized-neural-networks-with","title":"Fine-tuning Quantized Neural Networks with Zeroth-order Optimization","date":"2025-05-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"maifoundations/qzo","path":"large_language_models/metrics.py","file_url":"https://github.com/maifoundations/qzo/blob/HEAD/large_language_models/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2504.17454","paper":"/paper/adaptive-orchestration-of-modular-generative","title":"Adaptive Orchestration of Modular Generative Information Access Systems","date":"2025-04-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"informagi/AQA","path":"Adaptive-RAG/evaluate.py","file_url":"https://github.com/informagi/AQA/blob/HEAD/Adaptive-RAG/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2504.16084","paper":"/paper/ttrl-test-time-reinforcement-learning","title":"TTRL: Test-Time Reinforcement Learning","date":"2025-04-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"prime-rl/ttrl","path":"verl/verl/utils/reward_score/ttrl_math/math_normalize.py","file_url":"https://github.com/prime-rl/ttrl/blob/HEAD/verl/verl/utils/reward_score/ttrl_math/math_normalize.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"31113f93a6e0e98a","mcp_get_code":{"code_sha256":"31113f93a6e0e98a"}},{"arxiv_id":"2504.13079","paper":"/paper/retrieval-augmented-generation-with-3","title":"Retrieval-Augmented Generation with Conflicting Evidence","date":"2025-04-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hannight/ramdocs","path":"run_madam_rag.py","file_url":"https://github.com/hannight/ramdocs/blob/HEAD/run_madam_rag.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"42413e5aa5425a3e","mcp_get_code":{"code_sha256":"42413e5aa5425a3e"}},{"arxiv_id":"2504.08231","paper":"/paper/out-of-style-rag-s-fragility-to-linguistic","title":"Out of Style: RAG's Fragility to Linguistic Variation","date":"2025-04-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"springcty/rag-fragility-to-linguistic-variation","path":"LLM_generation/c_eval_generation.py","file_url":"https://github.com/springcty/rag-fragility-to-linguistic-variation/blob/HEAD/LLM_generation/c_eval_generation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"bf618b8796ac4942","mcp_get_code":{"code_sha256":"bf618b8796ac4942"}},{"arxiv_id":"2504.03101","paper":"/paper/single-pass-document-scanning-for-question","title":"Single-Pass Document Scanning for Question Answering","date":"2025-04-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mambaretriever/mambaretriever","path":"rag_pipeline/metrics.py","file_url":"https://github.com/mambaretriever/mambaretriever/blob/HEAD/rag_pipeline/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2504.01698","paper":"/paper/tom-rl-reinforcement-learning-unlocks-theory","title":"Do Theory of Mind Benchmarks Need Explicit Human-like Reasoning in Language Models?","date":"2025-04-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bigai-ai/ToM-RL","path":"verl/utils/reward_score/explore_tom.py","file_url":"https://github.com/bigai-ai/ToM-RL/blob/HEAD/verl/utils/reward_score/explore_tom.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1436322b957b9b34","mcp_get_code":{"code_sha256":"1436322b957b9b34"}},{"arxiv_id":"2504.01698","paper":"/paper/tom-rl-reinforcement-learning-unlocks-theory","title":"Do Theory of Mind Benchmarks Need Explicit Human-like Reasoning in Language Models?","date":"2025-04-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bigai-ai/ToM-RL","path":"verl/utils/reward_score/explore_tom_2.py","file_url":"https://github.com/bigai-ai/ToM-RL/blob/HEAD/verl/utils/reward_score/explore_tom_2.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c5322e5a65a9bea1","mcp_get_code":{"code_sha256":"c5322e5a65a9bea1"}},{"arxiv_id":"2503.19707","paper":"/paper/mind-the-gap-benchmarking-spatial-reasoning","title":"Mind the Gap: Benchmarking Spatial Reasoning in Vision-Language Models","date":"2025-03-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"stogiannidis/srbench","path":"src/eval/acc.py","file_url":"https://github.com/stogiannidis/srbench/blob/HEAD/src/eval/acc.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"117783c12219bab5","mcp_get_code":{"code_sha256":"117783c12219bab5"}},{"arxiv_id":"2503.10571","paper":"/paper/radar-fast-long-context-decoding-for-any","title":"Radar: Fast Long-Context Decoding for Any Transformer","date":"2025-03-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"borealisai/radar-decoding","path":"research/longbench/longbench.py","file_url":"https://github.com/borealisai/radar-decoding/blob/HEAD/research/longbench/longbench.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"LGPL-3.0","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2503.06580","paper":"/paper/agent-models-internalizing-chain-of-action","title":"Agent models: Internalizing Chain-of-Action Generation into Reasoning models","date":"2025-03-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"adam-bjtu/autocoa","path":"evaluation/metrics.py","file_url":"https://github.com/adam-bjtu/autocoa/blob/HEAD/evaluation/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"98a43410c5681732","mcp_get_code":{"code_sha256":"98a43410c5681732"}},{"arxiv_id":"2503.02199","paper":"/paper/words-or-vision-do-vision-language-models","title":"Words or Vision: Do Vision-Language Models Have Blind Faith in Text?","date":"2025-03-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"d-ailin/blind-faith-in-text","path":"raw_code/hf_evaluator.py","file_url":"https://github.com/d-ailin/blind-faith-in-text/blob/HEAD/raw_code/hf_evaluator.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"0bcff406512311b4","mcp_get_code":{"code_sha256":"0bcff406512311b4"}},{"arxiv_id":"2502.06703","paper":"/paper/can-1b-llm-surpass-405b-llm-rethinking","title":"Can 1B LLM Surpass 405B LLM? Rethinking Compute-Optimal Test-Time Scaling","date":"2025-02-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"RyanLiu112/compute-optimal-tts","path":"src/envs/MATH/verify_utils.py","file_url":"https://github.com/RyanLiu112/compute-optimal-tts/blob/HEAD/src/envs/MATH/verify_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"101b1e54fcdadf90","mcp_get_code":{"code_sha256":"101b1e54fcdadf90"}},{"arxiv_id":"2502.06205","paper":"/paper/c-3po-compact-plug-and-play-proxy","title":"C-3PO: Compact Plug-and-Play Proxy Optimization to Achieve Human-like Retrieval-Augmented Generation","date":"2025-02-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Chen-GX/C-3PO","path":"C-3PO/metrics.py","file_url":"https://github.com/Chen-GX/C-3PO/blob/HEAD/C-3PO/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"37e0a25b59a4cc3e","mcp_get_code":{"code_sha256":"37e0a25b59a4cc3e"}},{"arxiv_id":"2502.01456","paper":"/paper/process-reinforcement-through-implicit","title":"Process Reinforcement through Implicit Rewards","date":"2025-02-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"prime-rl/prime","path":"data_preprocessing/math_util/math_normalize.py","file_url":"https://github.com/prime-rl/prime/blob/HEAD/data_preprocessing/math_util/math_normalize.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"101b1e54fcdadf90","mcp_get_code":{"code_sha256":"101b1e54fcdadf90"}},{"arxiv_id":"2501.12835","paper":"/paper/adaptive-retrieval-without-self-knowledge","title":"Adaptive Retrieval Without Self-Knowledge? Bringing Uncertainty Back Home","date":"2025-01-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"s-nlp/AdaRAGUE","path":"Adaptive_Rag/evaluate.py","file_url":"https://github.com/s-nlp/AdaRAGUE/blob/HEAD/Adaptive_Rag/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2501.11120","paper":"/paper/tell-me-about-yourself-llms-are-aware-of","title":"Tell me about yourself: LLMs are aware of their learned behaviors","date":"2025-01-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xuchanbao/behavioral-self-awareness","path":"code/multiple-choice/evaluation/self-report-free-separate-rephrasings.py","file_url":"https://github.com/xuchanbao/behavioral-self-awareness/blob/HEAD/code/multiple-choice/evaluation/self-report-free-separate-rephrasings.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b75e3691451b3c33","mcp_get_code":{"code_sha256":"b75e3691451b3c33"}},{"arxiv_id":"2412.15280","paper":"/paper/context-dpo-aligning-language-models-for","title":"Context-DPO: Aligning Language Models for Context-Faithfulness","date":"2024-12-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2412.09560","paper":"/paper/foundational-large-language-models-for","title":"Foundational Large Language Models for Materials Research","date":"2024-12-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"M3RG-IITD/llamat","path":"src/Kshot-val.py","file_url":"https://github.com/M3RG-IITD/llamat/blob/HEAD/src/Kshot-val.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2412.06206","paper":"/paper/sirerag-indexing-similar-and-related","title":"SiReRAG: Indexing Similar and Related Information for Multihop Reasoning","date":"2024-12-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"SalesforceAIResearch/SiReRAG","path":"evaluate_2wiki.py","file_url":"https://github.com/SalesforceAIResearch/SiReRAG/blob/HEAD/evaluate_2wiki.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2412.01981","paper":"/paper/free-process-rewards-without-process-labels","title":"Free Process Rewards without Process Labels","date":"2024-12-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lifan-yuan/implicitprm","path":"eval/math_utils/math_normalize.py","file_url":"https://github.com/lifan-yuan/implicitprm/blob/HEAD/eval/math_utils/math_normalize.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"101b1e54fcdadf90","mcp_get_code":{"code_sha256":"101b1e54fcdadf90"}},{"arxiv_id":"2411.16495","paper":"/paper/atomr-atomic-operator-empowered-large","title":"AtomR: Atomic Operator-Empowered Large Language Models for Heterogeneous Knowledge Reasoning","date":"2024-11-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THU-KEG/AtomR","path":"src/calculate_metrics.py","file_url":"https://github.com/THU-KEG/AtomR/blob/HEAD/src/calculate_metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"702b1281050dbc40","mcp_get_code":{"code_sha256":"702b1281050dbc40"}},{"arxiv_id":"2411.09688","paper":"/paper/squeezed-attention-accelerating-long-context","title":"Squeezed Attention: Accelerating Long Context Length LLM Inference","date":"2024-11-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"SqueezeAILab/SqueezedAttention","path":"LongBench/metrics.py","file_url":"https://github.com/SqueezeAILab/SqueezedAttention/blob/HEAD/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2411.08147","paper":"/paper/large-language-models-can-self-improve-in","title":"Large Language Models Can Self-Improve in Long-context Reasoning","date":"2024-11-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sihengli99/sealong","path":"eval_longbench_qa.py","file_url":"https://github.com/sihengli99/sealong/blob/HEAD/eval_longbench_qa.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2410.19258","paper":"/paper/not-all-heads-matter-a-head-level-kv-cache","title":"Not All Heads Matter: A Head-Level KV Cache Compression Method with Integrated Retrieval and Reasoning","date":"2024-10-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"fyyfu/headkv","path":"metrics.py","file_url":"https://github.com/fyyfu/headkv/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2410.18050","paper":"/paper/longrag-a-dual-perspective-retrieval","title":"LongRAG: A Dual-Perspective Retrieval-Augmented Generation Paradigm for Long-Context Question Answering","date":"2024-10-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"QingFei1/LongRAG","path":"src/metric.py","file_url":"https://github.com/QingFei1/LongRAG/blob/HEAD/src/metric.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"2410.16843","paper":"/paper/trustworthy-alignment-of-retrieval-augmented","title":"Trustworthy Alignment of Retrieval-Augmented Large Language Models via Reinforcement Learning","date":"2024-10-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zmzhang2000/trustworthy-alignment","path":"dschat/utils/metrics.py","file_url":"https://github.com/zmzhang2000/trustworthy-alignment/blob/HEAD/dschat/utils/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9706a9a0b961096b","mcp_get_code":{"code_sha256":"9706a9a0b961096b"}},{"arxiv_id":"2410.13846","paper":"/paper/simlayerkv-a-simple-framework-for-layer-level","title":"SimLayerKV: A Simple Framework for Layer-Level KV Cache Reduction","date":"2024-10-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sail-sg/simlayerkv","path":"LongBench/metrics.py","file_url":"https://github.com/sail-sg/simlayerkv/blob/HEAD/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2410.13360","paper":"/paper/remember-retrieve-and-generate-understanding","title":"RAP: Retrieval-Augmented Personalization for Multimodal Large Language Models","date":"2024-10-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hoar012/rap-mllm","path":"eval/eval_qa.py","file_url":"https://github.com/hoar012/rap-mllm/blob/HEAD/eval/eval_qa.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"7927b07dcc328a1f","mcp_get_code":{"code_sha256":"7927b07dcc328a1f"}},{"arxiv_id":"2410.09629","paper":"/paper/synthetic-knowledge-ingestion-towards","title":"Synthetic Knowledge Ingestion: Towards Knowledge Refinement and Injection for Enhancing Large Language Models","date":"2024-10-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"intuit-ai-research/knowledge-infused-ai","path":"synthetic-knowledge-ingestion/finetuning/ft_llama_factory.py","file_url":"https://github.com/intuit-ai-research/knowledge-infused-ai/blob/HEAD/synthetic-knowledge-ingestion/finetuning/ft_llama_factory.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2410.08989","paper":"/paper/subzero-random-subspace-zeroth-order","title":"Zeroth-Order Fine-Tuning of LLMs in Random Subspaces","date":"2024-10-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zimingyy/subzero","path":"large_models/metrics.py","file_url":"https://github.com/zimingyy/subzero/blob/HEAD/large_models/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2410.07652","paper":"/paper/stableprompt-automatic-prompt-tuning-using","title":"StablePrompt: Automatic Prompt Tuning using Reinforcement Learning for Large Language Models","date":"2024-10-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kmc0207/Stableprompt","path":"utils.py","file_url":"https://github.com/kmc0207/Stableprompt/blob/HEAD/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"d8af265929dc927f","mcp_get_code":{"code_sha256":"d8af265929dc927f"}},{"arxiv_id":"2410.06617","paper":"/paper/learning-evolving-tools-for-large-language","title":"Learning Evolving Tools for Large Language Models","date":"2024-10-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Chen-GX/ToolEVO","path":"MCTS/src/evaluation.py","file_url":"https://github.com/Chen-GX/ToolEVO/blob/HEAD/MCTS/src/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"96793b1c7ee55523","mcp_get_code":{"code_sha256":"96793b1c7ee55523"}},{"arxiv_id":"2410.05162","paper":"/paper/deciphering-the-interplay-of-parametric-and","title":"Deciphering the Interplay of Parametric and Non-parametric Memory in Retrieval-augmented Language Models","date":"2024-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"m3hrdadfi/rag-memory-interplay","path":"src/atlas/evaluation.py","file_url":"https://github.com/m3hrdadfi/rag-memory-interplay/blob/HEAD/src/atlas/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6205dd62c2627dde","mcp_get_code":{"code_sha256":"6205dd62c2627dde"}},{"arxiv_id":"2410.05076","paper":"/paper/tidaldecode-fast-and-accurate-llm-decoding","title":"TidalDecode: Fast and Accurate LLM Decoding with Position Persistent Sparse Attention","date":"2024-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"DerrickYLJ/TidalDecode","path":"experiments/LongBench/metrics.py","file_url":"https://github.com/DerrickYLJ/TidalDecode/blob/HEAD/experiments/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2410.04139","paper":"/paper/from-reading-to-compressing-exploring-the","title":"From Reading to Compressing: Exploring the Multi-document Reader for Prompt Compression","date":"2024-10-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"eunseongc/r2c","path":"LLM_inference.py","file_url":"https://github.com/eunseongc/r2c/blob/HEAD/LLM_inference.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2c014e1529169f70","mcp_get_code":{"code_sha256":"2c014e1529169f70"}},{"arxiv_id":"2410.03834","paper":"/paper/graphrouter-a-graph-based-router-for-llm","title":"GraphRouter: A Graph-based Router for LLM Selections","date":"2024-10-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ulab-uiuc/graphrouter","path":"data_processing/utils.py","file_url":"https://github.com/ulab-uiuc/graphrouter/blob/HEAD/data_processing/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7f6987eac0aced8","mcp_get_code":{"code_sha256":"e7f6987eac0aced8"}},{"arxiv_id":"2410.02694","paper":"/paper/helmet-how-to-evaluate-long-context-language","title":"HELMET: How to Evaluate Long-Context Language Models Effectively and Thoroughly","date":"2024-10-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-nlp/helmet","path":"utils.py","file_url":"https://github.com/princeton-nlp/helmet/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6c668580324cfd97","mcp_get_code":{"code_sha256":"6c668580324cfd97"}},{"arxiv_id":"2410.01671","paper":"/paper/bridging-context-gaps-leveraging-coreference","title":"Bridging Context Gaps: Leveraging Coreference Resolution for Long Contextual Understanding","date":"2024-10-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"OceannTwT/LQCA","path":"predict.py","file_url":"https://github.com/OceannTwT/LQCA/blob/HEAD/predict.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"80f89e5ac2bbf130","mcp_get_code":{"code_sha256":"80f89e5ac2bbf130"}},{"arxiv_id":"2409.19804","paper":"/paper/does-rag-introduce-unfairness-in-llms","title":"Does RAG Introduce Unfairness in LLMs? Evaluating Fairness in Retrieval-Augmented Generation Systems","date":"2024-09-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"elviswxy/rag_fairness","path":"flashrag/evaluator/utils.py","file_url":"https://github.com/elviswxy/rag_fairness/blob/HEAD/flashrag/evaluator/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2409.17422","paper":"/paper/discovering-the-gems-in-early-layers","title":"Discovering the Gems in Early Layers: Accelerating Long-Context LLMs with 1000x Input Token Reduction","date":"2024-09-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"salesforceairesearch/gemfilter","path":"eval/LongBench/metrics.py","file_url":"https://github.com/salesforceairesearch/gemfilter/blob/HEAD/eval/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2409.12181","paper":"/paper/a-controlled-study-on-long-context-extension","title":"A Controlled Study on Long Context Extension and Generalization in LLMs","date":"2024-09-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"leooyii/lceg","path":"longbench/metrics.py","file_url":"https://github.com/leooyii/lceg/blob/HEAD/longbench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2409.07394","paper":"/paper/adacad-adaptively-decoding-to-balance","title":"AdaCAD: Adaptively Decoding to Balance Conflicts between Contextual and Parametric Knowledge","date":"2024-09-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2408.11745","paper":"/paper/focusllm-scaling-llm-s-context-by-parallel","title":"FocusLLM: Precise Understanding of Long Context by Dynamic Condensing","date":"2024-08-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"leezythu/focusllm","path":"infbench_src/compute_scores.py","file_url":"https://github.com/leezythu/focusllm/blob/HEAD/infbench_src/compute_scores.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4b44a4d3fd80a48d","mcp_get_code":{"code_sha256":"4b44a4d3fd80a48d"}},{"arxiv_id":"2408.07092","paper":"/paper/post-training-sparse-attention-with-double","title":"Post-Training Sparse Attention with Double Sparsity","date":"2024-08-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"andy-yang-1/doublesparse","path":"LongBench/metrics.py","file_url":"https://github.com/andy-yang-1/doublesparse/blob/HEAD/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2407.13803","paper":"/paper/less-is-more-sparse-watermarking-in-llms-with","title":"Less is More: Sparse Watermarking in LLMs with Enhanced Text Quality","date":"2024-07-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mail-research/sparse-llm-watermarking","path":"metrics.py","file_url":"https://github.com/mail-research/sparse-llm-watermarking/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2407.12784","paper":"/paper/agentpoison-red-teaming-llm-agents-via","title":"AgentPoison: Red-teaming LLM Agents via Poisoning Memory or Knowledge Bases","date":"2024-07-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"BillChan226/AgentPoison","path":"ReAct/wrappers.py","file_url":"https://github.com/BillChan226/AgentPoison/blob/HEAD/ReAct/wrappers.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2407.09450","paper":"/paper/human-like-episodic-memory-for-infinite","title":"Human-like Episodic Memory for Infinite Context LLMs","date":"2024-07-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"em-llm/EM-LLM-model","path":"benchmark/infinitebench_eval.py","file_url":"https://github.com/em-llm/EM-LLM-model/blob/HEAD/benchmark/infinitebench_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4b44a4d3fd80a48d","mcp_get_code":{"code_sha256":"4b44a4d3fd80a48d"}},{"arxiv_id":"2407.07071","paper":"/paper/lookback-lens-detecting-and-mitigating","title":"Lookback Lens: Detecting and Mitigating Contextual Hallucinations in Large Language Models Using Only Attention Maps","date":"2024-07-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"voidism/lookback-lens","path":"eval_exact_match.py","file_url":"https://github.com/voidism/lookback-lens/blob/HEAD/eval_exact_match.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9706a9a0b961096b","mcp_get_code":{"code_sha256":"9706a9a0b961096b"}},{"arxiv_id":"2407.03129","paper":"/paper/social-bias-evaluation-for-large-language","title":"Social Bias Evaluation for Large Language Models Requires Prompt Variations","date":"2024-07-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rem-h4/llm_socialbias_prompts","path":"evaluation/eval_bbq.py","file_url":"https://github.com/rem-h4/llm_socialbias_prompts/blob/HEAD/evaluation/eval_bbq.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ff05709c690ea408","mcp_get_code":{"code_sha256":"ff05709c690ea408"}},{"arxiv_id":"2407.01527","paper":"/paper/kv-cache-compression-but-what-must-we-give-in","title":"KV Cache Compression, But What Must We Give in Return? A Comprehensive Benchmark of Long Context Capable Approaches","date":"2024-07-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"henryzhongsc/longctx_bench","path":"eval/longbench_utils/metrics.py","file_url":"https://github.com/henryzhongsc/longctx_bench/blob/HEAD/eval/longbench_utils/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2407.01527","paper":"/paper/kv-cache-compression-but-what-must-we-give-in","title":"KV Cache Compression, But What Must We Give in Return? A Comprehensive Benchmark of Long Context Capable Approaches","date":"2024-07-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"henryzhongsc/longctx_bench","path":"eval/passkey_utils/eval_metrics.py","file_url":"https://github.com/henryzhongsc/longctx_bench/blob/HEAD/eval/passkey_utils/eval_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"92035fa2c8ef3cbe","mcp_get_code":{"code_sha256":"92035fa2c8ef3cbe"}},{"arxiv_id":"2406.18847","paper":"/paper/learning-retrieval-augmentation-for","title":"Learning Retrieval Augmentation for Personalized Dialogue Generation","date":"2024-06-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hqsiswiliam/lapdog","path":"src/evaluation.py","file_url":"https://github.com/hqsiswiliam/lapdog/blob/HEAD/src/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6205dd62c2627dde","mcp_get_code":{"code_sha256":"6205dd62c2627dde"}},{"arxiv_id":"2406.18532","paper":"/paper/symbolic-learning-enables-self-evolving","title":"Symbolic Learning Enables Self-Evolving Agents","date":"2024-06-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"aiwaves-cn/agents","path":"src/agents/datasets/hotpotqa.py","file_url":"https://github.com/aiwaves-cn/agents/blob/HEAD/src/agents/datasets/hotpotqa.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2406.18187","paper":"/paper/selective-prompting-tuning-for-personalized","title":"Selective Prompting Tuning for Personalized Conversations with LLMs","date":"2024-06-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hqsiswiliam/SPT","path":"evaluation.py","file_url":"https://github.com/hqsiswiliam/SPT/blob/HEAD/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6205dd62c2627dde","mcp_get_code":{"code_sha256":"6205dd62c2627dde"}},{"arxiv_id":"2406.17245","paper":"/paper/unlocking-continual-learning-abilities-in","title":"Unlocking Continual Learning Abilities in Language Models","date":"2024-06-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"wenyudu/migu","path":"src/compute_metrics.py","file_url":"https://github.com/wenyudu/migu/blob/HEAD/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2406.15187","paper":"/paper/uda-a-benchmark-suite-for-retrieval-augmented","title":"UDA: A Benchmark Suite for Retrieval Augmented Generation in Real-world Document Analysis","date":"2024-06-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"d6eb7fc276a790c2","mcp_get_code":{"code_sha256":"d6eb7fc276a790c2"}},{"arxiv_id":"2406.12809","paper":"/paper/can-large-language-models-always-solve-easy","title":"Can Large Language Models Always Solve Easy Problems if They Can Solve Harder Ones?","date":"2024-06-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"QwenLM/ConsisEval","path":"math_check/math_normalize.py","file_url":"https://github.com/QwenLM/ConsisEval/blob/HEAD/math_check/math_normalize.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"101b1e54fcdadf90","mcp_get_code":{"code_sha256":"101b1e54fcdadf90"}},{"arxiv_id":"2406.12382","paper":"/paper/from-instance-training-to-instruction","title":"From Instance Training to Instruction Learning: Task Adapters Generation from Instructions","date":"2024-06-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Xnhyacinth/TAGI","path":"src/compute_metrics.py","file_url":"https://github.com/Xnhyacinth/TAGI/blob/HEAD/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2406.10774","paper":"/paper/quest-query-aware-sparsity-for-efficient-long","title":"Quest: Query-Aware Sparsity for Efficient Long-Context LLM Inference","date":"2024-06-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mit-han-lab/Quest","path":"evaluation/LongBench/metrics.py","file_url":"https://github.com/mit-han-lab/Quest/blob/HEAD/evaluation/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2406.09714","paper":"/paper/large-language-model-validity-via-enhanced","title":"Large language model validity via enhanced conformal prediction methods","date":"2024-06-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dmis-lab/olaph","path":"Factscore/factscore/atomic_facts.py","file_url":"https://github.com/dmis-lab/olaph/blob/HEAD/Factscore/factscore/atomic_facts.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2406.09072","paper":"/paper/living-in-the-moment-can-large-language","title":"Living in the Moment: Can Large Language Models Grasp Co-Temporal Reasoning?","date":"2024-06-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhaochen0110/cotempqa","path":"config.py","file_url":"https://github.com/zhaochen0110/cotempqa/blob/HEAD/config.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e5b347d9d2887961","mcp_get_code":{"code_sha256":"e5b347d9d2887961"}},{"arxiv_id":"2406.07528","paper":"/paper/quickllama-query-aware-inference-acceleration","title":"QuickLLaMA: Query-aware Inference Acceleration for Large Language Models","date":"2024-06-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dvlab-research/q-llm","path":"benchmark/metrics.py","file_url":"https://github.com/dvlab-research/q-llm/blob/HEAD/benchmark/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2406.07528","paper":"/paper/quickllama-query-aware-inference-acceleration","title":"QuickLLaMA: Query-aware Inference Acceleration for Large Language Models","date":"2024-06-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dvlab-research/q-llm","path":"benchmark/infinitebench_eval.py","file_url":"https://github.com/dvlab-research/q-llm/blob/HEAD/benchmark/infinitebench_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"4b44a4d3fd80a48d","mcp_get_code":{"code_sha256":"4b44a4d3fd80a48d"}},{"arxiv_id":"2406.06469","paper":"/paper/husky-a-unified-open-source-language-agent","title":"Husky: A Unified, Open-Source Language Agent for Multi-Step Reasoning","date":"2024-06-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2406.06357","paper":"/paper/massw-a-new-dataset-and-benchmark-tasks-for","title":"MASSW: A New Dataset and Benchmark Tasks for AI-Assisted Scientific Workflows","date":"2024-06-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xingjian-zhang/massw","path":"factscore/atomic_facts.py","file_url":"https://github.com/xingjian-zhang/massw/blob/HEAD/factscore/atomic_facts.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"CC0-1.0","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2406.06326","paper":"/paper/self-tuning-instructing-llms-to-effectively","title":"Self-Tuning: Instructing LLMs to Effectively Acquire New Knowledge through Self-Teaching","date":"2024-06-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"2879260bc19381c8","mcp_get_code":{"code_sha256":"2879260bc19381c8"}},{"arxiv_id":"2406.05654","paper":"/paper/domainrag-a-chinese-benchmark-for-evaluating","title":"DomainRAG: A Chinese Benchmark for Evaluating Domain-specific Retrieval-Augmented Generation","date":"2024-06-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ShootingWong/DomainRAG","path":"BCM/evaluation/Evaluator.py","file_url":"https://github.com/ShootingWong/DomainRAG/blob/HEAD/BCM/evaluation/Evaluator.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6205dd62c2627dde","mcp_get_code":{"code_sha256":"6205dd62c2627dde"}},{"arxiv_id":"2406.04823","paper":"/paper/berts-are-generative-in-context-learners","title":"BERTs are Generative In-Context Learners","date":"2024-06-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ltgoslo/bert-in-context","path":"glue/record.py","file_url":"https://github.com/ltgoslo/bert-in-context/blob/HEAD/glue/record.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"2406.03482","paper":"/paper/qjl-1-bit-quantized-jl-transform-for-kv-cache","title":"QJL: 1-Bit Quantized JL Transform for KV Cache Quantization with Zero Overhead","date":"2024-06-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"amirzandieh/qjl","path":"metrics.py","file_url":"https://github.com/amirzandieh/qjl/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2406.02536","paper":"/paper/mitigate-position-bias-in-large-language","title":"Mitigate Position Bias in Large Language Models via Scaling a Single Dimension","date":"2024-06-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"PositionalHidden/PositionalHidden","path":"experiments/run_on_longbench/metrics.py","file_url":"https://github.com/PositionalHidden/PositionalHidden/blob/HEAD/experiments/run_on_longbench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2406.02536","paper":"/paper/mitigate-position-bias-in-large-language","title":"Mitigate Position Bias in Large Language Models via Scaling a Single Dimension","date":"2024-06-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"PositionalHidden/PositionalHidden","path":"experiments/lost_in_the_middle/metrics.py","file_url":"https://github.com/PositionalHidden/PositionalHidden/blob/HEAD/experiments/lost_in_the_middle/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9706a9a0b961096b","mcp_get_code":{"code_sha256":"9706a9a0b961096b"}},{"arxiv_id":"2406.00132","paper":"/paper/quanta-efficient-high-rank-fine-tuning-of","title":"QuanTA: Efficient High-Rank Fine-Tuning of LLMs with Quantum-Informed Tensor Adaptation","date":"2024-05-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"quanta-fine-tuning/quanta","path":"run/utils/metrics.py","file_url":"https://github.com/quanta-fine-tuning/quanta/blob/HEAD/run/utils/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2405.20978","paper":"/paper/enhancing-noise-robustness-of-retrieval","title":"Enhancing Noise Robustness of Retrieval-Augmented Language Models with Adaptive Adversarial Training","date":"2024-05-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"calubkk/RAAT","path":"tuner/metrics/em_f1.py","file_url":"https://github.com/calubkk/RAAT/blob/HEAD/tuner/metrics/em_f1.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"5bfb8eaceb8fc0c1","mcp_get_code":{"code_sha256":"5bfb8eaceb8fc0c1"}},{"arxiv_id":"2405.18727","paper":"/paper/ctrla-adaptive-retrieval-augmented-generation","title":"CtrlA: Adaptive Retrieval-Augmented Generation via Inherent Control","date":"2024-05-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hsliu-initial/ctrla","path":"code/utils.py","file_url":"https://github.com/hsliu-initial/ctrla/blob/HEAD/code/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c29fc928dae85749","mcp_get_code":{"code_sha256":"c29fc928dae85749"}},{"arxiv_id":"2405.16821","paper":"/paper/perturbation-restrained-sequential-model","title":"Perturbation-Restrained Sequential Model Editing","date":"2024-05-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mjy1111/PRUNE","path":"edit_load.py","file_url":"https://github.com/mjy1111/PRUNE/blob/HEAD/edit_load.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"4cca112e5e6da5dc","mcp_get_code":{"code_sha256":"4cca112e5e6da5dc"}},{"arxiv_id":"2404.16811","paper":"/paper/make-your-llm-fully-utilize-the-context","title":"Make Your LLM Fully Utilize the Context","date":"2024-04-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/FILM","path":"real_world_long/metrics.py","file_url":"https://github.com/microsoft/FILM/blob/HEAD/real_world_long/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2404.14469","paper":"/paper/snapkv-llm-knows-what-you-are-looking-for","title":"SnapKV: LLM Knows What You are Looking for Before Generation","date":"2024-04-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"fasterdecoding/snapkv","path":"experiments/LongBench/metrics.py","file_url":"https://github.com/fasterdecoding/snapkv/blob/HEAD/experiments/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2404.13081","paper":"/paper/sure-summarizing-retrievals-using-answer","title":"SuRe: Summarizing Retrievals using Answer Candidates for Open-domain QA of LLMs","date":"2024-04-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bbuing9/iclr24_sure","path":"functions/gen_candidates.py","file_url":"https://github.com/bbuing9/iclr24_sure/blob/HEAD/functions/gen_candidates.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"437b1a6b2bad3170","mcp_get_code":{"code_sha256":"437b1a6b2bad3170"}},{"arxiv_id":"2404.12452","paper":"/paper/characterizing-llm-abstention-behavior-in","title":"Characterizing LLM Abstention Behavior in Science QA with Context Perturbations","date":"2024-04-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bbwen/llm_scienceqa","path":"evaluation/qasper_evaluation.py","file_url":"https://github.com/bbwen/llm_scienceqa/blob/HEAD/evaluation/qasper_evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"d6eb7fc276a790c2","mcp_get_code":{"code_sha256":"d6eb7fc276a790c2"}},{"arxiv_id":"2404.12145","paper":"/paper/from-form-s-to-meaning-probing-the-semantic","title":"From Form(s) to Meaning: Probing the Semantic Depths of Language Models Using Multisense Consistency","date":"2024-04-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"facebookresearch/multisense_consistency","path":"utils/eval_metrics.py","file_url":"https://github.com/facebookresearch/multisense_consistency/blob/HEAD/utils/eval_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"9f890f8bdba1b1cc","mcp_get_code":{"code_sha256":"9f890f8bdba1b1cc"}},{"arxiv_id":"2404.07917","paper":"/paper/designqa-a-multimodal-benchmark-for","title":"DesignQA: A Multimodal Benchmark for Evaluating Large Language Models' Understanding of Engineering Documentation","date":"2024-04-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"anniedoris/design_qa","path":"eval/metrics/metrics.py","file_url":"https://github.com/anniedoris/design_qa/blob/HEAD/eval/metrics/metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"89531f5d564727da","mcp_get_code":{"code_sha256":"89531f5d564727da"}},{"arxiv_id":"2404.04793","paper":"/paper/squeezeattention-2d-management-of-kv-cache-in","title":"SqueezeAttention: 2D Management of KV-Cache in LLM Inference via Layer-wise Optimal Budget","date":"2024-04-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hetailang/squeezeattention","path":"metrics.py","file_url":"https://github.com/hetailang/squeezeattention/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2404.02078","paper":"/paper/advancing-llm-reasoning-generalists-with","title":"Advancing LLM Reasoning Generalists with Preference Trees","date":"2024-04-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"openbmb/eurus","path":"eval/utils/math_normalize.py","file_url":"https://github.com/openbmb/eurus/blob/HEAD/eval/utils/math_normalize.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7131520594af5fe7","mcp_get_code":{"code_sha256":"7131520594af5fe7"}},{"arxiv_id":"2403.17359","paper":"/paper/chain-of-action-faithful-and-multimodal","title":"Chain-of-Action: Faithful and Multimodal Question Answering through Large Language Models","date":"2024-03-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"MAGICS-LAB/Chain-of-Actions","path":"baselines/src/Search-in-the-Chain/Server/server.py","file_url":"https://github.com/MAGICS-LAB/Chain-of-Actions/blob/HEAD/baselines/src/Search-in-the-Chain/Server/server.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"4cca112e5e6da5dc","mcp_get_code":{"code_sha256":"4cca112e5e6da5dc"}},{"arxiv_id":"2403.15268","paper":"/paper/imagination-augmented-generation-learning-to","title":"Awakening Augmented Generation: Learning to Awaken Internal Knowledge of Large Language Models for Question Answering","date":"2024-03-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xnhyacinth/iag","path":"src/metrics.py","file_url":"https://github.com/xnhyacinth/iag/blob/HEAD/src/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"9706a9a0b961096b","mcp_get_code":{"code_sha256":"9706a9a0b961096b"}},{"arxiv_id":"2403.14403","paper":"/paper/adaptive-rag-learning-to-adapt-retrieval","title":"Adaptive-RAG: Learning to Adapt Retrieval-Augmented Large Language Models through Question Complexity","date":"2024-03-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"starsuzi/Adaptive-RAG","path":"evaluate.py","file_url":"https://github.com/starsuzi/Adaptive-RAG/blob/HEAD/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2403.12393","paper":"/paper/dr3-ask-large-language-models-not-to-give-off","title":"Dr3: Ask Large Language Models Not to Give Off-Topic Answers in Open Domain Multi-Hop Question Answering","date":"2024-03-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gy915/dr3","path":"Dr3/metrics.py","file_url":"https://github.com/gy915/dr3/blob/HEAD/Dr3/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2403.09040","paper":"/paper/ragged-towards-informed-design-of-retrieval","title":"RAGGED: Towards Informed Design of Retrieval Augmented Generation Systems","date":"2024-03-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"neulab/ragged","path":"reader/evaluate_top_k.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/reader/evaluate_top_k.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2403.06840","paper":"/paper/ra-isf-learning-to-answer-and-understand-from","title":"RA-ISF: Learning to Answer and Understand from Retrieval Augmentation via Iterative Self-Feedback","date":"2024-03-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"oceanntwt/ra-isf","path":"main_gpt.py","file_url":"https://github.com/oceanntwt/ra-isf/blob/HEAD/main_gpt.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"cc68703252773762","mcp_get_code":{"code_sha256":"cc68703252773762"}},{"arxiv_id":"2403.04797","paper":"/paper/found-in-the-middle-how-language-models-use","title":"Found in the Middle: How Language Models Use Long Contexts Better via Plug-and-Play Positional Encoding","date":"2024-03-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"vita-group/ms-poe","path":"utils/lost_in_the_middle/metrics.py","file_url":"https://github.com/vita-group/ms-poe/blob/HEAD/utils/lost_in_the_middle/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9706a9a0b961096b","mcp_get_code":{"code_sha256":"9706a9a0b961096b"}},{"arxiv_id":"2403.04317","paper":"/paper/online-adaptation-of-language-models-with-a","title":"Online Adaptation of Language Models with a Memory of Amortized Contexts","date":"2024-03-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jihoontack/MAC","path":"utils.py","file_url":"https://github.com/jihoontack/MAC/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"2403.03823","paper":"/paper/a-modular-approach-for-multimodal","title":"A Modular Approach for Multimodal Summarization of TV Shows","date":"2024-03-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shmsw25/FActScore","path":"factscore/atomic_facts.py","file_url":"https://github.com/shmsw25/FActScore/blob/HEAD/factscore/atomic_facts.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2403.00843","paper":"/paper/enhancing-long-term-recommendation-with-bi","title":"Large Language Models are Learnable Planners for Long-Term Recommendation","date":"2024-02-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jizhi-zhang/billp","path":"Agents/agent_reflexion.py","file_url":"https://github.com/jizhi-zhang/billp/blob/HEAD/Agents/agent_reflexion.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a6b5e8f7937e1e87","mcp_get_code":{"code_sha256":"a6b5e8f7937e1e87"}},{"arxiv_id":"2402.18150","paper":"/paper/unsupervised-information-refinement-training","title":"Unsupervised Information Refinement Training of Large Language Models for Retrieval-Augmented Generation","date":"2024-02-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xsc1234/info-rag","path":"training/train_info_rag.py","file_url":"https://github.com/xsc1234/info-rag/blob/HEAD/training/train_info_rag.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9891ffc5ad4f4684","mcp_get_code":{"code_sha256":"9891ffc5ad4f4684"}},{"arxiv_id":"2402.18045","paper":"/paper/multi-fact-assessing-multilingual-llms-multi","title":"Multi-FAct: Assessing Factuality of Multilingual LLMs using FActScore","date":"2024-02-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sheikhshafayat/multi-fact","path":"factscore/atomic_facts.py","file_url":"https://github.com/sheikhshafayat/multi-fact/blob/HEAD/factscore/atomic_facts.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2402.17124","paper":"/paper/fact-and-reflection-far-improves-confidence","title":"Fact-and-Reflection (FaR) Improves Confidence Calibration of Large Language Models","date":"2024-02-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"colinzhaoust/fact-and-reflection","path":"public_code.py","file_url":"https://github.com/colinzhaoust/fact-and-reflection/blob/HEAD/public_code.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"55e919890251f878","mcp_get_code":{"code_sha256":"55e919890251f878"}},{"arxiv_id":"2402.16617","paper":"/paper/long-context-language-modeling-with-parallel","title":"Long-Context Language Modeling with Parallel Context Encoding","date":"2024-02-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-nlp/cepe","path":"utils.py","file_url":"https://github.com/princeton-nlp/cepe/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6c668580324cfd97","mcp_get_code":{"code_sha256":"6c668580324cfd97"}},{"arxiv_id":"2402.12842","paper":"/paper/promptkd-distilling-student-friendly","title":"PromptKD: Distilling Student-Friendly Knowledge for Generative Language Models via Prompt Tuning","date":"2024-02-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gmkim-ai/promptkd","path":"rouge_metric.py","file_url":"https://github.com/gmkim-ai/promptkd/blob/HEAD/rouge_metric.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2402.12659","paper":"/paper/the-finben-an-holistic-financial-benchmark","title":"FinBen: A Holistic Financial Benchmark for Large Language Models","date":"2024-02-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"chancefocus/pixiu","path":"src/factscore_package/atomic_facts.py","file_url":"https://github.com/chancefocus/pixiu/blob/HEAD/src/factscore_package/atomic_facts.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2402.12563","paper":"/paper/confidence-matters-revisiting-intrinsic-self","title":"Confidence Matters: Revisiting Intrinsic Self-Correction Capabilities of Large Language Models","date":"2024-02-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mbzuai-clear/ioe-prompting","path":"run_Hotpot_IoE.py","file_url":"https://github.com/mbzuai-clear/ioe-prompting/blob/HEAD/run_Hotpot_IoE.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b1a674018a8b1ccf","mcp_get_code":{"code_sha256":"b1a674018a8b1ccf"}},{"arxiv_id":"2402.12052","paper":"/paper/small-models-big-insights-leveraging-slim","title":"Small Models, Big Insights: Leveraging Slim Proxy Models To Decide When and What to Retrieve for LLMs","date":"2024-02-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"plageon/slimplm","path":"SKR/skr.py","file_url":"https://github.com/plageon/slimplm/blob/HEAD/SKR/skr.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ae914f80513baf0f","mcp_get_code":{"code_sha256":"ae914f80513baf0f"}},{"arxiv_id":"2402.11893","paper":"/paper/discerning-and-resolving-knowledge-conflicts","title":"Discerning and Resolving Knowledge Conflicts through Adaptive Decoding with Contextual Information-Entropy Constraint","date":"2024-02-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2402.11626","paper":"/paper/metacognitive-retrieval-augmented-large","title":"Metacognitive Retrieval-Augmented Large Language Models","date":"2024-02-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ignorejjj/metarag","path":"utils.py","file_url":"https://github.com/ignorejjj/metarag/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2402.11592","paper":"/paper/revisiting-zeroth-order-optimization-for","title":"Revisiting Zeroth-Order Optimization for Memory-Efficient LLM Fine-Tuning: A Benchmark","date":"2024-02-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zo-bench/zo-llm","path":"zo-bench/metrics.py","file_url":"https://github.com/zo-bench/zo-llm/blob/HEAD/zo-bench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2402.11534","paper":"/paper/preact-predicting-future-in-react-enhances","title":"PreAct: Prediction Enhances Agent's Planning Ability","date":"2024-02-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"fu-dayuan/preact","path":"LanguageAgentTreeSearch/hotpot/wrappers.py","file_url":"https://github.com/fu-dayuan/preact/blob/HEAD/LanguageAgentTreeSearch/hotpot/wrappers.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2402.11417","paper":"/paper/loretta-low-rank-economic-tensor-train","title":"LoRETTA: Low-Rank Economic Tensor-Train Adaptation for Ultra-Low-Parameter Fine-Tuning of Large Language Models","date":"2024-02-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yifanycc/loretta","path":"bert_model/metrics.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/bert_model/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2402.04617","paper":"/paper/infllm-unveiling-the-intrinsic-capacity-of","title":"InfLLM: Training-Free Long-Context Extrapolation for LLMs with an Efficient Context Memory","date":"2024-02-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thunlp/infllm","path":"benchmark/metrics.py","file_url":"https://github.com/thunlp/infllm/blob/HEAD/benchmark/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2402.04617","paper":"/paper/infllm-unveiling-the-intrinsic-capacity-of","title":"InfLLM: Training-Free Long-Context Extrapolation for LLMs with an Efficient Context Memory","date":"2024-02-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thunlp/infllm","path":"benchmark/infinitebench_eval.py","file_url":"https://github.com/thunlp/infllm/blob/HEAD/benchmark/infinitebench_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4b44a4d3fd80a48d","mcp_get_code":{"code_sha256":"4b44a4d3fd80a48d"}},{"arxiv_id":"2402.04315","paper":"/paper/training-language-models-to-generate-text","title":"Training Language Models to Generate Text with Citations via Fine-grained Rewards","date":"2024-02-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hcy123902/atg-w-fg-rw","path":"evaluation/eval_utils.py","file_url":"https://github.com/hcy123902/atg-w-fg-rw/blob/HEAD/evaluation/eval_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2402.02750","paper":"/paper/kivi-a-tuning-free-asymmetric-2bit","title":"KIVI: A Tuning-Free Asymmetric 2bit Quantization for KV Cache","date":"2024-02-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jy-yuan/kivi","path":"metrics.py","file_url":"https://github.com/jy-yuan/kivi/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2402.02503","paper":"/paper/gerea-question-aware-prompt-captions-for","title":"GeReA: Question-Aware Prompt Captions for Knowledge-based Visual Question Answering","date":"2024-02-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"upper9527/gerea","path":"leaderboard_evaluation.py","file_url":"https://github.com/upper9527/gerea/blob/HEAD/leaderboard_evaluation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"b92b07bb3574992a","mcp_get_code":{"code_sha256":"b92b07bb3574992a"}},{"arxiv_id":"2402.01032","paper":"/paper/repeat-after-me-transformers-are-better-than","title":"Repeat After Me: Transformers are Better than State Space Models at Copying","date":"2024-02-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sjelassi/transformers_ssm_copy","path":"pretrained_exps/qa_evaluation_utils.py","file_url":"https://github.com/sjelassi/transformers_ssm_copy/blob/HEAD/pretrained_exps/qa_evaluation_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c29fc928dae85749","mcp_get_code":{"code_sha256":"c29fc928dae85749"}},{"arxiv_id":"2401.13246","paper":"/paper/seer-facilitating-structured-reasoning-and","title":"SEER: Facilitating Structured Reasoning and Explanation via Reinforcement Learning","date":"2024-01-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2401.12585","paper":"/paper/slang-new-concept-comprehension-of-large","title":"SLANG: New Concept Comprehension of Large Language Models","date":"2024-01-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"meirtz/focusonslang-toolbox","path":"eval_urban_f1.py","file_url":"https://github.com/meirtz/focusonslang-toolbox/blob/HEAD/eval_urban_f1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2401.11911","paper":"/paper/blinded-by-generated-contexts-how-language","title":"Blinded by Generated Contexts: How Language Models Merge Generated and Retrieved Contexts When Knowledge Conflicts?","date":"2024-01-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tan-hexiang/retrieveorgenerated","path":"src/evaluation.py","file_url":"https://github.com/tan-hexiang/retrieveorgenerated/blob/HEAD/src/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"4cca112e5e6da5dc","mcp_get_code":{"code_sha256":"4cca112e5e6da5dc"}},{"arxiv_id":"2401.07977","paper":"/paper/leveraging-external-knowledge-resources-to","title":"Towards Efficient Methods in Medical Question Answering using Knowledge Graph Embeddings","date":"2024-01-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"saptarshi059/cdqa-project","path":"covid_qa_baseline.py","file_url":"https://github.com/saptarshi059/cdqa-project/blob/HEAD/covid_qa_baseline.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2312.15184","paper":"/paper/zo-adamu-optimizer-adapting-perturbation-by","title":"ZO-AdaMU Optimizer: Adapting Perturbation by the Momentum and Uncertainty in Zeroth-order Optimization","date":"2023-12-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mathisall/zo-adamu","path":"metrics.py","file_url":"https://github.com/mathisall/zo-adamu/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2312.07000","paper":"/paper/alignment-for-honesty","title":"Alignment for Honesty","date":"2023-12-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"GAIR-NLP/alignment-for-honesty","path":"evaluation/triviaqa_nonambigqa_evaluation.py","file_url":"https://github.com/GAIR-NLP/alignment-for-honesty/blob/HEAD/evaluation/triviaqa_nonambigqa_evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c29fc928dae85749","mcp_get_code":{"code_sha256":"c29fc928dae85749"}},{"arxiv_id":"2312.02614","paper":"/paper/prompt-optimization-via-adversarial-in","title":"Prompt Optimization via Adversarial In-Context Learning","date":"2023-12-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhaoyiran924/adv-in-context-learning","path":"eval/automatic/evaluation.py","file_url":"https://github.com/zhaoyiran924/adv-in-context-learning/blob/HEAD/eval/automatic/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0cb6cc3626c8f2ec","mcp_get_code":{"code_sha256":"0cb6cc3626c8f2ec"}},{"arxiv_id":"2311.09821","paper":"/paper/towards-robust-temporal-reasoning-of-large","title":"Towards Robust Temporal Reasoning of Large Language Models via a Multi-Hop QA Dataset and Pseudo-Instruction Tuning","date":"2023-11-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nusnlp/complex-tr","path":"prompt_experiment/sem_metric/evaluate.py","file_url":"https://github.com/nusnlp/complex-tr/blob/HEAD/prompt_experiment/sem_metric/evaluate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2d391d2daa3c57fe","mcp_get_code":{"code_sha256":"2d391d2daa3c57fe"}},{"arxiv_id":"2311.09198","paper":"/paper/never-lost-in-the-middle-improving-large","title":"Never Lost in the Middle: Mastering Long-Context Question Answering with Position-Agnostic Decompositional Training","date":"2023-11-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hejunqing/never-lost-in-the-middle","path":"LongBench/src/metrics.py","file_url":"https://github.com/hejunqing/never-lost-in-the-middle/blob/HEAD/LongBench/src/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2311.07838","paper":"/paper/llatrieval-llm-verified-retrieval-for","title":"LLatrieval: LLM-Verified Retrieval for Verifiable Generation","date":"2023-11-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"beastyz/llm-verified-retrieval","path":"utils.py","file_url":"https://github.com/beastyz/llm-verified-retrieval/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2311.04934","paper":"/paper/prompt-cache-modular-attention-reuse-for-low","title":"Prompt Cache: Modular Attention Reuse for Low-Latency Inference","date":"2023-11-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yale-sys/prompt-cache","path":"metrics.py","file_url":"https://github.com/yale-sys/prompt-cache/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2311.03734","paper":"/paper/leveraging-structured-information-for","title":"Leveraging Structured Information for Explainable Multi-hop Question Answering and Reasoning","date":"2023-11-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bcdnlp/structure-qa","path":"src/hotpot_evaluate.py","file_url":"https://github.com/bcdnlp/structure-qa/blob/HEAD/src/hotpot_evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2311.00288","paper":"/paper/active-instruction-tuning-improving-cross","title":"Active Instruction Tuning: Improving Cross-Task Generalization by Training on Prompt Sensitive Tasks","date":"2023-11-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"pluslabnlp/active-it","path":"ActiveIT/src/compute_metrics.py","file_url":"https://github.com/pluslabnlp/active-it/blob/HEAD/ActiveIT/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2310.14503","paper":"/paper/diversify-question-generation-with-retrieval","title":"Diversify Question Generation with Retrieval-Augmented Style Transfer","date":"2023-10-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"gouqi666/RAST","path":"eval_squad.py","file_url":"https://github.com/gouqi666/RAST/blob/HEAD/eval_squad.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"5c30e376e356eaca","mcp_get_code":{"code_sha256":"5c30e376e356eaca"}},{"arxiv_id":"2310.14393","paper":"/paper/merging-generated-and-retrieved-knowledge-for","title":"Merging Generated and Retrieved Knowledge for Open-Domain QA","date":"2023-10-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yunx-z/combo","path":"passage_matching/evaluation.py","file_url":"https://github.com/yunx-z/combo/blob/HEAD/passage_matching/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"4cca112e5e6da5dc","mcp_get_code":{"code_sha256":"4cca112e5e6da5dc"}},{"arxiv_id":"2310.14152","paper":"/paper/orthogonal-subspace-learning-for-language","title":"Orthogonal Subspace Learning for Language Model Continual Learning","date":"2023-10-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"cmnfriend/o-lora","path":"src/compute_metrics.py","file_url":"https://github.com/cmnfriend/o-lora/blob/HEAD/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2310.12836","paper":"/paper/knowledge-augmented-language-model","title":"Knowledge-Augmented Language Model Verification","date":"2023-10-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jinheonbaek/kalmv","path":"metrics.py","file_url":"https://github.com/jinheonbaek/kalmv/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2310.12836","paper":"/paper/knowledge-augmented-language-model","title":"Knowledge-Augmented Language Model Verification","date":"2023-10-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"JinheonBaek/KALMV","path":"models/verifiers.py","file_url":"https://github.com/JinheonBaek/KALMV/blob/HEAD/models/verifiers.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"4d2f27c630abe8bf","mcp_get_code":{"code_sha256":"4d2f27c630abe8bf"}},{"arxiv_id":"2310.09639","paper":"/paper/dpzero-dimension-independent-and","title":"DPZero: Private Fine-Tuning of Language Models without Backpropagation","date":"2023-10-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Liang137/DPZero","path":"opt/src/metrics.py","file_url":"https://github.com/Liang137/DPZero/blob/HEAD/opt/src/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2310.05157","paper":"/paper/menatqa-a-new-dataset-for-testing-the","title":"MenatQA: A New Dataset for Testing the Temporal Comprehension and Reasoning Abilities of Large Language Models","date":"2023-10-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"weiyifan1023/MenatQA","path":"src/utils.py","file_url":"https://github.com/weiyifan1023/MenatQA/blob/HEAD/src/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2310.04408","paper":"/paper/recomp-improving-retrieval-augmented-lms-with","title":"RECOMP: Improving Retrieval-Augmented LMs with Compression and Selective Augmentation","date":"2023-10-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"carriex/recomp","path":"eval_utils.py","file_url":"https://github.com/carriex/recomp/blob/HEAD/eval_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"55e919890251f878","mcp_get_code":{"code_sha256":"55e919890251f878"}},{"arxiv_id":"2310.04406","paper":"/paper/language-agent-tree-search-unifies-reasoning","title":"Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models","date":"2023-10-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"andyz245/LanguageAgentTreeSearch","path":"hotpot/wrappers.py","file_url":"https://github.com/andyz245/LanguageAgentTreeSearch/blob/HEAD/hotpot/wrappers.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2309.07822","paper":"/paper/catfood-counterfactual-augmented-training-for","title":"CATfOOD: Counterfactual Augmented Training for Improving Out-of-Domain Performance and Calibration","date":"2023-09-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ukplab/catfood","path":"src/shortcuts/evaluate.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/shortcuts/evaluate.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"89c5a532983ce170","mcp_get_code":{"code_sha256":"89c5a532983ce170"}},{"arxiv_id":"2309.07822","paper":"/paper/catfood-counterfactual-augmented-training-for","title":"CATfOOD: Counterfactual Augmented Training for Improving Out-of-Domain Performance and Calibration","date":"2023-09-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ukplab/catfood","path":"src/calibration/baseline/calibration_metrics.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/calibration/baseline/calibration_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4f656af1dfa55eb8","mcp_get_code":{"code_sha256":"4f656af1dfa55eb8"}},{"arxiv_id":"2309.00667","paper":"/paper/taken-out-of-context-on-measuring-situational","title":"Taken out of context: On measuring situational awareness in LLMs","date":"2023-09-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"asacooperstickland/situational-awareness-evals","path":"sitaevals/models/common.py","file_url":"https://github.com/asacooperstickland/situational-awareness-evals/blob/HEAD/sitaevals/models/common.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2308.16475","paper":"/paper/transformer-compression-via-subspace","title":"$\\rm SP^3$: Enhancing Structured Pruning via PCA Projection","date":"2023-08-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hyx1999/sp3","path":"bert/metric/squad.py","file_url":"https://github.com/hyx1999/sp3/blob/HEAD/bert/metric/squad.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2308.16475","paper":"/paper/transformer-compression-via-subspace","title":"$\\rm SP^3$: Enhancing Structured Pruning via PCA Projection","date":"2023-08-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hyx1999/sp3","path":"bert/metric/squad_v2.py","file_url":"https://github.com/hyx1999/sp3/blob/HEAD/bert/metric/squad_v2.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2d391d2daa3c57fe","mcp_get_code":{"code_sha256":"2d391d2daa3c57fe"}},{"arxiv_id":"2308.11730","paper":"/paper/knowledge-graph-prompting-for-multi-document","title":"Knowledge Graph Prompting for Multi-Document Question Answering","date":"2023-08-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yuwvandy/kg-llm-mdqa","path":"DPR/evaluation.py","file_url":"https://github.com/yuwvandy/kg-llm-mdqa/blob/HEAD/DPR/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"4cca112e5e6da5dc","mcp_get_code":{"code_sha256":"4cca112e5e6da5dc"}},{"arxiv_id":"2308.10819","paper":"/paper/do-you-really-follow-me-adversarial","title":"Evaluating the Instruction-Following Robustness of Large Language Models to Prompt Injection","date":"2023-08-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"leezekun/instruction-following-robustness-eval","path":"qa_utils.py","file_url":"https://github.com/leezekun/instruction-following-robustness-eval/blob/HEAD/qa_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2308.08973","paper":"/paper/beam-retrieval-general-end-to-end-retrieval","title":"End-to-End Beam Retrieval for Multi-Hop Question Answering","date":"2023-08-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"canghongjian/beam_retriever","path":"gpt_turbo_exp.py","file_url":"https://github.com/canghongjian/beam_retriever/blob/HEAD/gpt_turbo_exp.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2308.07922","paper":"/paper/raven-in-context-learning-with-retrieval","title":"RAVEN: In-Context Learning with Retrieval-Augmented Encoder-Decoder Language Models","date":"2023-08-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jeffhj/raven","path":"src/evaluation.py","file_url":"https://github.com/jeffhj/raven/blob/HEAD/src/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6205dd62c2627dde","mcp_get_code":{"code_sha256":"6205dd62c2627dde"}},{"arxiv_id":"2308.03279","paper":"/paper/universalner-targeted-distillation-from-large","title":"UniversalNER: Targeted Distillation from Large Language Models for Open Named Entity Recognition","date":"2023-08-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"universal-ner/universal-ner","path":"src/eval/evaluate.py","file_url":"https://github.com/universal-ner/universal-ner/blob/HEAD/src/eval/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2306.13304","paper":"/paper/toolqa-a-dataset-for-llm-question-answering-1","title":"ToolQA: A Dataset for LLM Question Answering with External Tools","date":"2023-06-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"night-chen/toolqa","path":"benchmark/ReAct/code/environment.py","file_url":"https://github.com/night-chen/toolqa/blob/HEAD/benchmark/ReAct/code/environment.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"3f453625dacae10a","mcp_get_code":{"code_sha256":"3f453625dacae10a"}},{"arxiv_id":"2306.06191","paper":"/paper/open-data-on-github-unlocking-the-potential","title":"Open Data on GitHub: Unlocking the Potential of AI","date":"2023-06-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rajpurkar/SQuAD-explorer","path":"evaluate-v2.0.py","file_url":"https://github.com/rajpurkar/SQuAD-explorer/blob/HEAD/evaluate-v2.0.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2305.17333","paper":"/paper/fine-tuning-language-models-with-just-forward-1","title":"Fine-Tuning Language Models with Just Forward Passes","date":"2023-05-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-nlp/mezo","path":"large_models/metrics.py","file_url":"https://github.com/princeton-nlp/mezo/blob/HEAD/large_models/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2305.15387","paper":"/paper/peek-across-improving-multi-document-modeling","title":"Peek Across: Improving Multi-Document Modeling via Cross-Document Question-Answering","date":"2023-05-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"aviclu/peekacross","path":"trainer_seq2seq_qa.py","file_url":"https://github.com/aviclu/peekacross/blob/HEAD/trainer_seq2seq_qa.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"83cfdd6d3dfa70e0","mcp_get_code":{"code_sha256":"83cfdd6d3dfa70e0"}},{"arxiv_id":"2305.14627","paper":"/paper/enabling-large-language-models-to-generate","title":"Enabling Large Language Models to Generate Text with Citations","date":"2023-05-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-nlp/alce","path":"utils.py","file_url":"https://github.com/princeton-nlp/alce/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2305.14251","paper":"/paper/factscore-fine-grained-atomic-evaluation-of","title":"FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form Text Generation","date":"2023-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shmsw25/factscore","path":"factscore/atomic_facts.py","file_url":"https://github.com/shmsw25/factscore/blob/HEAD/factscore/atomic_facts.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2305.12421","paper":"/paper/evaluating-open-qa-evaluation-1","title":"Evaluating Open-QA Evaluation","date":"2023-05-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"wangcunxiang/QA-Eval","path":"lexical_match.py","file_url":"https://github.com/wangcunxiang/QA-Eval/blob/HEAD/lexical_match.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"adbb7ae4b3f749dd","mcp_get_code":{"code_sha256":"adbb7ae4b3f749dd"}},{"arxiv_id":"2305.09955","paper":"/paper/cook-empowering-general-purpose-language","title":"Knowledge Card: Filling LLMs' Knowledge Gaps with Plug-in Specialized Language Models","date":"2023-05-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bunsenfeng/knowledge_card","path":"eval_datasets/MidtermQA/odqa_utils.py","file_url":"https://github.com/bunsenfeng/knowledge_card/blob/HEAD/eval_datasets/MidtermQA/odqa_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"55e919890251f878","mcp_get_code":{"code_sha256":"55e919890251f878"}},{"arxiv_id":"2305.06984","paper":"/paper/evaluating-open-domain-question-answering-in","title":"Evaluating Open-Domain Question Answering in the Era of Large Language Models","date":"2023-05-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ehsk/openqa-eval","path":"src/oqaeval/squad_evaluate.py","file_url":"https://github.com/ehsk/openqa-eval/blob/HEAD/src/oqaeval/squad_evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2305.00944","paper":"/paper/poisoning-language-models-during-instruction","title":"Poisoning Language Models During Instruction Tuning","date":"2023-05-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"alexwan0/poisoning-instruction-tuned-models","path":"src/compute_metrics.py","file_url":"https://github.com/alexwan0/poisoning-instruction-tuned-models/blob/HEAD/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2303.11366","paper":"/paper/reflexion-language-agents-with-verbal","title":"Reflexion: Language Agents with Verbal Reinforcement Learning","date":"2023-03-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"noahshinn/reflexion","path":"hotpotqa_runs/environment.py","file_url":"https://github.com/noahshinn/reflexion/blob/HEAD/hotpotqa_runs/environment.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2302.14691","paper":"/paper/in-context-instruction-learning","title":"Investigating the Effectiveness of Task-Agnostic Prefix Prompt for Instruction Following","date":"2023-02-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"seonghyeonye/icil","path":"src/compute_metrics.py","file_url":"https://github.com/seonghyeonye/icil/blob/HEAD/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2302.11713","paper":"/paper/can-pre-trained-vision-and-language-models","title":"Can Pre-trained Vision and Language Models Answer Visual Information-Seeking Questions?","date":"2023-02-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"edchengg/infoseek_eval","path":"infoseek_eval.py","file_url":"https://github.com/edchengg/infoseek_eval/blob/HEAD/infoseek_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4925192ddc8dfa9a","mcp_get_code":{"code_sha256":"4925192ddc8dfa9a"}},{"arxiv_id":"2302.00083","paper":"/paper/in-context-retrieval-augmented-language","title":"In-Context Retrieval-Augmented Language Models","date":"2023-01-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2212.07919","paper":"/paper/roscoe-a-suite-of-metrics-for-scoring-step-by","title":"ROSCOE: A Suite of Metrics for Scoring Step-by-Step Reasoning","date":"2022-12-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"facebookresearch/ParlAI","path":"parlai/core/metrics.py","file_url":"https://github.com/facebookresearch/ParlAI/blob/HEAD/parlai/core/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"code_sha256_prefix":"da3bc8c42eec18d1","mcp_get_code":{"code_sha256":"da3bc8c42eec18d1"}},{"arxiv_id":"2212.03813","paper":"/paper/robustness-of-learning-from-task-instructions","title":"Robustness of Learning from Task Instructions","date":"2022-12-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jiashenggu/tk-instruct","path":"src/compute_metrics.py","file_url":"https://github.com/jiashenggu/tk-instruct/blob/HEAD/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2210.10861","paper":"/paper/qa-domain-adaptation-using-hidden-space","title":"QA Domain Adaptation using Hidden Space Augmentation and Self-Supervised Contrastive Adaptation","date":"2022-10-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"2210.03629","paper":"/paper/react-synergizing-reasoning-and-acting-in","title":"ReAct: Synergizing Reasoning and Acting in Language Models","date":"2022-10-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ysymyth/ReAct","path":"wrappers.py","file_url":"https://github.com/ysymyth/ReAct/blob/HEAD/wrappers.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2209.14610","paper":"/paper/dynamic-prompt-learning-via-policy-gradient","title":"Dynamic Prompt Learning via Policy Gradient for Semi-structured Mathematical Reasoning","date":"2022-09-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lupantech/promptpg","path":"run_gpt3_rl/learn_policy.py","file_url":"https://github.com/lupantech/promptpg/blob/HEAD/run_gpt3_rl/learn_policy.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"580caf49d1cd4e8e","mcp_get_code":{"code_sha256":"580caf49d1cd4e8e"}},{"arxiv_id":"2208.07652","paper":"/paper/corpusbrain-pre-train-a-generative-retrieval","title":"CorpusBrain: Pre-train a Generative Retrieval Model for Knowledge-Intensive Language Tasks","date":"2022-08-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ict-bigdatalab/corpusbrain","path":"kilt/eval_downstream.py","file_url":"https://github.com/ict-bigdatalab/corpusbrain/blob/HEAD/kilt/eval_downstream.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2207.06300","paper":"/paper/re2g-retrieve-rerank-generate-2","title":"Re2G: Retrieve, Rerank, Generate","date":"2022-07-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ibm/kgi-slot-filling","path":"slot_filling/rag_util.py","file_url":"https://github.com/ibm/kgi-slot-filling/blob/HEAD/slot_filling/rag_util.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2205.12609","paper":"/paper/towards-more-realistic-generation-of","title":"Generating Information-Seeking Conversations from Unlabeled Documents","date":"2022-05-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"facebookresearch/KILT","path":"kilt/eval_downstream.py","file_url":"https://github.com/facebookresearch/KILT/blob/HEAD/kilt/eval_downstream.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2205.12609","paper":"/paper/towards-more-realistic-generation-of","title":"Generating Information-Seeking Conversations from Unlabeled Documents","date":"2022-05-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"facebookresearch/KILT","path":"kilt/kilt_utils.py","file_url":"https://github.com/facebookresearch/KILT/blob/HEAD/kilt/kilt_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"code_sha256_prefix":"527517ab30580dfd","mcp_get_code":{"code_sha256":"527517ab30580dfd"}},{"arxiv_id":"2205.06435","paper":"/paper/tie-topological-information-enhanced","title":"TIE: Topological Information Enhanced Structural Reading Comprehension on Web Pages","date":"2022-05-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"x-lance/tie","path":"src/utils_evaluate.py","file_url":"https://github.com/x-lance/tie/blob/HEAD/src/utils_evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2204.07705","paper":"/paper/benchmarking-generalization-via-in-context","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","date":"2022-04-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"renzelou/pick-rank","path":"src/compute_metrics.py","file_url":"https://github.com/renzelou/pick-rank/blob/HEAD/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2109.13880","paper":"/paper/single-dataset-experts-for-multi-dataset","title":"Single-dataset Experts for Multi-dataset Question Answering","date":"2021-09-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-nlp/MADE","path":"src/utils/metrics.py","file_url":"https://github.com/princeton-nlp/MADE/blob/HEAD/src/utils/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2109.08133","paper":"/paper/phrase-retrieval-learns-passage-retrieval-too","title":"Phrase Retrieval Learns Passage Retrieval, Too","date":"2021-09-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-nlp/DensePhrases","path":"densephrases/utils/eval_utils.py","file_url":"https://github.com/princeton-nlp/DensePhrases/blob/HEAD/densephrases/utils/eval_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2109.08133","paper":"/paper/phrase-retrieval-learns-passage-retrieval-too","title":"Phrase Retrieval Learns Passage Retrieval, Too","date":"2021-09-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-nlp/DensePhrases","path":"densephrases/utils/squad_metrics.py","file_url":"https://github.com/princeton-nlp/DensePhrases/blob/HEAD/densephrases/utils/squad_metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2109.04912","paper":"/paper/reasonbert-pre-trained-to-reason-with-distant","title":"ReasonBERT: Pre-trained to Reason with Distant Supervision","date":"2021-09-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sunlab-osu/reasonbert","path":"model/metric.py","file_url":"https://github.com/sunlab-osu/reasonbert/blob/HEAD/model/metric.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"2108.13934","paper":"/paper/robust-retrieval-augmented-generation-for","title":"Robust Retrieval Augmented Generation for Zero-shot Slot Filling","date":"2021-08-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"IBM/retrieve-write-slot-filling","path":"slot_filling/rag_util.py","file_url":"https://github.com/IBM/retrieve-write-slot-filling/blob/HEAD/slot_filling/rag_util.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2108.06314","paper":"/paper/a-dataset-for-answering-time-sensitive","title":"A Dataset for Answering Time-Sensitive Questions","date":"2021-08-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"wenhuchen/time-sensitive-qa","path":"utils.py","file_url":"https://github.com/wenhuchen/time-sensitive-qa/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2108.00573","paper":"/paper/musique-multi-hop-questions-via-single-hop","title":"MuSiQue: Multihop Questions via Single-hop Question Composition","date":"2021-08-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mangopy/direct-rag-learning","path":"src/utilize/metrics.py","file_url":"https://github.com/mangopy/direct-rag-learning/blob/HEAD/src/utilize/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"97cb57dd2386abcb","mcp_get_code":{"code_sha256":"97cb57dd2386abcb"}},{"arxiv_id":"2106.03634","paper":"/paper/prost-physical-reasoning-of-objects-through","title":"PROST: Physical Reasoning of Objects through Space and Time","date":"2021-06-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"nala-cub/prost","path":"src/baselines/unifiedqa.py","file_url":"https://github.com/nala-cub/prost/blob/HEAD/src/baselines/unifiedqa.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"77c174355b5aa8d2","mcp_get_code":{"code_sha256":"77c174355b5aa8d2"}},{"arxiv_id":"2105.02692","paper":"/paper/learning-to-perturb-word-embeddings-for-out","title":"Learning to Perturb Word Embeddings for Out-of-distribution QA","date":"2021-05-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"seanie12/SWEP","path":"mrqa_utils.py","file_url":"https://github.com/seanie12/SWEP/blob/HEAD/mrqa_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2105.02692","paper":"/paper/learning-to-perturb-word-embeddings-for-out","title":"Learning to Perturb Word Embeddings for Out-of-distribution QA","date":"2021-05-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"seanie12/SWEP","path":"squad_metrics.py","file_url":"https://github.com/seanie12/SWEP/blob/HEAD/squad_metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2104.00640","paper":"/paper/evidence-based-verification-for-real-world","title":"AmbiFC: Fact-Checking Ambiguous Claims with Evidence","date":"2021-04-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"knot-fit-but/claimdissector","path":"src/common/eval_utils.py","file_url":"https://github.com/knot-fit-but/claimdissector/blob/HEAD/src/common/eval_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"8ddaf8aae0f76d1e","mcp_get_code":{"code_sha256":"8ddaf8aae0f76d1e"}},{"arxiv_id":"2101.10421","paper":"/paper/english-machine-reading-comprehension","title":"English Machine Reading Comprehension Datasets: A Survey","date":"2021-01-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"2879260bc19381c8","mcp_get_code":{"code_sha256":"2879260bc19381c8"}},{"arxiv_id":"2011.01060","paper":"/paper/constructing-a-multi-hop-qa-dataset-for","title":"Constructing A Multi-hop QA Dataset for Comprehensive Evaluation of Reasoning Steps","date":"2020-11-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2010.16021","paper":"/paper/cliniqg4qa-generating-diverse-questions-for","title":"CliniQG4QA: Generating Diverse Questions for Domain Adaptation of Clinical Question Answering","date":"2020-10-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2010.12527","paper":"/paper/retrieve-rerank-read-then-iterate-answering","title":"Answering Open-Domain Questions of Varying Reasoning Steps from Text","date":"2020-10-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"2010.11856","paper":"/paper/xor-qa-cross-lingual-open-retrieval-question","title":"XOR QA: Cross-lingual Open-Retrieval Question Answering","date":"2020-10-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mia-workshop/mia-shared-task-2022","path":"eval_scripts/eval_xor_full.py","file_url":"https://github.com/mia-workshop/mia-shared-task-2022/blob/HEAD/eval_scripts/eval_xor_full.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ed828b1e70bdbb40","mcp_get_code":{"code_sha256":"ed828b1e70bdbb40"}},{"arxiv_id":"2010.08983","paper":"/paper/towards-interpreting-bert-for-reading","title":"Towards Interpreting BERT for Reading Comprehension Based QA","date":"2020-10-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2010.04898","paper":"/paper/open-domain-question-answering-goes","title":"Open-Domain Question Answering Goes Conversational via Question Rewriting","date":"2020-10-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"apple/ml-qrecc","path":"utils/evaluate_qa.py","file_url":"https://github.com/apple/ml-qrecc/blob/HEAD/utils/evaluate_qa.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6e13a9c4b963076","mcp_get_code":{"code_sha256":"c6e13a9c4b963076"}},{"arxiv_id":"2008.02637","paper":"/paper/question-and-answer-test-train-overlap-in","title":"Question and Answer Test-Train Overlap in Open-Domain Question Answering Datasets","date":"2020-08-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"facebookresearch/QA-Overlap","path":"evaluate.py","file_url":"https://github.com/facebookresearch/QA-Overlap/blob/HEAD/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"2006.12467","paper":"/paper/limits-to-depth-efficiencies-of-self","title":"The Depth-to-Width Interplay in Self-Attention","date":"2020-06-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"uf-hobi-informatics-lab/GatorTron","path":"finetuning/qa/evaluate-v1.1.py","file_url":"https://github.com/uf-hobi-informatics-lab/GatorTron/blob/HEAD/finetuning/qa/evaluate-v1.1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2006.12467","paper":"/paper/limits-to-depth-efficiencies-of-self","title":"The Depth-to-Width Interplay in Self-Attention","date":"2020-06-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"uf-hobi-informatics-lab/GatorTron","path":"finetuning/qa/evaluate-squad-v2.0.py","file_url":"https://github.com/uf-hobi-informatics-lab/GatorTron/blob/HEAD/finetuning/qa/evaluate-squad-v2.0.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2005.09123","paper":"/paper/gpt-too-a-language-model-first-approach-for","title":"GPT-too: A language-model-first approach for AMR-to-text generation","date":"2020-05-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"IBM/GPT-too-AMR2text","path":"utils.py","file_url":"https://github.com/IBM/GPT-too-AMR2text/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2005.00806","paper":"/paper/teaching-machine-comprehension-with","title":"Teaching Machine Comprehension with Compositional Explanations","date":"2020-05-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2004.13922","paper":"/paper/revisiting-pre-trained-models-for-chinese","title":"Revisiting Pre-Trained Models for Chinese Natural Language Processing","date":"2020-04-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ymcui/Chinese-PreTrained-XLNet","path":"src/squad_utils.py","file_url":"https://github.com/ymcui/Chinese-PreTrained-XLNet/blob/HEAD/src/squad_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2004.07347","paper":"/paper/hybridqa-a-dataset-of-multi-hop-question","title":"HybridQA: A Dataset of Multi-Hop Question Answering over Tabular and Textual Data","date":"2020-04-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2003.11963","paper":"/paper/tldr-token-loss-dynamic-reweighting-for","title":"TLDR: Token Loss Dynamic Reweighting for Reducing Repetitive Utterance Generation","date":"2020-03-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ShaojieJiang/tldr","path":"parlai/core/metrics.py","file_url":"https://github.com/ShaojieJiang/tldr/blob/HEAD/parlai/core/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"code_sha256_prefix":"05b2d71076472acd","mcp_get_code":{"code_sha256":"05b2d71076472acd"}},{"arxiv_id":"1911.10470","paper":"/paper/learning-to-retrieve-reasoning-paths-over-1","title":"Learning to Retrieve Reasoning Paths over Wikipedia Graph for Question Answering","date":"2019-11-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"AkariAsai/learning_to_retrieve_reasoning_paths","path":"eval_utils.py","file_url":"https://github.com/AkariAsai/learning_to_retrieve_reasoning_paths/blob/HEAD/eval_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1911.03868","paper":"/paper/knowledge-guided-text-retrieval-and-reading","title":"Knowledge Guided Text Retrieval and Reading for Open Domain Question Answering","date":"2019-11-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shmsw25/GraphRetriever","path":"WikiData.py","file_url":"https://github.com/shmsw25/GraphRetriever/blob/HEAD/WikiData.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"1911.02896","paper":"/paper/contextualized-sparse-representation-with-1","title":"Contextualized Sparse Representations for Real-Time Open-Domain Question Answering","date":"2019-11-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jhyuklee/sparc","path":"eval_utils.py","file_url":"https://github.com/jhyuklee/sparc/blob/HEAD/eval_utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"1911.02896","paper":"/paper/contextualized-sparse-representation-with-1","title":"Contextualized Sparse Representations for Real-Time Open-Domain Question Answering","date":"2019-11-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jhyuklee/sparc","path":"evaluate-v1.1.py","file_url":"https://github.com/jhyuklee/sparc/blob/HEAD/evaluate-v1.1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1911.00484","paper":"/paper/select-answer-and-explain-interpretable-multi","title":"Select, Answer and Explain: Interpretable Multi-hop Reading Comprehension over Multiple Documents","date":"2019-11-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jd-ai-research-silicon-valley/sae","path":"hotpotqa_utils_joint.py","file_url":"https://github.com/jd-ai-research-silicon-valley/sae/blob/HEAD/hotpotqa_utils_joint.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"58c6794946fd1e26","mcp_get_code":{"code_sha256":"58c6794946fd1e26"}},{"arxiv_id":"1910.14520","paper":"/paper/do-multi-hop-readers-dream-of-reasoning","title":"Do Multi-hop Readers Dream of Reasoning Chains?","date":"2019-10-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"helloeve/bert-co-matching","path":"evaluate-v1.1-original.py","file_url":"https://github.com/helloeve/bert-co-matching/blob/HEAD/evaluate-v1.1-original.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1910.09753","paper":"/paper/mrqa-2019-shared-task-evaluating","title":"MRQA 2019 Shared Task: Evaluating Generalization in Reading Comprehension","date":"2019-10-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mrqa/MRQA-Shared-Task-2019","path":"mrqa_official_eval.py","file_url":"https://github.com/mrqa/MRQA-Shared-Task-2019/blob/HEAD/mrqa_official_eval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1910.07475","paper":"/paper/mlqa-evaluating-cross-lingual-extractive","title":"MLQA: Evaluating Cross-lingual Extractive Question Answering","date":"2019-10-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"facebookresearch/MLQA","path":"mlqa_evaluation_v1.py","file_url":"https://github.com/facebookresearch/MLQA/blob/HEAD/mlqa_evaluation_v1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"9cddf0b1a2750978","mcp_get_code":{"code_sha256":"9cddf0b1a2750978"}},{"arxiv_id":"1910.01244","paper":"/paper/linking-artificial-and-human-neural","title":"Linking artificial and human neural representations of language","date":"2019-10-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hans/nn-decoding","path":"bin/eval_squad.py","file_url":"https://github.com/hans/nn-decoding/blob/HEAD/bin/eval_squad.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"1909.06356","paper":"/paper/addressing-semantic-drift-in-question","title":"Addressing Semantic Drift in Question Generation for Semi-Supervised Question Answering","date":"2019-09-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ZhangShiyue/QGforQA","path":"LIB/EVAL/evaluate.py","file_url":"https://github.com/ZhangShiyue/QGforQA/blob/HEAD/LIB/EVAL/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1909.05863","paper":"/paper/finding-generalizable-evidence-by-learning-to","title":"Finding Generalizable Evidence by Learning to Convince Q&A Models","date":"2019-09-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ethanjperez/convince","path":"ParlAI/parlai/core/metrics.py","file_url":"https://github.com/ethanjperez/convince/blob/HEAD/ParlAI/parlai/core/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"38a6695b805ebd76","mcp_get_code":{"code_sha256":"38a6695b805ebd76"}},{"arxiv_id":"1909.05803","paper":"/paper/self-assembling-modular-networks-for","title":"Self-Assembling Modular Networks for Interpretable Multi-Hop Reasoning","date":"2019-09-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jiangycTarheel/NMN-MultiHopQA","path":"hotpotqa/evaluate-v1.1.py","file_url":"https://github.com/jiangycTarheel/NMN-MultiHopQA/blob/HEAD/hotpotqa/evaluate-v1.1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1909.01610","paper":"/paper/answers-unite-unsupervised-metrics-for","title":"Answers Unite! Unsupervised Metrics for Reinforced Summarization Models","date":"2019-09-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"recitalAI/summa-qa","path":"summaqa/f1_squad.py","file_url":"https://github.com/recitalAI/summa-qa/blob/HEAD/summaqa/f1_squad.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1908.08962","paper":"/paper/well-read-students-learn-better-the-impact-of","title":"Well-Read Students Learn Better: On the Importance of Pre-training Compact Models","date":"2019-08-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"somiltg/bert","path":"evaluate-v2.0.py","file_url":"https://github.com/somiltg/bert/blob/HEAD/evaluate-v2.0.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"1908.00059","paper":"/paper/graphflow-exploiting-conversation-flow-with","title":"GraphFlow: Exploiting Conversation Flow with Graph Neural Networks for Conversational Machine Comprehension","date":"2019-07-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"a2fdbbe11e2cf910","mcp_get_code":{"code_sha256":"a2fdbbe11e2cf910"}},{"arxiv_id":"1906.05807","paper":"/paper/real-time-open-domain-question-answering-with","title":"Real-Time Open-Domain Question Answering with Dense-Sparse Phrase Index","date":"2019-06-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"uwnlp/denspi","path":"evaluate-v1.1.py","file_url":"https://github.com/uwnlp/denspi/blob/HEAD/evaluate-v1.1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1906.05394","paper":"/paper/neural-arabic-question-answering","title":"Neural Arabic Question Answering","date":"2019-06-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"husseinmozannar/SOQAL","path":"baselines_reading/evaluate_baselines.py","file_url":"https://github.com/husseinmozannar/SOQAL/blob/HEAD/baselines_reading/evaluate_baselines.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e4b49776872650db","mcp_get_code":{"code_sha256":"e4b49776872650db"}},{"arxiv_id":"1906.01603","paper":"/paper/do-neural-dialog-systems-use-the-conversation","title":"Do Neural Dialog Systems Use the Conversation History Effectively? An Empirical Study","date":"2019-06-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"chinnadhurai/ParlAI","path":"parlai/core/metrics.py","file_url":"https://github.com/chinnadhurai/ParlAI/blob/HEAD/parlai/core/metrics.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"code_sha256_prefix":"da3bc8c42eec18d1","mcp_get_code":{"code_sha256":"da3bc8c42eec18d1"}},{"arxiv_id":"1905.13453","paper":"/paper/multiqa-an-empirical-investigation-of","title":"MultiQA: An Empirical Investigation of Generalization and Transfer in Reading Comprehension","date":"2019-05-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"alontalmor/multiqa","path":"common/official_eval.py","file_url":"https://github.com/alontalmor/multiqa/blob/HEAD/common/official_eval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1905.05460","paper":"/paper/cognitive-graph-for-multi-hop-reading","title":"Cognitive Graph for Multi-Hop Reading Comprehension at Scale","date":"2019-05-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THUDM/CogQA","path":"hotpot_evaluate_v1.py","file_url":"https://github.com/THUDM/CogQA/blob/HEAD/hotpot_evaluate_v1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"1902.01718","paper":"/paper/end-to-end-open-domain-question-answering","title":"End-to-End Open-Domain Question Answering with BERTserini","date":"2019-02-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rsvp-ai/bertserini","path":"bertserini/utils/utils_squad_metrics.py","file_url":"https://github.com/rsvp-ai/bertserini/blob/HEAD/bertserini/utils/utils_squad_metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"1810.04805","paper":"/paper/bert-pre-training-of-deep-bidirectional","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","date":"2018-10-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Microsoft/AzureML-BERT","path":"finetune/evaluate_squad.py","file_url":"https://github.com/Microsoft/AzureML-BERT/blob/HEAD/finetune/evaluate_squad.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1810.04805","paper":"/paper/bert-pre-training-of-deep-bidirectional","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","date":"2018-10-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Zehui127/SQUAD_BERT","path":"SQUAD/evaluate-v2.0.py","file_url":"https://github.com/Zehui127/SQUAD_BERT/blob/HEAD/SQUAD/evaluate-v2.0.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"1809.09600","paper":"/paper/hotpotqa-a-dataset-for-diverse-explainable","title":"HotpotQA: A Dataset for Diverse, Explainable Multi-hop Question Answering","date":"2018-09-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hotpotqa/hotpot","path":"hotpot_evaluate_v1.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/hotpot_evaluate_v1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"1808.07042","paper":"/paper/coqa-a-conversational-question-answering","title":"CoQA: A Conversational Question Answering Challenge","date":"2018-08-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"stanfordnlp/coqa-baselines","path":"scripts/gen_pipeline_data.py","file_url":"https://github.com/stanfordnlp/coqa-baselines/blob/HEAD/scripts/gen_pipeline_data.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a2fdbbe11e2cf910","mcp_get_code":{"code_sha256":"a2fdbbe11e2cf910"}},{"arxiv_id":"1804.07927","paper":"/paper/duorc-towards-complex-language-understanding","title":"DuoRC: Towards Complex Language Understanding with Paraphrased Reading Comprehension","date":"2018-04-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"duorc/duorc","path":"evaluate.py","file_url":"https://github.com/duorc/duorc/blob/HEAD/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1804.07726","paper":"/paper/phrase-indexed-question-answering-a-new","title":"Phrase-Indexed Question Answering: A New Challenge for Scalable Document Comprehension","date":"2018-04-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"uwnlp/piqa","path":"squad/piqa_evaluate.py","file_url":"https://github.com/uwnlp/piqa/blob/HEAD/squad/piqa_evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1803.09720","paper":"/paper/clicr-a-dataset-of-clinical-case-reports-for","title":"CliCR: A Dataset of Clinical Case Reports for Machine Reading Comprehension","date":"2018-03-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"clips/clicr","path":"dataset-code/evaluate.py","file_url":"https://github.com/clips/clicr/blob/HEAD/dataset-code/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2d16293904cd4c35","mcp_get_code":{"code_sha256":"2d16293904cd4c35"}},{"arxiv_id":"1711.05116","paper":"/paper/evidence-aggregation-for-answer-re-ranking-in","title":"Evidence Aggregation for Answer Re-Ranking in Open-Domain Question Answering","date":"2017-11-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shuohangwang/mprc","path":"trainedmodel/evaluation/quasart/evaluate-v1.1.py","file_url":"https://github.com/shuohangwang/mprc/blob/HEAD/trainedmodel/evaluation/quasart/evaluate-v1.1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1711.05116","paper":"/paper/evidence-aggregation-for-answer-re-ranking-in","title":"Evidence Aggregation for Answer Re-Ranking in Open-Domain Question Answering","date":"2017-11-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shuohangwang/mprc","path":"trainedmodel/evaluation/unftriviaqa/triviaqa_evaluation.py","file_url":"https://github.com/shuohangwang/mprc/blob/HEAD/trainedmodel/evaluation/unftriviaqa/triviaqa_evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"2879260bc19381c8","mcp_get_code":{"code_sha256":"2879260bc19381c8"}},{"arxiv_id":"1705.03551","paper":"/paper/triviaqa-a-large-scale-distantly-supervised","title":"TriviaQA: A Large Scale Distantly Supervised Challenge Dataset for Reading Comprehension","date":"2017-05-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mandarjoshi90/triviaqa","path":"evaluation/triviaqa_evaluation.py","file_url":"https://github.com/mandarjoshi90/triviaqa/blob/HEAD/evaluation/triviaqa_evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"2879260bc19381c8","mcp_get_code":{"code_sha256":"2879260bc19381c8"}},{"arxiv_id":"1611.09268","paper":"/paper/ms-marco-a-human-generated-machine-reading","title":"MS MARCO: A Human Generated MAchine Reading COmprehension Dataset","date":"2016-11-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/OpenKP","path":"evaluate.py","file_url":"https://github.com/microsoft/OpenKP/blob/HEAD/evaluate.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ae96a7f4fe58827f","mcp_get_code":{"code_sha256":"ae96a7f4fe58827f"}},{"arxiv_id":"1611.01603","paper":"/paper/bidirectional-attention-flow-for-machine","title":"Bidirectional Attention Flow for Machine Comprehension","date":"2016-11-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ghus75/Question_Answering","path":"code/evaluate.py","file_url":"https://github.com/ghus75/Question_Answering/blob/HEAD/code/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1608.07905","paper":"/paper/machine-comprehension-using-match-lstm-and","title":"Machine Comprehension Using Match-LSTM and Answer Pointer","date":"2016-08-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1606.05250","paper":"/paper/squad-100000-questions-for-machine","title":"SQuAD: 100,000+ Questions for Machine Comprehension of Text","date":"2016-06-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"1508.06615","paper":"/paper/character-aware-neural-language-models","title":"Character-Aware Neural Language Models","date":"2015-08-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"NLPLearn/QANet","path":"evaluate-v1.1.py","file_url":"https://github.com/NLPLearn/QANet/blob/HEAD/evaluate-v1.1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"openreview_trSWJ99WzS","paper":null,"title":"arXiv:openreview_trSWJ99WzS","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"DerrickYLJ/LessIsMore","path":"experiments/LongBench/metrics.py","file_url":"https://github.com/DerrickYLJ/LessIsMore/blob/HEAD/experiments/LongBench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"ijcai2024_0687","paper":null,"title":"arXiv:ijcai2024_0687","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"zjunlp/FactCHD","path":"evaluate/metrics_plus.py","file_url":"https://github.com/zjunlp/FactCHD/blob/HEAD/evaluate/metrics_plus.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4cc38ac08b499ecc","mcp_get_code":{"code_sha256":"4cc38ac08b499ecc"}},{"arxiv_id":"aaai_34690","paper":null,"title":"arXiv:aaai_34690","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"ZongyueQin/DSBD","path":"sampling/utils.py","file_url":"https://github.com/ZongyueQin/DSBD/blob/HEAD/sampling/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"aaai_16733","paper":null,"title":"arXiv:aaai_16733","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"rajammanabrolu/WorldGeneration","path":"neural-based/KG-extraction/utils_squad_evaluate.py","file_url":"https://github.com/rajammanabrolu/WorldGeneration/blob/HEAD/neural-based/KG-extraction/utils_squad_evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2025.findings-emnlp.429","paper":null,"title":"arXiv:2025.findings-emnlp.429","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"zhzihao/QPruningKV","path":"metrics.py","file_url":"https://github.com/zhzihao/QPruningKV/blob/HEAD/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2025.findings-acl.652","paper":null,"title":"arXiv:2025.findings-acl.652","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"HSLiu-Initial/CtrlA","path":"code/utils.py","file_url":"https://github.com/HSLiu-Initial/CtrlA/blob/HEAD/code/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c29fc928dae85749","mcp_get_code":{"code_sha256":"c29fc928dae85749"}},{"arxiv_id":"2025.findings-acl.588","paper":null,"title":"arXiv:2025.findings-acl.588","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"princeton-nlp/ALCE","path":"utils.py","file_url":"https://github.com/princeton-nlp/ALCE/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2025.emnlp-main.443","paper":null,"title":"arXiv:2025.emnlp-main.443","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"adlnlp/Gali","path":"longbench/metrics.py","file_url":"https://github.com/adlnlp/Gali/blob/HEAD/longbench/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e7e75981cb464788","mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"arxiv_id":"2025.emnlp-main.292","paper":null,"title":"arXiv:2025.emnlp-main.292","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"RUC-NLPIR/OmniEval","path":"utils/evaluation.py","file_url":"https://github.com/RUC-NLPIR/OmniEval/blob/HEAD/utils/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6205dd62c2627dde","mcp_get_code":{"code_sha256":"6205dd62c2627dde"}},{"arxiv_id":"2025.acl-long.191","paper":null,"title":"arXiv:2025.acl-long.191","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"UKPLab/acl2025-diverse-cot","path":"src/hotpotqa_evaluation.py","file_url":"https://github.com/UKPLab/acl2025-diverse-cot/blob/HEAD/src/hotpotqa_evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2025.acl-long.1500","paper":null,"title":"arXiv:2025.acl-long.1500","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"leezythu/FocusLLM","path":"infbench_src/compute_scores.py","file_url":"https://github.com/leezythu/FocusLLM/blob/HEAD/infbench_src/compute_scores.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4b44a4d3fd80a48d","mcp_get_code":{"code_sha256":"4b44a4d3fd80a48d"}},{"arxiv_id":"2025.acl-long.1142","paper":null,"title":"arXiv:2025.acl-long.1142","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"GenIRAG/PER-PSE","path":"flashrag/evaluator/utils.py","file_url":"https://github.com/GenIRAG/PER-PSE/blob/HEAD/flashrag/evaluator/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2024.findings-emnlp.379","paper":null,"title":"arXiv:2024.findings-emnlp.379","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"wenyudu/MIGU","path":"src/compute_metrics.py","file_url":"https://github.com/wenyudu/MIGU/blob/HEAD/src/compute_metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2024.findings-emnlp.365","paper":null,"title":"arXiv:2024.findings-emnlp.365","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"UKPLab/m2qa","path":"Experiments/M2QA_Metric/m2qa_metric.py","file_url":"https://github.com/UKPLab/m2qa/blob/HEAD/Experiments/M2QA_Metric/m2qa_metric.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"2d391d2daa3c57fe","mcp_get_code":{"code_sha256":"2d391d2daa3c57fe"}},{"arxiv_id":"2023.findings-emnlp.836","paper":null,"title":"arXiv:2023.findings-emnlp.836","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"IBM/ensemble-instruct","path":"ensemble_instruct/ensemble_output.py","file_url":"https://github.com/IBM/ensemble-instruct/blob/HEAD/ensemble_instruct/ensemble_output.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"52a9f448d8cbd82a","mcp_get_code":{"code_sha256":"52a9f448d8cbd82a"}},{"arxiv_id":"2023.findings-emnlp.835","paper":null,"title":"arXiv:2023.findings-emnlp.835","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"THU-KEG/ProbTree","path":"src/2wiki/RoHT/evaluate.py","file_url":"https://github.com/THU-KEG/ProbTree/blob/HEAD/src/2wiki/RoHT/evaluate.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae7ab386661a4f4","mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"arxiv_id":"2023.emnlp-main.803","paper":null,"title":"arXiv:2023.emnlp-main.803","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"yisunlp/Anti-CF","path":"utils/metrics.py","file_url":"https://github.com/yisunlp/Anti-CF/blob/HEAD/utils/metrics.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6a96435eba311b08","mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"arxiv_id":"2021.acl-long.48","paper":null,"title":"arXiv:2021.acl-long.48","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"PluviophileYU/COSY","path":"XQA/src/evaluate_v1_1.py","file_url":"https://github.com/PluviophileYU/COSY/blob/HEAD/XQA/src/evaluate_v1_1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c6a80c065d2e4851","mcp_get_code":{"code_sha256":"c6a80c065d2e4851"}},{"arxiv_id":"2021.acl-long.48","paper":null,"title":"arXiv:2021.acl-long.48","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"PluviophileYU/COSY","path":"XQA/src/mlqa_evaluation_v1.py","file_url":"https://github.com/PluviophileYU/COSY/blob/HEAD/XQA/src/mlqa_evaluation_v1.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9cddf0b1a2750978","mcp_get_code":{"code_sha256":"9cddf0b1a2750978"}}]}