{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/build-prompt","entry":"build_prompt","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":78,"n_papers_ran":39,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":89,"n_samples_ran":41,"n_samples_fingerprinted":12,"n_places":94,"n_places_pointer_only":43,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":9,"ran_fixture":0,"ran":32,"unverified":48},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2609.01788","paper":"/paper/arxiv-2609-01788","title":"VakyArth: Evaluating Pragmatic Competence in LLMs across Indic Languages","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"Usneek1/VakyArth","path":"src/tasks/mcq.py","file_url":"https://github.com/Usneek1/VakyArth/blob/HEAD/src/tasks/mcq.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"de9ae4128c68a13e","mcp_get_code":{"code_sha256":"de9ae4128c68a13e"}},{"arxiv_id":"2609.01788","paper":"/paper/arxiv-2609-01788","title":"VakyArth: Evaluating Pragmatic Competence in LLMs across Indic Languages","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"Usneek1/VakyArth","path":"src/tasks/nli.py","file_url":"https://github.com/Usneek1/VakyArth/blob/HEAD/src/tasks/nli.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"e8db056ec5c17672","mcp_get_code":{"code_sha256":"e8db056ec5c17672"}},{"arxiv_id":"2609.01148","paper":"/paper/arxiv-2609-01148","title":"Dotting the Eye: An Intent-Driven Image Retouching Agent for Visual Focus Enhancement","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"DragonisCV/EyeControl","path":"src/inference/infer.py","file_url":"https://github.com/DragonisCV/EyeControl/blob/HEAD/src/inference/infer.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a69c968fe0bfe0bd","mcp_get_code":{"code_sha256":"a69c968fe0bfe0bd"}},{"arxiv_id":"2608.30413","paper":"/paper/arxiv-2608-30413","title":"DERELAB: Probing Defeasible Reasoning and Confirmation Bias in LLMs with a Generative Benchmark","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"Jayanta47/DeReLab","path":"generate_paraphrases.py","file_url":"https://github.com/Jayanta47/DeReLab/blob/HEAD/generate_paraphrases.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"395655befcf85fb6","mcp_get_code":{"code_sha256":"395655befcf85fb6"}},{"arxiv_id":"2608.26550","paper":"/paper/arxiv-2608-26550","title":"SPEAR: Distilling Domain-Adaptive Reasoning Skeletons via Sequential Symbolic Alignment in Reinforcement Learning","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"zhuochunli/SPEAR","path":"utils.py","file_url":"https://github.com/zhuochunli/SPEAR/blob/HEAD/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a383de780ac0aa57","mcp_get_code":{"code_sha256":"a383de780ac0aa57"}},{"arxiv_id":"2608.26119","paper":"/paper/arxiv-2608-26119","title":"DeflectBench: A Benchmark for Evaluating Rhetorical Fallacy Generation in LLMs","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"ArtKanke/DeflectBench","path":"config.py","file_url":"https://github.com/ArtKanke/DeflectBench/blob/HEAD/config.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8737904f0167a07d","mcp_get_code":{"code_sha256":"8737904f0167a07d"}},{"arxiv_id":"2608.25655","paper":"/paper/arxiv-2608-25655","title":"Reconstructing the Right Episode: Evaluating Interleaved Conversational Memory Beyond Long Context","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"LordTARN1SHED/SCALE-QA","path":"tsim_reference/src/eval/run_eval.py","file_url":"https://github.com/LordTARN1SHED/SCALE-QA/blob/HEAD/tsim_reference/src/eval/run_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"ef22962a80a1e411","mcp_get_code":{"code_sha256":"ef22962a80a1e411"}},{"arxiv_id":"2608.16190","paper":"/paper/arxiv-2608-16190","title":"Decorrelation Is Not Complementarity: Skill, Not Lineage, Governs Trusted-Monitor Ensembles","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"anik-jha/challenger-panels","path":"src/monitor.py","file_url":"https://github.com/anik-jha/challenger-panels/blob/HEAD/src/monitor.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5ca319bba2a6961c","mcp_get_code":{"code_sha256":"5ca319bba2a6961c"}},{"arxiv_id":"2607.26873","paper":"/paper/arxiv-2607-26873","title":"SERPO: Self-Evolving Rubric Policy Optimization for Open-Ended Test-Time Reinforcement Learning","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"chiefovoavicii/SERPO","path":"eval/eval_gpqa_diamond.py","file_url":"https://github.com/chiefovoavicii/SERPO/blob/HEAD/eval/eval_gpqa_diamond.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"ae8e147c9885175d","mcp_get_code":{"code_sha256":"ae8e147c9885175d"}},{"arxiv_id":"2607.14552","paper":"/paper/arxiv-2607-14552","title":"Answer-Conditioned Chains of Thought Degrade Verifiable-Reasoning Distillation in Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"js-lee-AI/answer-leakage","path":"answer_leakage/prompts.py","file_url":"https://github.com/js-lee-AI/answer-leakage/blob/HEAD/answer_leakage/prompts.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6d43f4e39a356fd7","mcp_get_code":{"code_sha256":"6d43f4e39a356fd7"}},{"arxiv_id":"2607.12787","paper":"/paper/arxiv-2607-12787","title":"Do We Really Need Multimodal Emotion Language Models Larger Than 1B Parameters?","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"GAIR-Lab/Light-MER","path":"train_stage2_mgrpo.py","file_url":"https://github.com/GAIR-Lab/Light-MER/blob/HEAD/train_stage2_mgrpo.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"09028445218f8ba8","mcp_get_code":{"code_sha256":"09028445218f8ba8"}},{"arxiv_id":"2606.31718","paper":"/paper/arxiv-2606-31718","title":"Cross-lingual Relation Extraction with Large Language Models: Zero-Shot, Few-Shot, and Fine-Tuned Evaluation on Romanian","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"DS4AI-UPB/crosslingual-romanian-re","path":"infer_classification.py","file_url":"https://github.com/DS4AI-UPB/crosslingual-romanian-re/blob/HEAD/infer_classification.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a326b5c18eb8d0da","mcp_get_code":{"code_sha256":"a326b5c18eb8d0da"}},{"arxiv_id":"2606.31718","paper":"/paper/arxiv-2606-31718","title":"Cross-lingual Relation Extraction with Large Language Models: Zero-Shot, Few-Shot, and Fine-Tuned Evaluation on Romanian","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"DS4AI-UPB/crosslingual-romanian-re","path":"infer_e2e.py","file_url":"https://github.com/DS4AI-UPB/crosslingual-romanian-re/blob/HEAD/infer_e2e.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"dc60f75e190164fc","mcp_get_code":{"code_sha256":"dc60f75e190164fc"}},{"arxiv_id":"2606.28531","paper":"/paper/arxiv-2606-28531","title":"A Good Talk Doesn't Look Like a Summary, It Teaches You! Measuring Takeaways from Paper-to-Video Talks","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"showlab/Paper2Video","path":"src/evaluation/MetaSim_content.py","file_url":"https://github.com/showlab/Paper2Video/blob/HEAD/src/evaluation/MetaSim_content.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5f2503b0d7ac1d38","mcp_get_code":{"code_sha256":"5f2503b0d7ac1d38"}},{"arxiv_id":"2606.23104","paper":"/paper/arxiv-2606-23104","title":"ReNIO: Reweighting Negative Trajectory Importance for LLM On-Policy Distillation","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"BDML-lab/ReNIO","path":"eval/evaluate_code.py","file_url":"https://github.com/BDML-lab/ReNIO/blob/HEAD/eval/evaluate_code.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"736f1be23541cbcd","mcp_get_code":{"code_sha256":"736f1be23541cbcd"}},{"arxiv_id":"2606.22723","paper":"/paper/arxiv-2606-22723","title":"BLUEX v2: Benchmarking LLMs on Open-Ended Questions from Brazilian University Entrance Exams","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"TropicAI-Research/BLUEXv2","path":"inference/run_inference.py","file_url":"https://github.com/TropicAI-Research/BLUEXv2/blob/HEAD/inference/run_inference.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a542c779b01fcdc2","mcp_get_code":{"code_sha256":"a542c779b01fcdc2"}},{"arxiv_id":"2606.08397","paper":"/paper/arxiv-2606-08397","title":"TrustMargin: Training-Free Arbitration between Parametric Memory and Retrieved Evidence in Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"mojixu/TrustMargin","path":"src/trustmargin.py","file_url":"https://github.com/mojixu/TrustMargin/blob/HEAD/src/trustmargin.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1d8f494789547560","mcp_get_code":{"code_sha256":"1d8f494789547560"}},{"arxiv_id":"2606.06302","paper":"/paper/arxiv-2606-06302","title":"Tangram: Unlocking Non-Uniform KV Cache Compression for Efficient Multi-turn LLM Serving","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"aiha-lab/TANGRAM","path":"benchmarks/tangram/ruler_local.py","file_url":"https://github.com/aiha-lab/TANGRAM/blob/HEAD/benchmarks/tangram/ruler_local.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"bcd9d69939662364","mcp_get_code":{"code_sha256":"bcd9d69939662364"}},{"arxiv_id":"2606.00869","paper":"/paper/arxiv-2606-00869","title":"Enhancing LLM Metacognition via Cognitive Pairwise Training","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"Tsinghua-dhy/CPT","path":"data_construction/4_build_sft/build_sft_dataset.py","file_url":"https://github.com/Tsinghua-dhy/CPT/blob/HEAD/data_construction/4_build_sft/build_sft_dataset.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"74944fb1c9af1d08","mcp_get_code":{"code_sha256":"74944fb1c9af1d08"}},{"arxiv_id":"2605.25920","paper":"/paper/arxiv-2605-25920","title":"Can LLMs Time Travel? Enhancing Temporal Consistency in Legal Agentic Search through Reinforcement Learning","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"AlexFanw/LegalSearch-R1","path":"user/legalsearch_data_process.py","file_url":"https://github.com/AlexFanw/LegalSearch-R1/blob/HEAD/user/legalsearch_data_process.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"d6beb2151fab8888","mcp_get_code":{"code_sha256":"d6beb2151fab8888"}},{"arxiv_id":"2605.17565","paper":"/paper/arxiv-2605-17565","title":"Generalization or Memorization? Brittleness Testing for Chess-Trained Language Models","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"ethanjtang/GAMBIT","path":"eval_models_on_puzzles/eval_all_models_base.py","file_url":"https://github.com/ethanjtang/GAMBIT/blob/HEAD/eval_models_on_puzzles/eval_all_models_base.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"e3266e4f554042c3","mcp_get_code":{"code_sha256":"e3266e4f554042c3"}},{"arxiv_id":"2604.24594","paper":"/paper/arxiv-2604-24594","title":"Skill Retrieval Augmentation for Agentic AI","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"oneal2000/SR-Agents","path":"src/sragents/prompts.py","file_url":"https://github.com/oneal2000/SR-Agents/blob/HEAD/src/sragents/prompts.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"87fb084b0ab65529","mcp_get_code":{"code_sha256":"87fb084b0ab65529"}},{"arxiv_id":"2604.11048","paper":"/paper/arxiv-2604-11048","title":"A Systematic Analysis of the Impact of Persona Steering on LLM Capabilities","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"cjia7/DPR","path":"src/npti/eval/gpt4_score.py","file_url":"https://github.com/cjia7/DPR/blob/HEAD/src/npti/eval/gpt4_score.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"16529904d2d5be9f","mcp_get_code":{"code_sha256":"16529904d2d5be9f"}},{"arxiv_id":"2604.07035","paper":"/paper/arxiv-2604-07035","title":"Unified Deployment-Aware Evaluation of Open Reasoning Language Models","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"mkboch/UDAE","path":"prompts/builder.py","file_url":"https://github.com/mkboch/UDAE/blob/HEAD/prompts/builder.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"35b0ce59f15f774d","mcp_get_code":{"code_sha256":"35b0ce59f15f774d"}},{"arxiv_id":"2604.05564","paper":"/paper/arxiv-2604-05564","title":"THIVLVC: Retrieval Augmented Dependency Parsing for Latin","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"l-pommeret/THIVLVC","path":"src/pipeline/stage1_rag_with_baseline.py","file_url":"https://github.com/l-pommeret/THIVLVC/blob/HEAD/src/pipeline/stage1_rag_with_baseline.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"c22afcb8a1f118a3","mcp_get_code":{"code_sha256":"c22afcb8a1f118a3"}},{"arxiv_id":"2604.05096","paper":"/paper/arxiv-2604-05096","title":"RAG or Learning? Understanding the Limits of LLM Adaptation under Continuous Knowledge Drift in the Real World","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"hbing-l/chronos","path":"baselines/memit.py","file_url":"https://github.com/hbing-l/chronos/blob/HEAD/baselines/memit.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"22a46266c7fa5cbf","mcp_get_code":{"code_sha256":"22a46266c7fa5cbf"}},{"arxiv_id":"2604.04469","paper":"/paper/arxiv-2604-04469","title":"Same Geometry, Opposite Noise: Transformer Magnitude Representations Lack Scalar Variability","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"synthiumjp/weber","path":"m3_pilot/m3_rerun_identification.py","file_url":"https://github.com/synthiumjp/weber/blob/HEAD/m3_pilot/m3_rerun_identification.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a192415d50000694","mcp_get_code":{"code_sha256":"a192415d50000694"}},{"arxiv_id":"2603.28387","paper":"/paper/arxiv-2603-28387","title":"Prompts Without Evidence: How Neuroimaging Mentions Shift Clinical Vision-Language Model Predictions","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"long21wt/scaffold-effect","path":"src/inference_joint.py","file_url":"https://github.com/long21wt/scaffold-effect/blob/HEAD/src/inference_joint.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a1ee925fdd89e7a4","mcp_get_code":{"code_sha256":"a1ee925fdd89e7a4"}},{"arxiv_id":"2603.26332","paper":"/paper/arxiv-2603-26332","title":"CALRK-Bench: Evaluating Context-Aware Legal Reasoning in Korean Law","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"jhCOR/CALRKBench","path":"src/type2_eval.py","file_url":"https://github.com/jhCOR/CALRKBench/blob/HEAD/src/type2_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"324410cfb2d1081e","mcp_get_code":{"code_sha256":"324410cfb2d1081e"}},{"arxiv_id":"2603.22882","paper":"/paper/arxiv-2603-22882","title":"TreeTeaming: Autonomous Red-Teaming of Vision-Language Models via Hierarchical Strategy Exploration","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"ChunXiaostudy/TreeTeaming","path":"generate/batch_qwen_image_edit.py","file_url":"https://github.com/ChunXiaostudy/TreeTeaming/blob/HEAD/generate/batch_qwen_image_edit.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"05b3c14592dae070","mcp_get_code":{"code_sha256":"05b3c14592dae070"}},{"arxiv_id":"2603.22754","paper":"/paper/arxiv-2603-22754","title":"PRISM: A Dual View of LLM Reasoning through Semantic Flow and Latent Computation","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"chili-lab/PRISM","path":"prism_code/classifier.py","file_url":"https://github.com/chili-lab/PRISM/blob/HEAD/prism_code/classifier.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"6a9d7dbe0ac8d9bf","mcp_get_code":{"code_sha256":"6a9d7dbe0ac8d9bf"}},{"arxiv_id":"2603.05262","paper":"/paper/arxiv-2603-05262","title":"VietJobs: A Vietnamese Job Advertisement Dataset","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"VinNLP/VietJobs","path":"format_prompt_category.py","file_url":"https://github.com/VinNLP/VietJobs/blob/HEAD/format_prompt_category.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ff09cb63990754ad","mcp_get_code":{"code_sha256":"ff09cb63990754ad"}},{"arxiv_id":"2603.00634","paper":"/paper/arxiv-2603-00634","title":"BLUFF: Benchmarking the Detection of False and Synthetic Content across 58 Low-Resource Languages","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"jsl5710/BLUFF","path":"code-base/generation_codes/eng_x_f/eng_x_f.py","file_url":"https://github.com/jsl5710/BLUFF/blob/HEAD/code-base/generation_codes/eng_x_f/eng_x_f.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"a328dee234881191","mcp_get_code":{"code_sha256":"a328dee234881191"}},{"arxiv_id":"2603.00634","paper":"/paper/arxiv-2603-00634","title":"BLUFF: Benchmarking the Detection of False and Synthetic Content across 58 Low-Resource Languages","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"jsl5710/BLUFF","path":"code-base/generation_codes/eng_x_r/eng_x_r.py","file_url":"https://github.com/jsl5710/BLUFF/blob/HEAD/code-base/generation_codes/eng_x_r/eng_x_r.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"239c2c0b061fb692","mcp_get_code":{"code_sha256":"239c2c0b061fb692"}},{"arxiv_id":"2603.00634","paper":"/paper/arxiv-2603-00634","title":"BLUFF: Benchmarking the Detection of False and Synthetic Content across 58 Low-Resource Languages","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"jsl5710/BLUFF","path":"code-base/generation_codes/x_eng_f/x_eng_f.py","file_url":"https://github.com/jsl5710/BLUFF/blob/HEAD/code-base/generation_codes/x_eng_f/x_eng_f.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"42ff7e217bee8614","mcp_get_code":{"code_sha256":"42ff7e217bee8614"}},{"arxiv_id":"2602.15189","paper":"/paper/arxiv-2602-15189","title":"ScrapeGraphAI-100k: Dataset for Schema-Constrained LLM Generation","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"ScrapeGraphAI/scrapegraph-100k-paper","path":"modelling/convert_awq.py","file_url":"https://github.com/ScrapeGraphAI/scrapegraph-100k-paper/blob/HEAD/modelling/convert_awq.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9cca23f2500b9d75","mcp_get_code":{"code_sha256":"9cca23f2500b9d75"}},{"arxiv_id":"2602.14470","paper":"/paper/arxiv-2602-14470","title":"HyperRAG: Reasoning N-ary Facts over Hypergraphs for Retrieval Augmented Generation","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"Vincent-Lien/HyperRAG","path":"dataset/query2text.py","file_url":"https://github.com/Vincent-Lien/HyperRAG/blob/HEAD/dataset/query2text.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a6e6bdcb966b6c62","mcp_get_code":{"code_sha256":"a6e6bdcb966b6c62"}},{"arxiv_id":"2601.17197","paper":"/paper/arxiv-2601-17197","title":"Reasoning Beyond Literal: Cross-style Multimodal Reasoning for Figurative Language Understanding","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"scheshmi/CrossStyle-MMR","path":"infer/vllm_baseline.py","file_url":"https://github.com/scheshmi/CrossStyle-MMR/blob/HEAD/infer/vllm_baseline.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"2d4a4f42c50d8de2","mcp_get_code":{"code_sha256":"2d4a4f42c50d8de2"}},{"arxiv_id":"2601.15429","paper":"/paper/arxiv-2601-15429","title":"Domain-Specific Knowledge Graphs in RAG-Enhanced Healthcare LLMs","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"sydneyanuyah/RAGComparison","path":"artifacts/csv_coref_switcher.py","file_url":"https://github.com/sydneyanuyah/RAGComparison/blob/HEAD/artifacts/csv_coref_switcher.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"21a18e971bb86c8c","mcp_get_code":{"code_sha256":"21a18e971bb86c8c"}},{"arxiv_id":"2601.15429","paper":"/paper/arxiv-2601-15429","title":"Domain-Specific Knowledge Graphs in RAG-Enhanced Healthcare LLMs","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"sydneyanuyah/RAGComparison","path":"artifacts/complex_simplify_sentences.py","file_url":"https://github.com/sydneyanuyah/RAGComparison/blob/HEAD/artifacts/complex_simplify_sentences.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"718b35386828443e","mcp_get_code":{"code_sha256":"718b35386828443e"}},{"arxiv_id":"2601.15429","paper":"/paper/arxiv-2601-15429","title":"Domain-Specific Knowledge Graphs in RAG-Enhanced Healthcare LLMs","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"sydneyanuyah/RAGComparison","path":"artifacts/rel_extraction_pipeline.py","file_url":"https://github.com/sydneyanuyah/RAGComparison/blob/HEAD/artifacts/rel_extraction_pipeline.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"36c1eae64901861f","mcp_get_code":{"code_sha256":"36c1eae64901861f"}},{"arxiv_id":"2601.15429","paper":"/paper/arxiv-2601-15429","title":"Domain-Specific Knowledge Graphs in RAG-Enhanced Healthcare LLMs","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"sydneyanuyah/RAGComparison","path":"artifacts/simplify_by_label.py","file_url":"https://github.com/sydneyanuyah/RAGComparison/blob/HEAD/artifacts/simplify_by_label.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1b77216c9a0e9270","mcp_get_code":{"code_sha256":"1b77216c9a0e9270"}},{"arxiv_id":"2601.10960","paper":"/paper/arxiv-2601-10960","title":"Steering Language Models Before They Speak: Logit-Level Interventions","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"hsannn/swai","path":"logit_steering.py","file_url":"https://github.com/hsannn/swai/blob/HEAD/logit_steering.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"824dcd5664e63215","mcp_get_code":{"code_sha256":"824dcd5664e63215"}},{"arxiv_id":"2601.09703","paper":"/paper/arxiv-2601-09703","title":"ShortCoder: Knowledge-Augmented Syntax Optimization for Token-Efficient Code Generation","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"DeepSoftwareAnalytics/ShorterCode","path":"reference_lora.py","file_url":"https://github.com/DeepSoftwareAnalytics/ShorterCode/blob/HEAD/reference_lora.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"df9332e0280d4162","mcp_get_code":{"code_sha256":"df9332e0280d4162"}},{"arxiv_id":"2601.09402","paper":"/paper/arxiv-2601-09402","title":"SEEK: Steering LLM Reasoning for RAG via Internal Reasoning Sketches","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"OpenBMB/PAGER","path":"src/infer_page.py","file_url":"https://github.com/OpenBMB/PAGER/blob/HEAD/src/infer_page.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"508d2df531171de9","mcp_get_code":{"code_sha256":"508d2df531171de9"}},{"arxiv_id":"2601.06992","paper":"/paper/arxiv-2601-06992","title":"FINCARDS: Card-Based Analyst Reranking for Financial Document Question Answering","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"XanderZhou2022/FINCARDS","path":"pipeline/stage0_query_intent.py","file_url":"https://github.com/XanderZhou2022/FINCARDS/blob/HEAD/pipeline/stage0_query_intent.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e244d9d2aab28bd0","mcp_get_code":{"code_sha256":"e244d9d2aab28bd0"}},{"arxiv_id":"2601.05903","paper":"/paper/arxiv-2601-05903","title":"HAPS: Hierarchical LLM Routing with Joint Architecture and Parameter Search","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"zihangtian/HAPS","path":"HotpotQA/joint_rl/rl_batch.py","file_url":"https://github.com/zihangtian/HAPS/blob/HEAD/HotpotQA/joint_rl/rl_batch.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"07274e6bcd20a571","mcp_get_code":{"code_sha256":"07274e6bcd20a571"}},{"arxiv_id":"2510.10020","paper":"/paper/arxiv-2510-10020","title":"Calibrating Generative Models to Distributional Constraints","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"smithhenryd/cgm","path":"gemma/cgm_gemma.py","file_url":"https://github.com/smithhenryd/cgm/blob/HEAD/gemma/cgm_gemma.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"083602ed7c768390","mcp_get_code":{"code_sha256":"083602ed7c768390"}},{"arxiv_id":"2510.03215","paper":"/paper/arxiv-2510-03215","title":"Cache-to-Cache: Direct Semantic Communication Between Large Language Models","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"thu-nics/C2C","path":"rosetta/utils/evaluate.py","file_url":"https://github.com/thu-nics/C2C/blob/HEAD/rosetta/utils/evaluate.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"472ca2ea3d9cecca","mcp_get_code":{"code_sha256":"472ca2ea3d9cecca"}},{"arxiv_id":"2509.17289","paper":"/paper/arxiv-2509-17289","title":"Automated Knowledge Graph Construction using Large Language Models and Sentence Complexity Modelling","date":null,"month_inferred_from_arxiv_id":"2025-09","title_source":"syntology","repo":"KaushikMahmud/CoDe-KG_EMNLP_2025","path":"CoDe-KG/prompts.py","file_url":"https://github.com/KaushikMahmud/CoDe-KG_EMNLP_2025/blob/HEAD/CoDe-KG/prompts.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ec50c014e2ffb4f5","mcp_get_code":{"code_sha256":"ec50c014e2ffb4f5"}},{"arxiv_id":"2509.06493","paper":"/paper/arxiv-2509-06493","title":"Scaling up Multi-Turn Off-Policy RL and Multi-Agent Tree Search for LLM Step-Provers","date":"2025-09-08","month_inferred_from_arxiv_id":null,"title_source":"syntology","repo":"ByteDance-Seed/BFS-Prover-V2","path":"src/plan/generate.py","file_url":"https://github.com/ByteDance-Seed/BFS-Prover-V2/blob/HEAD/src/plan/generate.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7bf0d8bea27ef263","mcp_get_code":{"code_sha256":"7bf0d8bea27ef263"}},{"arxiv_id":"2506.20100","paper":"/paper/mirage-a-benchmark-for-multimodal-information","title":"MIRAGE: A Benchmark for Multimodal Information-Seeking and Reasoning in Agricultural Expert-Guided Conversations","date":"2025-06-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mirage-benchmark/mirage-benchmark","path":"MMMT/src/prompt_builder.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/prompt_builder.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"73f970619d92b562","mcp_get_code":{"code_sha256":"73f970619d92b562"}},{"arxiv_id":"2506.02973","paper":"/paper/expanding-before-inferring-enhancing","title":"Expanding before Inferring: Enhancing Factuality in Large Language Models through Premature Layers Interpolation","date":"2025-06-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"CuSO4-Chen/PLI","path":"src/evaluation/gsm8k_eval.py","file_url":"https://github.com/CuSO4-Chen/PLI/blob/HEAD/src/evaluation/gsm8k_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2c2ea637ced966fa","mcp_get_code":{"code_sha256":"2c2ea637ced966fa"}},{"arxiv_id":"2504.21582","paper":"/paper/mf-llm-simulating-collective-decision","title":"MF-LLM: Simulating Population Decision Dynamics via a Mean-Field Large Language Model Framework","date":"2025-04-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Miracle1207/Mean-Field-LLM","path":"mf_llm/evaluate/evaluate_gpt_batch.py","file_url":"https://github.com/Miracle1207/Mean-Field-LLM/blob/HEAD/mf_llm/evaluate/evaluate_gpt_batch.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"07ad866b84a13b94","mcp_get_code":{"code_sha256":"07ad866b84a13b94"}},{"arxiv_id":"2503.13342","paper":"/paper/valid-text-to-sql-generation-with-unification","title":"Valid Text-to-SQL Generation with Unification-based DeepStochLog","date":"2025-03-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"abcf216a0f054de7","mcp_get_code":{"code_sha256":"abcf216a0f054de7"}},{"arxiv_id":"2502.16971","paper":"/paper/longsafety-evaluating-long-context-safety-of","title":"LongSafety: Evaluating Long-Context Safety of Large Language Models","date":"2025-02-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thu-coai/LongSafety","path":"src/gen_model_response.py","file_url":"https://github.com/thu-coai/LongSafety/blob/HEAD/src/gen_model_response.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3e34927af2210615","mcp_get_code":{"code_sha256":"3e34927af2210615"}},{"arxiv_id":"2502.11191","paper":"/paper/primus-a-pioneering-collection-of-open-source","title":"Primus: A Pioneering Collection of Open-Source Datasets for Cybersecurity LLM Training","date":"2025-02-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"huggingface/cosmopedia","path":"prompts/auto_math_text/build_science_prompts.py","file_url":"https://github.com/huggingface/cosmopedia/blob/HEAD/prompts/auto_math_text/build_science_prompts.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"35dcf93579591fec","mcp_get_code":{"code_sha256":"35dcf93579591fec"}},{"arxiv_id":"2501.14275","paper":"/paper/leveraging-online-olympiad-level-math","title":"Leveraging Online Olympiad-Level Math Problems for LLMs Training and Contamination-Resistant Evaluation","date":"2025-01-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dsl-lab/aops","path":"classify_aops.py","file_url":"https://github.com/dsl-lab/aops/blob/HEAD/classify_aops.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"403712b231130602","mcp_get_code":{"code_sha256":"403712b231130602"}},{"arxiv_id":"2409.14083","paper":"/paper/surf-teaching-large-vision-language-models-to","title":"SURf: Teaching Large Vision-Language Models to Selectively Utilize Retrieved Information","date":"2024-09-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"GasolSun36/SURf","path":"eval/pope.py","file_url":"https://github.com/GasolSun36/SURf/blob/HEAD/eval/pope.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c8c51dd88ad58cb1","mcp_get_code":{"code_sha256":"c8c51dd88ad58cb1"}},{"arxiv_id":"2409.14083","paper":"/paper/surf-teaching-large-vision-language-models-to","title":"SURf: Teaching Large Vision-Language Models to Selectively Utilize Retrieved Information","date":"2024-09-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"GasolSun36/SURf","path":"initial/data_filter.py","file_url":"https://github.com/GasolSun36/SURf/blob/HEAD/initial/data_filter.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"48f2a9620f2ac197","mcp_get_code":{"code_sha256":"48f2a9620f2ac197"}},{"arxiv_id":"2408.12325","paper":"/paper/improving-factuality-in-large-language-models","title":"Improving Factuality in Large Language Models via Decoding-Time Hallucinatory and Truthful Comparators","date":"2024-08-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ydk122024/cdt","path":"src/benchmark_evaluation/alpaca_eval.py","file_url":"https://github.com/ydk122024/cdt/blob/HEAD/src/benchmark_evaluation/alpaca_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"b13ae15599bf9354","mcp_get_code":{"code_sha256":"b13ae15599bf9354"}},{"arxiv_id":"2408.12325","paper":"/paper/improving-factuality-in-large-language-models","title":"Improving Factuality in Large Language Models via Decoding-Time Hallucinatory and Truthful Comparators","date":"2024-08-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ydk122024/cdt","path":"src/benchmark_evaluation/knight_eval.py","file_url":"https://github.com/ydk122024/cdt/blob/HEAD/src/benchmark_evaluation/knight_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a3be7b801f8fda87","mcp_get_code":{"code_sha256":"a3be7b801f8fda87"}},{"arxiv_id":"2408.12325","paper":"/paper/improving-factuality-in-large-language-models","title":"Improving Factuality in Large Language Models via Decoding-Time Hallucinatory and Truthful Comparators","date":"2024-08-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ydk122024/cdt","path":"src/benchmark_evaluation/xsum_eval.py","file_url":"https://github.com/ydk122024/cdt/blob/HEAD/src/benchmark_evaluation/xsum_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"d80c60be83bb2afa","mcp_get_code":{"code_sha256":"d80c60be83bb2afa"}},{"arxiv_id":"2407.10834","paper":"/paper/metallm-a-high-performant-and-cost-efficient","title":"MetaLLM: A High-performant and Cost-efficient Dynamic Framework for Wrapping LLMs","date":"2024-07-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mail-research/metallm-wrapper","path":"benchmark.py","file_url":"https://github.com/mail-research/metallm-wrapper/blob/HEAD/benchmark.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"10a73330aef4702b","mcp_get_code":{"code_sha256":"10a73330aef4702b"}},{"arxiv_id":"2406.12793","paper":"/paper/chatglm-a-family-of-large-language-models","title":"ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools","date":"2024-06-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thudm/chatglm-6b","path":"cli_demo.py","file_url":"https://github.com/thudm/chatglm-6b/blob/HEAD/cli_demo.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"20f3c3d6999848e9","mcp_get_code":{"code_sha256":"20f3c3d6999848e9"}},{"arxiv_id":"2406.12793","paper":"/paper/chatglm-a-family-of-large-language-models","title":"ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools","date":"2024-06-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thudm/chatglm-6b","path":"cli_demo_vision.py","file_url":"https://github.com/thudm/chatglm-6b/blob/HEAD/cli_demo_vision.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7ce6ba0a6898058f","mcp_get_code":{"code_sha256":"7ce6ba0a6898058f"}},{"arxiv_id":"2406.12793","paper":"/paper/chatglm-a-family-of-large-language-models","title":"ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools","date":"2024-06-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thudm/chatglm","path":"basic_demo/cli_demo.py","file_url":"https://github.com/thudm/chatglm/blob/HEAD/basic_demo/cli_demo.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"45ac4198df42b5f0","mcp_get_code":{"code_sha256":"45ac4198df42b5f0"}},{"arxiv_id":"2405.18386","paper":"/paper/instruct-musicgen-unlocking-text-to-music","title":"Instruct-MusicGen: Unlocking Text-to-Music Editing for Music Language Models via Instruction Tuning","date":"2024-05-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"MS-P3/code5","path":"chatglm3/cli_demo.py","file_url":"https://github.com/MS-P3/code5/blob/HEAD/chatglm3/cli_demo.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"45ac4198df42b5f0","mcp_get_code":{"code_sha256":"45ac4198df42b5f0"}},{"arxiv_id":"2405.14170","paper":"/paper/large-language-models-guided-dynamic","title":"Large Language Models-guided Dynamic Adaptation for Temporal Knowledge Graph Reasoning","date":"2024-05-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jiapuwang/LLM-DA","path":"Iteration_reasoning.py","file_url":"https://github.com/jiapuwang/LLM-DA/blob/HEAD/Iteration_reasoning.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ce7c550a5364d864","mcp_get_code":{"code_sha256":"ce7c550a5364d864"}},{"arxiv_id":"2404.09526","paper":"/paper/loongserve-efficiently-serving-long-context","title":"LoongServe: Efficiently Serving Long-Context Large Language Models with Elastic Sequence Parallelism","date":"2024-04-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"LoongServe/LoongServe","path":"loongserve/longserve_server/build_prompt.py","file_url":"https://github.com/LoongServe/LoongServe/blob/HEAD/loongserve/longserve_server/build_prompt.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b710e60c3d04dc58","mcp_get_code":{"code_sha256":"b710e60c3d04dc58"}},{"arxiv_id":"2404.00699","paper":"/paper/how-much-are-llms-contaminated-a","title":"How Much are Large Language Models Contaminated? A Comprehensive Survey and the LLMSanitize Library","date":"2024-03-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ntunlp/llmsanitize","path":"llmsanitize/closed_data_methods/ts_guessing_question_based.py","file_url":"https://github.com/ntunlp/llmsanitize/blob/HEAD/llmsanitize/closed_data_methods/ts_guessing_question_based.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"a29d8491ca5349bd","mcp_get_code":{"code_sha256":"a29d8491ca5349bd"}},{"arxiv_id":"2403.14364","paper":"/paper/wikifactdiff-a-large-realistic-and-temporally","title":"WikiFactDiff: A Large, Realistic, and Temporally Adaptable Dataset for Atomic Factual Knowledge Update in Causal Language Models","date":"2024-03-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"orange-opensource/wikifactdiff","path":"evaluate/baselines/prompt.py","file_url":"https://github.com/orange-opensource/wikifactdiff/blob/HEAD/evaluate/baselines/prompt.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5d99924f6d4fd65d","mcp_get_code":{"code_sha256":"5d99924f6d4fd65d"}},{"arxiv_id":"2403.13187","paper":"/paper/evolutionary-optimization-of-model-merging","title":"Evolutionary Optimization of Model Merging Recipes","date":"2024-03-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"sakanaai/evolutionary-model-merge","path":"evomerge/models/llava.py","file_url":"https://github.com/sakanaai/evolutionary-model-merge/blob/HEAD/evomerge/models/llava.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"cd57ef201d9c8c10","mcp_get_code":{"code_sha256":"cd57ef201d9c8c10"}},{"arxiv_id":"2402.11725","paper":"/paper/how-susceptible-are-large-language-models-to","title":"How Susceptible are Large Language Models to Ideological Manipulation?","date":"2024-02-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kaichen23/llm_ideo_manipulate","path":"code/run_tuned_llama2.py","file_url":"https://github.com/kaichen23/llm_ideo_manipulate/blob/HEAD/code/run_tuned_llama2.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ab98f3588bdf16a7","mcp_get_code":{"code_sha256":"ab98f3588bdf16a7"}},{"arxiv_id":"2402.05785","paper":"/paper/limits-of-transformer-language-models-on","title":"Limits of Transformer Language Models on Learning to Compose Algorithms","date":"2024-02-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ibm/limitations-lm-algorithmic-compositional-learning","path":"prompting_experiment_utils/prompt_builder_perm.py","file_url":"https://github.com/ibm/limitations-lm-algorithmic-compositional-learning/blob/HEAD/prompting_experiment_utils/prompt_builder_perm.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7bfe356714c7bb6d","mcp_get_code":{"code_sha256":"7bfe356714c7bb6d"}},{"arxiv_id":"2401.15391","paper":"/paper/multihop-rag-benchmarking-retrieval-augmented","title":"MultiHop-RAG: Benchmarking Retrieval-Augmented Generation for Multi-Hop Queries","date":"2024-01-27","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yixuantt/MultiHop-RAG","path":"qa_llama.py","file_url":"https://github.com/yixuantt/MultiHop-RAG/blob/HEAD/qa_llama.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"0768880b580a1361","mcp_get_code":{"code_sha256":"0768880b580a1361"}},{"arxiv_id":"2401.07950","paper":"/paper/sciglm-training-scientific-language-models","title":"SciInstruct: a Self-Reflective Instruction Annotated Dataset for Training Scientific Language Models","date":"2024-01-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THUDM/SciGLM","path":"inference/cli_demo.py","file_url":"https://github.com/THUDM/SciGLM/blob/HEAD/inference/cli_demo.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"938a3cf8db709378","mcp_get_code":{"code_sha256":"938a3cf8db709378"}},{"arxiv_id":"2401.06877","paper":"/paper/promptly-predicting-structures-the-return-of","title":"Promptly Predicting Structures: The Return of Inference","date":"2024-01-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"utahnlp/prompts-for-structures","path":"src/tasks/srl/qasrl2/prompts.py","file_url":"https://github.com/utahnlp/prompts-for-structures/blob/HEAD/src/tasks/srl/qasrl2/prompts.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"c050aaa66fbc820f","mcp_get_code":{"code_sha256":"c050aaa66fbc820f"}},{"arxiv_id":"2312.15710","paper":"/paper/alleviating-hallucinations-of-large-language","title":"Alleviating Hallucinations of Large Language Models through Induced Hallucinations","date":"2023-12-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hillzhang1999/icd","path":"src/benchmark_evaluation/factscore_eval.py","file_url":"https://github.com/hillzhang1999/icd/blob/HEAD/src/benchmark_evaluation/factscore_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"db8a8ad434c0dd70","mcp_get_code":{"code_sha256":"db8a8ad434c0dd70"}},{"arxiv_id":"2311.16832","paper":"/paper/characterglm-customizing-chinese","title":"CharacterGLM: Customizing Chinese Conversational AI Characters with Large Language Models","date":"2023-11-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"thu-coai/characterglm-6b","path":"basic_demo/cli_demo.py","file_url":"https://github.com/thu-coai/characterglm-6b/blob/HEAD/basic_demo/cli_demo.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"0b952c9d3758f73e","mcp_get_code":{"code_sha256":"0b952c9d3758f73e"}},{"arxiv_id":"2311.07064","paper":"/paper/propane-prompt-design-as-an-inverse-problem","title":"Prompts have evil twins","date":"2023-11-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rimon15/propane","path":"evil_twins/prompt_optim.py","file_url":"https://github.com/rimon15/propane/blob/HEAD/evil_twins/prompt_optim.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-2.0","inline_ok":false,"code_sha256_prefix":"9180e51a8ac82897","mcp_get_code":{"code_sha256":"9180e51a8ac82897"}},{"arxiv_id":"2311.01908","paper":"/paper/llm-driven-multimodal-target-volume","title":"LLM-driven Multimodal Target Volume Contouring in Radiation Oncology","date":"2023-11-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tvseg/mm-llm-ro","path":"utils/data_utils.py","file_url":"https://github.com/tvseg/mm-llm-ro/blob/HEAD/utils/data_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9bd73c5e00265752","mcp_get_code":{"code_sha256":"9bd73c5e00265752"}},{"arxiv_id":"2309.13308","paper":"/paper/calibrating-llm-based-evaluator","title":"Calibrating LLM-Based Evaluator","date":"2023-09-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"FKIRSTE/emnlp2024-personalized-meeting-sum","path":"baseline/G-all.py","file_url":"https://github.com/FKIRSTE/emnlp2024-personalized-meeting-sum/blob/HEAD/baseline/G-all.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7fc2fb12340dc132","mcp_get_code":{"code_sha256":"7fc2fb12340dc132"}},{"arxiv_id":"2309.13308","paper":"/paper/calibrating-llm-based-evaluator","title":"Calibrating LLM-Based Evaluator","date":"2023-09-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"FKIRSTE/emnlp2024-personalized-meeting-sum","path":"baseline/OUT-P-all.py","file_url":"https://github.com/FKIRSTE/emnlp2024-personalized-meeting-sum/blob/HEAD/baseline/OUT-P-all.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"101c21be672b0d7b","mcp_get_code":{"code_sha256":"101c21be672b0d7b"}},{"arxiv_id":"2309.13308","paper":"/paper/calibrating-llm-based-evaluator","title":"Calibrating LLM-Based Evaluator","date":"2023-09-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"FKIRSTE/emnlp2024-personalized-meeting-sum","path":"baseline/P-all.py","file_url":"https://github.com/FKIRSTE/emnlp2024-personalized-meeting-sum/blob/HEAD/baseline/P-all.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"8d97761d2b2796e5","mcp_get_code":{"code_sha256":"8d97761d2b2796e5"}},{"arxiv_id":"2309.13308","paper":"/paper/calibrating-llm-based-evaluator","title":"Calibrating LLM-Based Evaluator","date":"2023-09-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"FKIRSTE/emnlp2024-personalized-meeting-sum","path":"baseline/P-none.py","file_url":"https://github.com/FKIRSTE/emnlp2024-personalized-meeting-sum/blob/HEAD/baseline/P-none.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"228f2ec1f2816b90","mcp_get_code":{"code_sha256":"228f2ec1f2816b90"}},{"arxiv_id":"2306.05087","paper":"/paper/pandalm-an-automatic-evaluation-benchmark-for","title":"PandaLM: An Automatic Evaluation Benchmark for LLM Instruction Tuning Optimization","date":"2023-06-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"weopenml/pandalm","path":"pandalm/run-gradio.py","file_url":"https://github.com/weopenml/pandalm/blob/HEAD/pandalm/run-gradio.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f4bb8eecfe1a4f76","mcp_get_code":{"code_sha256":"f4bb8eecfe1a4f76"}},{"arxiv_id":"2306.04188","paper":"/paper/a-new-dataset-and-empirical-study-for","title":"A New Dataset and Empirical Study for Sentence Simplification in Chinese","date":"2023-06-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THUDM/ChatGLM-6B","path":"cli_demo.py","file_url":"https://github.com/THUDM/ChatGLM-6B/blob/HEAD/cli_demo.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"20f3c3d6999848e9","mcp_get_code":{"code_sha256":"20f3c3d6999848e9"}},{"arxiv_id":"2306.04188","paper":"/paper/a-new-dataset-and-empirical-study-for","title":"A New Dataset and Empirical Study for Sentence Simplification in Chinese","date":"2023-06-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THUDM/ChatGLM-6B","path":"cli_demo_vision.py","file_url":"https://github.com/THUDM/ChatGLM-6B/blob/HEAD/cli_demo_vision.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7ce6ba0a6898058f","mcp_get_code":{"code_sha256":"7ce6ba0a6898058f"}},{"arxiv_id":"2305.12524","paper":"/paper/theoremqa-a-theorem-driven-question-answering","title":"TheoremQA: A Theorem-driven Question Answering dataset","date":"2023-05-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"THUDM/VisualGLM-6B","path":"cli_demo_hf.py","file_url":"https://github.com/THUDM/VisualGLM-6B/blob/HEAD/cli_demo_hf.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b7583b2677118645","mcp_get_code":{"code_sha256":"b7583b2677118645"}},{"arxiv_id":"2106.12574","paper":"/paper/deepstochlog-neural-stochastic-logic","title":"DeepStochLog: Neural Stochastic Logic Programming","date":"2021-06-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"abcf216a0f054de7","mcp_get_code":{"code_sha256":"abcf216a0f054de7"}},{"arxiv_id":"1809.08887","paper":"/paper/spider-a-large-scale-human-labeled-dataset","title":"Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task","date":"2018-09-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ML-KULeuven/deepstochlog-lm","path":"src/task2/task2.py","file_url":"https://github.com/ML-KULeuven/deepstochlog-lm/blob/HEAD/src/task2/task2.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"abcf216a0f054de7","mcp_get_code":{"code_sha256":"abcf216a0f054de7"}},{"arxiv_id":"openreview_XtIRCAEYoJ","paper":null,"title":"arXiv:openreview_XtIRCAEYoJ","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"WayneTomas/Artemis","path":"infer_artemis.py","file_url":"https://github.com/WayneTomas/Artemis/blob/HEAD/infer_artemis.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"5b32b161ba880bb2","mcp_get_code":{"code_sha256":"5b32b161ba880bb2"}},{"arxiv_id":"2025.findings-acl.804","paper":null,"title":"arXiv:2025.findings-acl.804","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"WalterPaci/IMPAQTS-PID","path":"MCG_task.py","file_url":"https://github.com/WalterPaci/IMPAQTS-PID/blob/HEAD/MCG_task.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1db7ebc69f9ff2cb","mcp_get_code":{"code_sha256":"1db7ebc69f9ff2cb"}}]}