{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/construct-prompt","entry":"construct_prompt","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":24,"n_papers_ran":11,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":22,"n_samples_ran":8,"n_samples_fingerprinted":2,"n_places":26,"n_places_pointer_only":10,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":5,"ran_fixture":0,"ran":3,"unverified":14},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2608.07430","paper":"/paper/arxiv-2608-07430","title":"Diffusion LLMs as Targets and Adversaries: Mechanistic Safety Exploits","date":null,"month_inferred_from_arxiv_id":"2026-08","title_source":"syntology","repo":"ellyoana/sn-guided-diffusion","path":"utils.py","file_url":"https://github.com/ellyoana/sn-guided-diffusion/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"af81b308c704f33f","mcp_get_code":{"code_sha256":"af81b308c704f33f"}},{"arxiv_id":"2603.20572","paper":"/paper/arxiv-2603-20572","title":"LJ-Bench: Ontology-Based Benchmark for U.S. Crime","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"AndreaTseng/LJ-Bench","path":"src/format_prompt.py","file_url":"https://github.com/AndreaTseng/LJ-Bench/blob/HEAD/src/format_prompt.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"645c86835778befe","mcp_get_code":{"code_sha256":"645c86835778befe"}},{"arxiv_id":"2601.04435","paper":"/paper/arxiv-2601-04435","title":"Accommodation and Epistemic Vigilance: A Pragmatic Account of Why LLMs Fail to Challenge Harmful Beliefs","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"myracheng/accommodation","path":"evaluation_scripts/cancereval.py","file_url":"https://github.com/myracheng/accommodation/blob/HEAD/evaluation_scripts/cancereval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c0260539abdf7bc3","mcp_get_code":{"code_sha256":"c0260539abdf7bc3"}},{"arxiv_id":"2601.04435","paper":"/paper/arxiv-2601-04435","title":"Accommodation and Epistemic Vigilance: A Pragmatic Account of Why LLMs Fail to Challenge Harmful Beliefs","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"myracheng/accommodation","path":"evaluation_scripts/cancernfpeval.py","file_url":"https://github.com/myracheng/accommodation/blob/HEAD/evaluation_scripts/cancernfpeval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"232b2b752503f49b","mcp_get_code":{"code_sha256":"232b2b752503f49b"}},{"arxiv_id":"2510.20342","paper":"/paper/arxiv-2510-20342","title":"Teaching Language Models to Reason with Tools","date":null,"month_inferred_from_arxiv_id":"2025-10","title_source":"syntology","repo":"ChengpengLi1003/CoRT","path":"infer/utils.py","file_url":"https://github.com/ChengpengLi1003/CoRT/blob/HEAD/infer/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ecdacf069c9d10db","mcp_get_code":{"code_sha256":"ecdacf069c9d10db"}},{"arxiv_id":"2505.13778","paper":"/paper/coin-counting-the-invisible-reasoning-tokens","title":"CoIn: Counting the Invisible Reasoning Tokens in Commercial Opaque LLM APIs","date":"2025-05-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"case-lab-umd/llm-auditing-coin","path":"6_discussion/LLM_predict.py","file_url":"https://github.com/case-lab-umd/llm-auditing-coin/blob/HEAD/6_discussion/LLM_predict.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2843e3f35ce11d01","mcp_get_code":{"code_sha256":"2843e3f35ce11d01"}},{"arxiv_id":"2505.13778","paper":"/paper/coin-counting-the-invisible-reasoning-tokens","title":"CoIn: Counting the Invisible Reasoning Tokens in Commercial Opaque LLM APIs","date":"2025-05-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"case-lab-umd/llm-auditing-coin","path":"6_discussion/LLM_reconstruct.py","file_url":"https://github.com/case-lab-umd/llm-auditing-coin/blob/HEAD/6_discussion/LLM_reconstruct.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"577fc0eeba8dfc18","mcp_get_code":{"code_sha256":"577fc0eeba8dfc18"}},{"arxiv_id":"2408.17267","paper":"/paper/urbench-a-comprehensive-benchmark-for","title":"UrBench: A Comprehensive Benchmark for Evaluating Large Multimodal Models in Multi-View Urban Scenarios","date":"2024-08-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"opendatalab/urbench","path":"lmms_eval/tasks/urbench/utils.py","file_url":"https://github.com/opendatalab/urbench/blob/HEAD/lmms_eval/tasks/urbench/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"5096e10842a99b97","mcp_get_code":{"code_sha256":"5096e10842a99b97"}},{"arxiv_id":"2406.16852","paper":"/paper/long-context-transfer-from-language-to-vision","title":"Long Context Transfer from Language to Vision","date":"2024-06-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"2e94fa2078ca5f7c","mcp_get_code":{"code_sha256":"2e94fa2078ca5f7c"}},{"arxiv_id":"2406.14852","paper":"/paper/is-a-picture-worth-a-thousand-words-delving","title":"Is A Picture Worth A Thousand Words? Delving Into Spatial Reasoning for Vision Language Models","date":"2024-06-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"BAAI-DCAI/Bunny","path":"bunny/eval/model_vqa_cmmmu.py","file_url":"https://github.com/BAAI-DCAI/Bunny/blob/HEAD/bunny/eval/model_vqa_cmmmu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"2fd1673b298c086c","mcp_get_code":{"code_sha256":"2fd1673b298c086c"}},{"arxiv_id":"2406.11503","paper":"/paper/geogpt4v-towards-geometric-multi-modal-large","title":"GeoGPT4V: Towards Geometric Multi-modal Large Language Models with Geometric Image Generation","date":"2024-06-17","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lanyu0303/geogpt4v_project","path":"pipeline/gen_instruction_mp.py","file_url":"https://github.com/lanyu0303/geogpt4v_project/blob/HEAD/pipeline/gen_instruction_mp.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8a9188d48800ce95","mcp_get_code":{"code_sha256":"8a9188d48800ce95"}},{"arxiv_id":"2406.01424","paper":"/paper/universal-in-context-approximation-by","title":"Universal In-Context Approximation By Prompting Fully Recurrent Models","date":"2024-06-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"aleksandarpetrov/lsrl","path":"lsrl/utils.py","file_url":"https://github.com/aleksandarpetrov/lsrl/blob/HEAD/lsrl/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6c84f0afac911592","mcp_get_code":{"code_sha256":"6c84f0afac911592"}},{"arxiv_id":"2405.05949","paper":"/paper/cumo-scaling-multimodal-llm-with-co-upcycled","title":"CuMo: Scaling Multimodal LLM with Co-Upcycled Mixture-of-Experts","date":"2024-05-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"shi-labs/cumo","path":"cumo/eval/model_vqa_mathvista.py","file_url":"https://github.com/shi-labs/cumo/blob/HEAD/cumo/eval/model_vqa_mathvista.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"ed80aaedae521e4f","mcp_get_code":{"code_sha256":"ed80aaedae521e4f"}},{"arxiv_id":"2403.09488","paper":"/paper/rectifying-demonstration-shortcut-in-in","title":"Rectifying Demonstration Shortcut in In-Context Learning","date":"2024-03-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lainshower/in-context-calibration","path":"utils.py","file_url":"https://github.com/lainshower/in-context-calibration/blob/HEAD/utils.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e37bbdf75acd586c","mcp_get_code":{"code_sha256":"e37bbdf75acd586c"}},{"arxiv_id":"2402.11530","paper":"/paper/efficient-multimodal-learning-from-data","title":"Efficient Multimodal Learning from Data-centric Perspective","date":"2024-02-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"baai-dcai/bunny","path":"bunny/eval/model_vqa_cmmmu.py","file_url":"https://github.com/baai-dcai/bunny/blob/HEAD/bunny/eval/model_vqa_cmmmu.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"2fd1673b298c086c","mcp_get_code":{"code_sha256":"2fd1673b298c086c"}},{"arxiv_id":"2402.02823","paper":"/paper/evading-data-contamination-detection-for","title":"Evading Data Contamination Detection for Language Models is (too) Easy","date":"2024-02-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"eth-sri/malicious-contamination","path":"src/contamination/llm_decontaminator.py","file_url":"https://github.com/eth-sri/malicious-contamination/blob/HEAD/src/contamination/llm_decontaminator.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"255132d5b9dc4ec5","mcp_get_code":{"code_sha256":"255132d5b9dc4ec5"}},{"arxiv_id":"2312.04021","paper":"/paper/a-study-on-the-calibration-of-in-context","title":"A Study on the Calibration of In-context Learning","date":"2023-12-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"hlzhang109/icl-calibration","path":"run_classification_cot.py","file_url":"https://github.com/hlzhang109/icl-calibration/blob/HEAD/run_classification_cot.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ef2f02ab0c2e6b74","mcp_get_code":{"code_sha256":"ef2f02ab0c2e6b74"}},{"arxiv_id":"2310.01889","paper":"/paper/ring-attention-with-blockwise-transformers","title":"Ring Attention with Blockwise Transformers for Near-Infinite Context","date":"2023-10-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jzhang38/EasyContext","path":"eval_needle.py","file_url":"https://github.com/jzhang38/EasyContext/blob/HEAD/eval_needle.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"2e94fa2078ca5f7c","mcp_get_code":{"code_sha256":"2e94fa2078ca5f7c"}},{"arxiv_id":"2309.17452","paper":"/paper/tora-a-tool-integrated-reasoning-agent-for","title":"ToRA: A Tool-Integrated Reasoning Agent for Mathematical Problem Solving","date":"2023-09-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/tora","path":"src/utils/utils.py","file_url":"https://github.com/microsoft/tora/blob/HEAD/src/utils/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ecdacf069c9d10db","mcp_get_code":{"code_sha256":"ecdacf069c9d10db"}},{"arxiv_id":"2309.14509","paper":"/paper/deepspeed-ulysses-system-optimizations-for","title":"DeepSpeed Ulysses: System Optimizations for Enabling Training of Extreme Long Sequence Transformer Models","date":"2023-09-25","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":null,"path":"","file_url":null,"status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"code_sha256_prefix":"2e94fa2078ca5f7c","mcp_get_code":{"code_sha256":"2e94fa2078ca5f7c"}},{"arxiv_id":"2306.08568","paper":"/paper/wizardcoder-empowering-code-large-language","title":"WizardCoder: Empowering Code Large Language Models with Evol-Instruct","date":"2023-06-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kyle-lyu/data-efficient-finetuning","path":"utils/dataset.py","file_url":"https://github.com/kyle-lyu/data-efficient-finetuning/blob/HEAD/utils/dataset.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"fb57aad9f9c18c48","mcp_get_code":{"code_sha256":"fb57aad9f9c18c48"}},{"arxiv_id":"2303.08774","paper":"/paper/gpt-4-technical-report-1","title":"GPT-4 Technical Report","date":"2023-03-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"AUCOHL/RTL-Repo","path":"src/utils.py","file_url":"https://github.com/AUCOHL/RTL-Repo/blob/HEAD/src/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"522809481fa31587","mcp_get_code":{"code_sha256":"522809481fa31587"}},{"arxiv_id":"2212.09741","paper":"/paper/one-embedder-any-task-instruction-finetuned","title":"One Embedder, Any Task: Instruction-Finetuned Text Embeddings","date":"2022-12-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"HKUNLP/instructor-embedding","path":"evaluation/prompt_retrieval/main_geo.py","file_url":"https://github.com/HKUNLP/instructor-embedding/blob/HEAD/evaluation/prompt_retrieval/main_geo.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"012e0e03723d52ec","mcp_get_code":{"code_sha256":"012e0e03723d52ec"}},{"arxiv_id":"2204.07705","paper":"/paper/benchmarking-generalization-via-in-context","title":"Super-NaturalInstructions: Generalization via Declarative Instructions on 1600+ NLP Tasks","date":"2022-04-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"naver-ai/rethinking-proxy-reward","path":"evaluation.py","file_url":"https://github.com/naver-ai/rethinking-proxy-reward/blob/HEAD/evaluation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"308ab826d7562425","mcp_get_code":{"code_sha256":"308ab826d7562425"}},{"arxiv_id":"2025.findings-emnlp.103","paper":null,"title":"arXiv:2025.findings-emnlp.103","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"yha9806/VULCA-EMNLP2025","path":"src/evaluate.py","file_url":"https://github.com/yha9806/VULCA-EMNLP2025/blob/HEAD/src/evaluate.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"6da92e6b14d39573","mcp_get_code":{"code_sha256":"6da92e6b14d39573"}},{"arxiv_id":"2025.findings-acl.491","paper":null,"title":"arXiv:2025.findings-acl.491","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"biodatlab/ec-raft","path":"evaluate.py","file_url":"https://github.com/biodatlab/ec-raft/blob/HEAD/evaluate.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"bbd9e3b2b65460ef","mcp_get_code":{"code_sha256":"bbd9e3b2b65460ef"}}]}