{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/num-tokens-from-string","entry":"num_tokens_from_string","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":17,"n_papers_ran":1,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":15,"n_samples_ran":1,"n_samples_fingerprinted":0,"n_places":17,"n_places_pointer_only":9,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":0,"ran_fixture":0,"ran":1,"unverified":14},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2505.18673","paper":"/paper/cross-lingual-pitfalls-automatic-probing","title":"Cross-Lingual Pitfalls: Automatic Probing Cross-Lingual Weakness of Multilingual Large Language Models","date":"2025-05-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"xzx34/Cross-Lingual-Pitfalls","path":"utils/tools.py","file_url":"https://github.com/xzx34/Cross-Lingual-Pitfalls/blob/HEAD/utils/tools.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1ab2a84c0b4c9b0a","mcp_get_code":{"code_sha256":"1ab2a84c0b4c9b0a"}},{"arxiv_id":"2502.18754","paper":"/paper/agentsociety-challenge-designing-llm-agents","title":"AgentSociety Challenge: Designing LLM Agents for User Modeling and Recommendation on Web Platforms","date":"2025-02-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tsinghua-fib-lab/agentsocietychallenge","path":"GTsimulation/RecGTAgent.py","file_url":"https://github.com/tsinghua-fib-lab/agentsocietychallenge/blob/HEAD/GTsimulation/RecGTAgent.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"24ed49c54abe8af9","mcp_get_code":{"code_sha256":"24ed49c54abe8af9"}},{"arxiv_id":"2502.14768","paper":"/paper/logic-rl-unleashing-llm-reasoning-with-rule","title":"Logic-RL: Unleashing LLM Reasoning with Rule-Based Reinforcement Learning","date":"2025-02-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Unakar/Logic-RL","path":"eval_kk/kk_processor.py","file_url":"https://github.com/Unakar/Logic-RL/blob/HEAD/eval_kk/kk_processor.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"26b075196c1c10bf","mcp_get_code":{"code_sha256":"26b075196c1c10bf"}},{"arxiv_id":"2411.04998","paper":"/paper/hourvideo-1-hour-video-language-understanding","title":"HourVideo: 1-Hour Video-Language Understanding","date":"2024-11-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"keshik6/HourVideo","path":"hourvideo/llm_utils.py","file_url":"https://github.com/keshik6/HourVideo/blob/HEAD/hourvideo/llm_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"dfc109b0a442de2e","mcp_get_code":{"code_sha256":"dfc109b0a442de2e"}},{"arxiv_id":"2410.14211","paper":"/paper/paths-over-graph-knowledge-graph-enpowered","title":"Paths-over-Graph: Knowledge Graph Empowered Large Language Model Reasoning","date":"2024-10-18","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"SteveTANTAN/PoG","path":"PoG/PoG_multi.py","file_url":"https://github.com/SteveTANTAN/PoG/blob/HEAD/PoG/PoG_multi.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"51dbd68f4d5aba9e","mcp_get_code":{"code_sha256":"51dbd68f4d5aba9e"}},{"arxiv_id":"2407.12861","paper":"/paper/citeme-can-language-models-accurately-cite","title":"CiteME: Can Language Models Accurately Cite Scientific Claims?","date":"2024-07-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bethgelab/CiteME","path":"src/utils/tokens.py","file_url":"https://github.com/bethgelab/CiteME/blob/HEAD/src/utils/tokens.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"0600b1d3da30c23a","mcp_get_code":{"code_sha256":"0600b1d3da30c23a"}},{"arxiv_id":"2406.02746","paper":"/paper/ratt-athought-structure-for-coherent-and","title":"RATT: A Thought Structure for Coherent and Correct LLM Reasoning","date":"2024-06-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jinghanzhang1998/ratt","path":"example_creative_writing.py","file_url":"https://github.com/jinghanzhang1998/ratt/blob/HEAD/example_creative_writing.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e670e93bbcabe5da","mcp_get_code":{"code_sha256":"e670e93bbcabe5da"}},{"arxiv_id":"2404.04960","paper":"/paper/pairaug-what-can-augmented-image-text-pairs","title":"PairAug: What Can Augmented Image-Text Pairs Do for Radiology?","date":"2024-04-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"YtongXie/PairAug","path":"InterAug_Step1.py","file_url":"https://github.com/YtongXie/PairAug/blob/HEAD/InterAug_Step1.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"4511b96bc04402fb","mcp_get_code":{"code_sha256":"4511b96bc04402fb"}},{"arxiv_id":"2402.05668","paper":"/paper/comprehensive-assessment-of-jailbreak-attacks","title":"JailbreakRadar: Comprehensive Assessment of Jailbreak Attacks Against LLMs","date":"2024-02-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"TrustAIRLab/Comprehensive_Jailbreak_Assessment","path":"scripts_label/label.py","file_url":"https://github.com/TrustAIRLab/Comprehensive_Jailbreak_Assessment/blob/HEAD/scripts_label/label.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"51dbd68f4d5aba9e","mcp_get_code":{"code_sha256":"51dbd68f4d5aba9e"}},{"arxiv_id":"2311.16714","paper":"/paper/embodied-multi-modal-agent-trained-by-an-llm","title":"Embodied Multi-Modal Agent trained by an LLM from a Parallel TextWorld","date":"2023-11-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"stevenyangyj/emma-alfworld","path":"LAVIS/emma_utils.py","file_url":"https://github.com/stevenyangyj/emma-alfworld/blob/HEAD/LAVIS/emma_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"918526f0d411a7aa","mcp_get_code":{"code_sha256":"918526f0d411a7aa"}},{"arxiv_id":"2311.09648","paper":"/paper/event-causality-is-key-to-computational-story","title":"Event Causality Is Key to Computational Story Understanding","date":"2023-11-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"insundaycathy/event-causality-extraction","path":"story_eval/OpenAI_API_score.py","file_url":"https://github.com/insundaycathy/event-causality-extraction/blob/HEAD/story_eval/OpenAI_API_score.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"22d60148a6f4dbcb","mcp_get_code":{"code_sha256":"22d60148a6f4dbcb"}},{"arxiv_id":"2310.20499","paper":"/paper/leveraging-word-guessing-games-to-assess-the","title":"Leveraging Word Guessing Games to Assess the Intelligence of Large Language Models","date":"2023-10-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"skytliang/spygame","path":"utils/openai_utils.py","file_url":"https://github.com/skytliang/spygame/blob/HEAD/utils/openai_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"code_sha256_prefix":"0600b1d3da30c23a","mcp_get_code":{"code_sha256":"0600b1d3da30c23a"}},{"arxiv_id":"2310.15239","paper":"/paper/crow-benchmarking-commonsense-reasoning-in","title":"CRoW: Benchmarking Commonsense Reasoning in Real-World Tasks","date":"2023-10-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mismayil/crow","path":"utils.py","file_url":"https://github.com/mismayil/crow/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c8b4564640ec9706","mcp_get_code":{"code_sha256":"c8b4564640ec9706"}},{"arxiv_id":"2310.15147","paper":"/paper/s3eval-a-synthetic-scalable-systematic","title":"S3Eval: A Synthetic, Scalable, Systematic Evaluation Suite for Large Language Models","date":"2023-10-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lfy79001/sqleval","path":"eval/llama2.py","file_url":"https://github.com/lfy79001/sqleval/blob/HEAD/eval/llama2.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"5ee9501ae032577a","mcp_get_code":{"code_sha256":"5ee9501ae032577a"}},{"arxiv_id":"openreview_8ppVmLtA2V","paper":null,"title":"arXiv:openreview_8ppVmLtA2V","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"yanweiyue/Mem-T","path":"llm_stats.py","file_url":"https://github.com/yanweiyue/Mem-T/blob/HEAD/llm_stats.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"baba60077ac5b966","mcp_get_code":{"code_sha256":"baba60077ac5b966"}},{"arxiv_id":"2024.findings-emnlp.253","paper":null,"title":"arXiv:2024.findings-emnlp.253","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"WilliamZR/ProTrix","path":"evaluation/sql_tool.py","file_url":"https://github.com/WilliamZR/ProTrix/blob/HEAD/evaluation/sql_tool.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"336dce91b100570d","mcp_get_code":{"code_sha256":"336dce91b100570d"}},{"arxiv_id":"2024.findings-acl.326","paper":null,"title":"arXiv:2024.findings-acl.326","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"dscc-admin-ch/statbot.swiss","path":"src/generation.py","file_url":"https://github.com/dscc-admin-ch/statbot.swiss/blob/HEAD/src/generation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"code_sha256_prefix":"674d436096c536c0","mcp_get_code":{"code_sha256":"674d436096c536c0"}}]}