{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/benchmark","entry":"benchmark","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":19,"n_papers_ran":5,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":19,"n_samples_ran":5,"n_samples_fingerprinted":0,"n_places":20,"n_places_pointer_only":6,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":1,"ran_fixture":1,"ran":3,"unverified":14},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2604.04192","paper":"/paper/arxiv-2604-04192","title":"Graphic-Design-Bench: A Comprehensive Benchmark for Evaluating AI on Graphic Design Tasks","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"purvanshi-lica/lica-bench","path":"src/design_benchmarks/base.py","file_url":"https://github.com/purvanshi-lica/lica-bench/blob/HEAD/src/design_benchmarks/base.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"fad64c10e9cfb2d0","mcp_get_code":{"code_sha256":"fad64c10e9cfb2d0"}},{"arxiv_id":"2410.08131","paper":"/paper/deconstructing-equivariant-representations-in","title":"Deconstructing equivariant representations in molecular systems","date":"2024-10-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"IntelLabs/EquiTriton","path":"src/equitriton/benchmark.py","file_url":"https://github.com/IntelLabs/EquiTriton/blob/HEAD/src/equitriton/benchmark.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"d6908997456703d6","mcp_get_code":{"code_sha256":"d6908997456703d6"}},{"arxiv_id":"2410.07765","paper":"/paper/gametraversalbenchmark-evaluating-planning","title":"GameTraversalBenchmark: Evaluating Planning Abilities Of Large Language Models Through Traversing 2D Game Maps","date":"2024-10-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"umair-nasir14/game-traversal-benchmark","path":"gtb/gtbench.py","file_url":"https://github.com/umair-nasir14/game-traversal-benchmark/blob/HEAD/gtb/gtbench.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e29c7994358b3427","mcp_get_code":{"code_sha256":"e29c7994358b3427"}},{"arxiv_id":"2408.11743","paper":"/paper/marlin-mixed-precision-auto-regressive","title":"MARLIN: Mixed-Precision Auto-Regressive Parallel Inference on Large Language Models","date":"2024-08-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ist-daslab/sparse-marlin","path":"bench_2_4.py","file_url":"https://github.com/ist-daslab/sparse-marlin/blob/HEAD/bench_2_4.py","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"f7a0c2b4b3e78765","mcp_get_code":{"code_sha256":"f7a0c2b4b3e78765"}},{"arxiv_id":"2408.05148","paper":"/paper/impacts-of-floating-point-non-associativity","title":"Impacts of floating-point non-associativity on reproducibility for HPC and deep learning applications","date":"2024-08-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"minnervva/correctness","path":"codes/test-suite/scaffold.py","file_url":"https://github.com/minnervva/correctness/blob/HEAD/codes/test-suite/scaffold.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"39e71e53f18485c7","mcp_get_code":{"code_sha256":"39e71e53f18485c7"}},{"arxiv_id":"2408.05148","paper":"/paper/impacts-of-floating-point-non-associativity","title":"Impacts of floating-point non-associativity on reproducibility for HPC and deep learning applications","date":"2024-08-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"minnervva/correctness","path":"codes/test-suite/.ipynb_checkpoints/scaffold-checkpoint.py","file_url":"https://github.com/minnervva/correctness/blob/HEAD/codes/test-suite/.ipynb_checkpoints/scaffold-checkpoint.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"617854884f5a9e34","mcp_get_code":{"code_sha256":"617854884f5a9e34"}},{"arxiv_id":"2406.09936","paper":"/paper/algm-adaptive-local-then-global-token-merging-1","title":"ALGM: Adaptive Local-then-Global Token Merging for Efficient Semantic Segmentation with Plain Vision Transformers","date":"2024-06-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tue-mps/algm-segmenter","path":"algm/algm/utils.py","file_url":"https://github.com/tue-mps/algm-segmenter/blob/HEAD/algm/algm/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"eaef07a98678e866","mcp_get_code":{"code_sha256":"eaef07a98678e866"}},{"arxiv_id":"2405.08958","paper":"/paper/learned-radio-interferometric-imaging-for","title":"Learned radio interferometric imaging for varying visibility coverage","date":"2024-05-14","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"astro-informatics/leia","path":"src/benchmark.py","file_url":"https://github.com/astro-informatics/leia/blob/HEAD/src/benchmark.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1af9166732c94e2a","mcp_get_code":{"code_sha256":"1af9166732c94e2a"}},{"arxiv_id":"2405.03633","paper":"/paper/neural-graph-mapping-for-dense-slam-with","title":"Neural Graph Map: Dense Mapping with Efficient Loop Closure Integration","date":"2024-05-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kth-rpl/neural_graph_mapping","path":"src/neural_graph_mapping/utils.py","file_url":"https://github.com/kth-rpl/neural_graph_mapping/blob/HEAD/src/neural_graph_mapping/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"2554d294968c8675","mcp_get_code":{"code_sha256":"2554d294968c8675"}},{"arxiv_id":"2401.00588","paper":"/paper/fairness-in-serving-large-language-models","title":"Fairness in Serving Large Language Models","date":"2023-12-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"s-lora/s-lora","path":"benchmarks/run_exp.py","file_url":"https://github.com/s-lora/s-lora/blob/HEAD/benchmarks/run_exp.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6e238d2d144eed9b","mcp_get_code":{"code_sha256":"6e238d2d144eed9b"}},{"arxiv_id":"2312.05388","paper":"/paper/higher-order-equivariant-neural-networks-for","title":"Higher-Order Equivariant Neural Networks for Charge Density Prediction in Materials","date":"2023-12-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"teddykoker/e3nn.c","path":"extra/benchmark_python.py","file_url":"https://github.com/teddykoker/e3nn.c/blob/HEAD/extra/benchmark_python.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1fbb35c3cdd6e9db","mcp_get_code":{"code_sha256":"1fbb35c3cdd6e9db"}},{"arxiv_id":"2310.19060","paper":"/paper/testa-temporal-spatial-token-aggregation-for","title":"TESTA: Temporal-Spatial Token Aggregation for Long-form Video-Language Understanding","date":"2023-10-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"renshuhuai-andy/testa","path":"testa/utils.py","file_url":"https://github.com/renshuhuai-andy/testa/blob/HEAD/testa/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"eaef07a98678e866","mcp_get_code":{"code_sha256":"eaef07a98678e866"}},{"arxiv_id":"2306.12517","paper":"/paper/ffcv-accelerating-training-by-removing-data-1","title":"FFCV: Accelerating Training by Removing Data Bottlenecks","date":"2023-06-21","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"libffcv/ffcv","path":"ffcv/benchmarks/decorator.py","file_url":"https://github.com/libffcv/ffcv/blob/HEAD/ffcv/benchmarks/decorator.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"fa27dedd4c856c14","mcp_get_code":{"code_sha256":"fa27dedd4c856c14"}},{"arxiv_id":"2306.00989","paper":"/paper/hiera-a-hierarchical-vision-transformer","title":"Hiera: A Hierarchical Vision Transformer without the Bells-and-Whistles","date":"2023-06-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"facebookresearch/hiera","path":"hiera/benchmarking.py","file_url":"https://github.com/facebookresearch/hiera/blob/HEAD/hiera/benchmarking.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b160eae20550159c","mcp_get_code":{"code_sha256":"b160eae20550159c"}},{"arxiv_id":"2210.06423","paper":"/paper/foundation-transformers","title":"Foundation Transformers","date":"2022-10-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"fkodom/dilated-attention-pytorch","path":"benchmark.py","file_url":"https://github.com/fkodom/dilated-attention-pytorch/blob/HEAD/benchmark.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"36a98ee3c7774373","mcp_get_code":{"code_sha256":"36a98ee3c7774373"}},{"arxiv_id":"2203.04353","paper":"/paper/unrolled-primal-dual-networks-for-lensless","title":"Unrolled Primal-Dual Networks for Lensless Cameras","date":"2022-03-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"oliland/lensless-primal-dual","path":"experiment.py","file_url":"https://github.com/oliland/lensless-primal-dual/blob/HEAD/experiment.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"107a9ffd3182f853","mcp_get_code":{"code_sha256":"107a9ffd3182f853"}},{"arxiv_id":"2202.09888","paper":"/paper/benchmarking-the-linear-algebra-awareness-of","title":"Benchmarking the Linear Algebra Awareness of TensorFlow and PyTorch","date":"2022-02-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"chrispsa/lamp_benchmark","path":"python/numpy/benchmarker.py","file_url":"https://github.com/chrispsa/lamp_benchmark/blob/HEAD/python/numpy/benchmarker.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"code_sha256_prefix":"5f6adb6e03177270","mcp_get_code":{"code_sha256":"5f6adb6e03177270"}},{"arxiv_id":"1907.10621","paper":"/paper/madminer-machine-learning-based-inference-for","title":"MadMiner: Machine learning-based inference for particle physics","date":"2019-07-24","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"aghoshpub/LikelihoodFreeInterference","path":"madminer/sampling.py","file_url":"https://github.com/aghoshpub/LikelihoodFreeInterference/blob/HEAD/madminer/sampling.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8e6678d029a7aa7f","mcp_get_code":{"code_sha256":"8e6678d029a7aa7f"}},{"arxiv_id":"1703.09710","paper":"/paper/170309710","title":"Fast and scalable Gaussian process modeling with applications to astronomical time series","date":null,"month_inferred_from_arxiv_id":"2017-03","title_source":"archive","repo":"dfm/celerite","path":"celerite/timer.py","file_url":"https://github.com/dfm/celerite/blob/HEAD/celerite/timer.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"89742dd28562c866","mcp_get_code":{"code_sha256":"89742dd28562c866"}},{"arxiv_id":"quant-ph/0406196","paper":"/paper/improved-simulation-of-stabilizer-circuits","title":"Improved Simulation of Stabilizer Circuits","date":null,"month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"quantumlib/stim","path":"src/stim/circuit/circuit_pybind_perf.py","file_url":"https://github.com/quantumlib/stim/blob/HEAD/src/stim/circuit/circuit_pybind_perf.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"1205728b447c787a","mcp_get_code":{"code_sha256":"1205728b447c787a"}}]}