{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/metric","entry":"metric","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":14,"n_papers_ran":3,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":18,"n_samples_ran":3,"n_samples_fingerprinted":1,"n_places":20,"n_places_pointer_only":3,"by_status":{"ran_honours":0,"ran_violates":1,"ran_draft_wrong":0,"ran_fixture":0,"ran":2,"unverified":15},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2609.04105","paper":"/paper/arxiv-2609-04105","title":"Hardware-Aware FP4 FlashAttention-4","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"MrHuff/fp4-fa4","path":"results/fp4_fa4_unified_20260801/build_summary.py","file_url":"https://github.com/MrHuff/fp4-fa4/blob/HEAD/results/fp4_fa4_unified_20260801/build_summary.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e5b3157e0edbda66","mcp_get_code":{"code_sha256":"e5b3157e0edbda66"}},{"arxiv_id":"2605.01148","paper":"/paper/arxiv-2605-01148","title":"Arithmetic in the Wild: Llama uses Base-10 Addition to Reason About Cyclic Concepts","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"goodfire-ai/arithmetic-wild","path":"src/utils.py","file_url":"https://github.com/goodfire-ai/arithmetic-wild/blob/HEAD/src/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"a66b49b30c54c8d3","mcp_get_code":{"code_sha256":"a66b49b30c54c8d3"}},{"arxiv_id":"2505.19715","paper":"/paper/graceful-forgetting-in-generative-language","title":"Graceful Forgetting in Generative Language Models","date":"2025-05-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rubickkcibur/LWF","path":"dataset_processor/gsm8k.py","file_url":"https://github.com/rubickkcibur/LWF/blob/HEAD/dataset_processor/gsm8k.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"613767452ccfb987","mcp_get_code":{"code_sha256":"613767452ccfb987"}},{"arxiv_id":"2505.19715","paper":"/paper/graceful-forgetting-in-generative-language","title":"Graceful Forgetting in Generative Language Models","date":"2025-05-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rubickkcibur/LWF","path":"dataset_processor/gsm8k_es.py","file_url":"https://github.com/rubickkcibur/LWF/blob/HEAD/dataset_processor/gsm8k_es.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"0229ae5a5851afa2","mcp_get_code":{"code_sha256":"0229ae5a5851afa2"}},{"arxiv_id":"2505.19715","paper":"/paper/graceful-forgetting-in-generative-language","title":"Graceful Forgetting in Generative Language Models","date":"2025-05-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rubickkcibur/LWF","path":"dataset_processor/gsm8k_it.py","file_url":"https://github.com/rubickkcibur/LWF/blob/HEAD/dataset_processor/gsm8k_it.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"30c7c57bcdd3cbfd","mcp_get_code":{"code_sha256":"30c7c57bcdd3cbfd"}},{"arxiv_id":"2505.19715","paper":"/paper/graceful-forgetting-in-generative-language","title":"Graceful Forgetting in Generative Language Models","date":"2025-05-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rubickkcibur/LWF","path":"dataset_processor/gsm8k_tr.py","file_url":"https://github.com/rubickkcibur/LWF/blob/HEAD/dataset_processor/gsm8k_tr.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"a4dfc117d1bf8efc","mcp_get_code":{"code_sha256":"a4dfc117d1bf8efc"}},{"arxiv_id":"2504.00762","paper":"/paper/do-we-truly-need-so-many-samples-multi-llm","title":"Do We Truly Need So Many Samples? Multi-LLM Repeated Sampling Efficiently Scales Test-Time Compute","date":"2025-04-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jianhaochen-nju/modelswitch","path":"src/Model_switch.py","file_url":"https://github.com/jianhaochen-nju/modelswitch/blob/HEAD/src/Model_switch.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"1810436c16248ab1","mcp_get_code":{"code_sha256":"1810436c16248ab1"}},{"arxiv_id":"2412.09972","paper":"/paper/efficient-large-scale-traffic-forecasting","title":"Efficient Large-Scale Traffic Forecasting with Transformers: A Spatial Data Management Perspective","date":"2024-12-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lmissher/patchstg","path":"lib/utils.py","file_url":"https://github.com/lmissher/patchstg/blob/HEAD/lib/utils.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5b230abb2d7a66c3","mcp_get_code":{"code_sha256":"5b230abb2d7a66c3"}},{"arxiv_id":"2412.09972","paper":"/paper/efficient-large-scale-traffic-forecasting","title":"Efficient Large-Scale Traffic Forecasting with Transformers: A Spatial Data Management Perspective","date":"2024-12-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"lmissher/stgnn","path":"utils.py","file_url":"https://github.com/lmissher/stgnn/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1ba47a23adf07065","mcp_get_code":{"code_sha256":"1ba47a23adf07065"}},{"arxiv_id":"2405.12203","paper":"/paper/accelerating-relative-entropy-coding-with","title":"Accelerating Relative Entropy Coding with Space Partitioning","date":"2024-05-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"cambridge-mlg/RECOMBINER","path":"utils.py","file_url":"https://github.com/cambridge-mlg/RECOMBINER/blob/HEAD/utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9dfabc490a555cab","mcp_get_code":{"code_sha256":"9dfabc490a555cab"}},{"arxiv_id":"2306.09104","paper":"/paper/on-strengthening-and-defending-graph","title":"On Strengthening and Defending Graph Reconstruction Attack with Markov Chain Approximation","date":"2023-06-15","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"tmlr-group/MC-GRA","path":"MC-GRA/gaussian_parameterized.py","file_url":"https://github.com/tmlr-group/MC-GRA/blob/HEAD/MC-GRA/gaussian_parameterized.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ffdd100106716576","mcp_get_code":{"code_sha256":"ffdd100106716576"}},{"arxiv_id":"2302.02738","paper":"/paper/increase-inductive-graph-representation","title":"INCREASE: Inductive Graph Representation Learning for Spatio-Temporal Kriging","date":"2023-02-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"Aminsheykh98/INCREASE-pytorch","path":"utils.py","file_url":"https://github.com/Aminsheykh98/INCREASE-pytorch/blob/HEAD/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Unlicense","inline_ok":true,"code_sha256_prefix":"34ed646e464cadd0","mcp_get_code":{"code_sha256":"34ed646e464cadd0"}},{"arxiv_id":"2302.02738","paper":"/paper/increase-inductive-graph-representation","title":"INCREASE: Inductive Graph Representation Learning for Spatio-Temporal Kriging","date":"2023-02-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhengchuanpan/increase","path":"Beijing/INCREASE/utils.py","file_url":"https://github.com/zhengchuanpan/increase/blob/HEAD/Beijing/INCREASE/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"48ccda09903c4168","mcp_get_code":{"code_sha256":"48ccda09903c4168"}},{"arxiv_id":"2302.02738","paper":"/paper/increase-inductive-graph-representation","title":"INCREASE: Inductive Graph Representation Learning for Spatio-Temporal Kriging","date":"2023-02-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhengchuanpan/increase","path":"METR-LA/INCREASE/utils.py","file_url":"https://github.com/zhengchuanpan/increase/blob/HEAD/METR-LA/INCREASE/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"6c6008cb92b19595","mcp_get_code":{"code_sha256":"6c6008cb92b19595"}},{"arxiv_id":"2106.00948","paper":"/paper/unsupervised-out-of-domain-detection-via-pre","title":"Unsupervised Out-of-Domain Detection via Pre-trained Transformers","date":"2021-06-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rivercold/BERT-unsupervised-OOD","path":"calculate_log.py","file_url":"https://github.com/rivercold/BERT-unsupervised-OOD/blob/HEAD/calculate_log.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"511013ffe412491c","mcp_get_code":{"code_sha256":"511013ffe412491c"}},{"arxiv_id":"2105.14399","paper":"/paper/improving-entropic-out-of-distribution","title":"Enhanced Isotropy Maximization Loss: Seamless and High-Performance Out-of-Distribution Detection Simply Replacing the SoftMax Loss","date":"2021-05-30","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"dlmacedo/entropic-out-of-distribution-detection","path":"calculate_log.py","file_url":"https://github.com/dlmacedo/entropic-out-of-distribution-detection/blob/HEAD/calculate_log.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b84ce30ecbbc77e4","mcp_get_code":{"code_sha256":"b84ce30ecbbc77e4"}},{"arxiv_id":"1912.12510","paper":"/paper/detecting-out-of-distribution-examples-with","title":"Detecting Out-of-Distribution Examples with In-distribution Examples and Gram Matrices","date":"2019-12-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"VectorInstitute/gram-ood-detection","path":"calculate_log.py","file_url":"https://github.com/VectorInstitute/gram-ood-detection/blob/HEAD/calculate_log.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"511013ffe412491c","mcp_get_code":{"code_sha256":"511013ffe412491c"}},{"arxiv_id":"1911.08415","paper":"/paper/gman-a-graph-multi-attention-network-for","title":"GMAN: A Graph Multi-Attention Network for Traffic Prediction","date":"2019-11-11","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhengchuanpan/GMAN","path":"METR/utils.py","file_url":"https://github.com/zhengchuanpan/GMAN/blob/HEAD/METR/utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"1ba47a23adf07065","mcp_get_code":{"code_sha256":"1ba47a23adf07065"}},{"arxiv_id":"1904.02399","paper":"/paper/riemannian-normalizing-flow-on-variational","title":"Riemannian Normalizing Flow on Variational Wasserstein Autoencoder for Text Modeling","date":"2019-04-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"kingofspace0wzz/wae-rnf-lm","path":"stochastic/flow.py","file_url":"https://github.com/kingofspace0wzz/wae-rnf-lm/blob/HEAD/stochastic/flow.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"ca442985ac3b7e16","mcp_get_code":{"code_sha256":"ca442985ac3b7e16"}},{"arxiv_id":"Tan_Neighbor_Relations_Matter_in_Video_Scene_Detection_CVPR_2024_paper","paper":null,"title":"arXiv:Tan_Neighbor_Relations_Matter_in_Video_Scene_Detection_CVPR_2024_paper","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"ExMorgan-Alter/NeighborNet","path":"metric.py","file_url":"https://github.com/ExMorgan-Alter/NeighborNet/blob/HEAD/metric.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"d4023529d9e8bd06","mcp_get_code":{"code_sha256":"d4023529d9e8bd06"}}]}