{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/int-an-inequality-benchmark-for-evaluating","title":"INT: An Inequality Benchmark for Evaluating Generalization in Theorem Proving","arxiv_id":"2007.02924","date":"2020-07-06","proceeding":"ICLR 2021 1","authors":["Yuhuai Wu","Albert Qiaochu Jiang","Jimmy Ba","Roger Grosse"],"abstract":"In learning-assisted theorem proving, one of the most critical challenges is to generalize to theorems unlike those seen at training time. In this paper, we introduce INT, an INequality Theorem proving benchmark, specifically designed to test agents' generalization ability. INT is based on a procedure for generating theorems and proofs; this procedure's knobs allow us to measure 6 different types of generalization, each reflecting a distinct challenge characteristic to automated theorem proving. In addition, unlike prior benchmarks for learning-assisted theorem proving, INT provides a lightweight and user-friendly theorem proving environment with fast simulations, conducive to performing learning-based and search-based research. We introduce learning-based baselines and evaluate them across 6 dimensions of generalization with the benchmark. We then evaluate the same agents augmented with Monte Carlo Tree Search (MCTS) at test time, and show that MCTS can help to prove new theorems.","url_abs":"https://arxiv.org/abs/2007.02924v2","url_pdf":"https://arxiv.org/pdf/2007.02924v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"int-an-inequality-benchmark-for-evaluating","repo_url":"https://github.com/albertqjiang/INT","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"automated-theorem-proving","task_name":"Automated Theorem Proving"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2007.02924","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.02924"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/albertqjiang/INT","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":7},"by_repo_kind":{"official":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c59747934355efa5","entry":"execute_according_to_dictionary","repo":"albertqjiang/INT","repo_kind":"official","path":"int_environment/algos/eval_seq.py","file_url":"https://github.com/albertqjiang/INT/blob/HEAD/int_environment/algos/eval_seq.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c59747934355efa5"}},{"code_sha256_prefix":"3ff091098a71d9dd","entry":"flatten","repo":"albertqjiang/INT","repo_kind":"official","path":"int_environment/algos/lib/ops.py","file_url":"https://github.com/albertqjiang/INT/blob/HEAD/int_environment/algos/lib/ops.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ff091098a71d9dd"}},{"code_sha256_prefix":"95e11c5901e291dd","entry":"load_checkpoint","repo":"albertqjiang/INT","repo_kind":"official","path":"int_environment/algos/rl.py","file_url":"https://github.com/albertqjiang/INT/blob/HEAD/int_environment/algos/rl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"95e11c5901e291dd"}},{"code_sha256_prefix":"ea3d9cdbcf7a9b4e","entry":"load_model","repo":"albertqjiang/INT","repo_kind":"official","path":"int_environment/visualization/model_debug.py","file_url":"https://github.com/albertqjiang/INT/blob/HEAD/int_environment/visualization/model_debug.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea3d9cdbcf7a9b4e"}},{"code_sha256_prefix":"d760315462516b6b","entry":"one_hot","repo":"albertqjiang/INT","repo_kind":"official","path":"int_environment/algos/thm_env.py","file_url":"https://github.com/albertqjiang/INT/blob/HEAD/int_environment/algos/thm_env.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d760315462516b6b"}},{"code_sha256_prefix":"a36605330ae69c8d","entry":"one_hot","repo":"albertqjiang/INT","repo_kind":"official","path":"int_environment/algos/lib/ops.py","file_url":"https://github.com/albertqjiang/INT/blob/HEAD/int_environment/algos/lib/ops.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a36605330ae69c8d"}},{"code_sha256_prefix":"720f82c493f038b5","entry":"resume_checkpoint","repo":"albertqjiang/INT","repo_kind":"official","path":"int_environment/algos/rl.py","file_url":"https://github.com/albertqjiang/INT/blob/HEAD/int_environment/algos/rl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"720f82c493f038b5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}