{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/cate-computation-aware-neural-architecture","title":"CATE: Computation-aware Neural Architecture Encoding with Transformers","arxiv_id":"2102.07108","date":"2021-02-14","proceeding":null,"authors":["Shen Yan","Kaiqiang Song","Fei Liu","Mi Zhang"],"abstract":"Recent works (White et al., 2020a; Yan et al., 2020) demonstrate the importance of architecture encodings in Neural Architecture Search (NAS). These encodings encode either structure or computation information of the neural architectures. Compared to structure-aware encodings, computation-aware encodings map architectures with similar accuracies to the same region, which improves the downstream architecture search performance (Zhang et al., 2019; White et al., 2020a). In this work, we introduce a Computation-Aware Transformer-based Encoding method called CATE. Different from existing computation-aware encodings based on fixed transformation (e.g. path encoding), CATE employs a pairwise pre-training scheme to learn computation-aware encodings using Transformers with cross-attention. Such learned encodings contain dense and contextualized computation information of neural architectures. We compare CATE with eleven encodings under three major encoding-dependent NAS subroutines in both small and large search spaces. Our experiments show that CATE is beneficial to the downstream search, especially in the large search space. Moreover, the outside search space experiment demonstrates its superior generalization ability beyond the search space on which it was trained. Our code is available at: https://github.com/MSU-MLSys-Lab/CATE.","url_abs":"https://arxiv.org/abs/2102.07108v2","url_pdf":"https://arxiv.org/pdf/2102.07108v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"cate-computation-aware-neural-architecture","repo_url":"https://github.com/MSU-MLSys-Lab/CATE","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"automl","task_name":"AutoML"},{"task_slug":"architecture-search","task_name":"Neural Architecture Search"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"unsupervised-pre-training","task_name":"Unsupervised Pre-training"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/neural-architecture-search-on-cifar-10","task":"Neural Architecture Search","dataset":"CIFAR-10","model":"CATE","rank_in_archive_order":15,"of":41,"metrics":{"Parameters":"4.1","Search Time (GPU days)":"10.3","Top-1 Error Rate":"2.46%"},"uses_additional_data":false},{"leaderboard":"/sota/architecture-search-on-cifar-10-image","task":"Neural Architecture Search","dataset":"CIFAR-10 Image Classification","model":"CATE","rank_in_archive_order":11,"of":19,"metrics":{"Params":"4.1","Percentage error":"2.46","Search Time (GPU days)":"10.3"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2102.07108","atlas_url":"https://app.syntology.ai/?focus=2102.07108","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.07108"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/MSU-MLSys-Lab/CATE","reach":null}],"summary":{"ran":7,"ran_draft_wrong":2,"unverified":1},"by_repo_kind":{"official":{"samples":10,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ad464810b30d6d25","entry":"Encoder","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ad464810b30d6d25"}},{"code_sha256_prefix":"2b2a73a9b0d3825e","entry":"EncoderBlock","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2b2a73a9b0d3825e"}},{"code_sha256_prefix":"9e5a8995a0d0b684","entry":"FeedForwardBlock","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9e5a8995a0d0b684"}},{"code_sha256_prefix":"ef066f2fb629b698","entry":"MultiHeadAttention","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ef066f2fb629b698"}},{"code_sha256_prefix":"21d20b888bdba6a4","entry":"PositionalEmbedding","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"21d20b888bdba6a4"}},{"code_sha256_prefix":"73d36f4b75757093","entry":"PositionwiseFeedForward","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"73d36f4b75757093"}},{"code_sha256_prefix":"52117ec9c56aa17a","entry":"SelfAttentionBlock","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"52117ec9c56aa17a"}},{"code_sha256_prefix":"bbc7c22f9d8732dc","entry":"attention","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bbc7c22f9d8732dc"}},{"code_sha256_prefix":"d013dfb22c365f67","entry":"clones","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d013dfb22c365f67"}},{"code_sha256_prefix":"014afd02662c3b66","entry":"Transformer","repo":"MSU-MLSys-Lab/CATE","repo_kind":"official","path":"layers/transformer.py","file_url":"https://github.com/MSU-MLSys-Lab/CATE/blob/HEAD/layers/transformer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"014afd02662c3b66"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}