{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/set-transformer-a-framework-for-attention","title":"Set Transformer: A Framework for Attention-based Permutation-Invariant Neural Networks","arxiv_id":"1810.00825","date":"2018-10-01","proceeding":null,"authors":["Juho Lee","Yoonho Lee","Jungtaek Kim","Adam R. Kosiorek","Seungjin Choi","Yee Whye Teh"],"abstract":"Many machine learning tasks such as multiple instance learning, 3D shape recognition, and few-shot image classification are defined on sets of instances. Since solutions to such problems do not depend on the order of elements of the set, models used to address them should be permutation invariant. We present an attention-based neural network module, the Set Transformer, specifically designed to model interactions among elements in the input set. The model consists of an encoder and a decoder, both of which rely on attention mechanisms. In an effort to reduce computational complexity, we introduce an attention scheme inspired by inducing point methods from sparse Gaussian process literature. It reduces the computation time of self-attention from quadratic to linear in the number of elements in the set. We show that our model is theoretically attractive and we evaluate it on a range of tasks, demonstrating the state-of-the-art performance compared to recent methods for set-structured data.","url_abs":"https://arxiv.org/abs/1810.00825v3","url_pdf":"https://arxiv.org/pdf/1810.00825v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/juho-lee/set_transformer","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/DLii-Research/tf-set-transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/Enigmatisms/Set-Transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/Rishit-dagli/isab","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/Xianchao-Wu/perceiver-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/lucidrains/agent-attention-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/lucidrains/perceiver-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/pientist/ballradar","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"set-transformer-a-framework-for-attention","repo_url":"https://github.com/rasoulnorouzi/Health-Recipes-Recommender/tree/main/Models/Transformers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"3d-shape-recognition","task_name":"3D Shape Recognition"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"few-shot-image-classification","task_name":"Few-Shot Image Classification"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"multiple-instance-learning","task_name":"Multiple Instance Learning"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gaussian-process","method_name":"Gaussian Process"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"set-transformer","method_name":"Set Transformer"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[{"slug":"set-transformer","name":"Set Transformer","full_name":"Set Transformer"}],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1810.00825","atlas_url":"https://app.syntology.ai/?focus=1810.00825","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1810.00825"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Enigmatisms/Set-Transformer","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DLii-Research/tf-set-transformer","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Xianchao-Wu/perceiver-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rasoulnorouzi/Health-Recipes-Recommender/tree/main/Models/Transformers","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Rishit-dagli/isab","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/perceiver-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pientist/ballradar","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/juho-lee/set_transformer","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/agent-attention-pytorch","reach":null}],"summary":{"ran_draft_wrong":2,"ran_violates":3,"unverified":4},"by_repo_kind":{"official":{"samples":4,"ran":0,"repositories":1},"listed":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"26aaad5a3c3071dd","entry":"FeedForward","repo":"lucidrains/agent-attention-pytorch","repo_kind":"listed","path":"agent_attention_pytorch/agent_transformer.py","file_url":"https://github.com/lucidrains/agent-attention-pytorch/blob/HEAD/agent_attention_pytorch/agent_transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"26aaad5a3c3071dd"}},{"code_sha256_prefix":"8458f9ba0361ff43","entry":"cache_fn","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"8458f9ba0361ff43"}},{"code_sha256_prefix":"60fff7c3c400d7ff","entry":"default","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"60fff7c3c400d7ff"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"b7c3487f192e31b7","entry":"exists","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"b7c3487f192e31b7"}},{"code_sha256_prefix":"5d719edec199c51d","entry":"augment","repo":"juho-lee/set_transformer","repo_kind":"official","path":"data_modelnet40.py","file_url":"https://github.com/juho-lee/set_transformer/blob/HEAD/data_modelnet40.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5d719edec199c51d"}},{"code_sha256_prefix":"ce0c3175f3788408","entry":"rotate_z","repo":"juho-lee/set_transformer","repo_kind":"official","path":"data_modelnet40.py","file_url":"https://github.com/juho-lee/set_transformer/blob/HEAD/data_modelnet40.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ce0c3175f3788408"}},{"code_sha256_prefix":"a497385cd0d3e5cf","entry":"scatter","repo":"juho-lee/set_transformer","repo_kind":"official","path":"plots.py","file_url":"https://github.com/juho-lee/set_transformer/blob/HEAD/plots.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a497385cd0d3e5cf"}},{"code_sha256_prefix":"fd91947623969891","entry":"standardize","repo":"juho-lee/set_transformer","repo_kind":"official","path":"data_modelnet40.py","file_url":"https://github.com/juho-lee/set_transformer/blob/HEAD/data_modelnet40.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fd91947623969891"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}