{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/bhvit-binarized-hybrid-vision-transformer","title":"BHViT: Binarized Hybrid Vision Transformer","arxiv_id":"2503.02394","date":"2025-03-04","proceeding":"CVPR 2025 1","authors":["Tian Gao","Zhiyuan Zhang","Yu Zhang","Huajun Liu","Kaijie Yin","Chengzhong Xu","Hui Kong"],"abstract":"Model binarization has made significant progress in enabling real-time and energy-efficient computation for convolutional neural networks (CNN), offering a potential solution to the deployment challenges faced by Vision Transformers (ViTs) on edge devices. However, due to the structural differences between CNN and Transformer architectures, simply applying binary CNN strategies to the ViT models will lead to a significant performance drop. To tackle this challenge, we propose BHViT, a binarization-friendly hybrid ViT architecture and its full binarization model with the guidance of three important observations. Initially, BHViT utilizes the local information interaction and hierarchical feature aggregation technique from coarse to fine levels to address redundant computations stemming from excessive tokens. Then, a novel module based on shift operations is proposed to enhance the performance of the binary Multilayer Perceptron (MLP) module without significantly increasing computational overhead. In addition, an innovative attention matrix binarization method based on quantization decomposition is proposed to evaluate the token's importance in the binarized attention matrix. Finally, we propose a regularization loss to address the inadequate optimization caused by the incompatibility between the weight oscillation in the binary layers and the Adam Optimizer. Extensive experimental results demonstrate that our proposed algorithm achieves SOTA performance among binary ViT methods.","url_abs":"https://arxiv.org/abs/2503.02394v2","url_pdf":"https://arxiv.org/pdf/2503.02394v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"bhvit-binarized-hybrid-vision-transformer","repo_url":"https://github.com/IMRL/BHViT","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"binarization","task_name":"Binarization"},{"task_slug":"quantization","task_name":"Quantization"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2503.02394","atlas_url":"https://app.syntology.ai/?focus=2503.02394","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.02394"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/IMRL/BHViT","reach":null}],"summary":{"ran":15,"ran_fixture":1,"unverified":13},"by_repo_kind":{"official":{"samples":29,"ran":16,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7e1832d29c4089e3","entry":"BHViTEmbeddings","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7e1832d29c4089e3"}},{"code_sha256_prefix":"a1f47c9b7b8a2b6a","entry":"BHViTPatchEmbeddings","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a1f47c9b7b8a2b6a"}},{"code_sha256_prefix":"e1be08d992cdbec6","entry":"BinaryActivation_Attention","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e1be08d992cdbec6"}},{"code_sha256_prefix":"7864a7f2d635f1cc","entry":"BinaryQuantizer","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7864a7f2d635f1cc"}},{"code_sha256_prefix":"bc00cba6dd4fd931","entry":"GSB_Attention","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bc00cba6dd4fd931"}},{"code_sha256_prefix":"5a33cb0660a4f8f5","entry":"LayerScale","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5a33cb0660a4f8f5"}},{"code_sha256_prefix":"80ac18a2d3335ec2","entry":"LearnableBiasnn","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"80ac18a2d3335ec2"}},{"code_sha256_prefix":"f74b4e57f68230ce","entry":"PatchEmbed","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f74b4e57f68230ce"}},{"code_sha256_prefix":"5053a360d7323930","entry":"RPReLU","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5053a360d7323930"}},{"code_sha256_prefix":"8a8583dbbaae1dd0","entry":"Shift","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8a8583dbbaae1dd0"}},{"code_sha256_prefix":"2383072bc4ad455a","entry":"Shift2","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2383072bc4ad455a"}},{"code_sha256_prefix":"818b750a4ca6f3f2","entry":"Shift_channel_mix","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"818b750a4ca6f3f2"}},{"code_sha256_prefix":"a22c340e89f0aa29","entry":"SymQuantizer","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a22c340e89f0aa29"}},{"code_sha256_prefix":"7da1d198566520b7","entry":"Token_for_Attention","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7da1d198566520b7"}},{"code_sha256_prefix":"f04ba986abc61036","entry":"TwnQuantizer","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f04ba986abc61036"}},{"code_sha256_prefix":"00e8fb5b0d52f96f","entry":"windows_split","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00e8fb5b0d52f96f"}},{"code_sha256_prefix":"80e2ed60aa621a51","entry":"BHViTAttention","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"80e2ed60aa621a51"}},{"code_sha256_prefix":"6937c2e66bd08ef0","entry":"BHViTEncoder","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6937c2e66bd08ef0"}},{"code_sha256_prefix":"39ca817be9205f9e","entry":"BHViTLayer","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"39ca817be9205f9e"}},{"code_sha256_prefix":"061e33af6401f92a","entry":"BHViTModel","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"061e33af6401f92a"}},{"code_sha256_prefix":"8856810408ef8784","entry":"BHViTSelfAttention","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8856810408ef8784"}},{"code_sha256_prefix":"58380fdab4ebd58e","entry":"BHViTSelfOutput","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"58380fdab4ebd58e"}},{"code_sha256_prefix":"50f22547e2f27990","entry":"BinaryPatchEmbed","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"50f22547e2f27990"}},{"code_sha256_prefix":"505e71bceab8cb1a","entry":"GCLayer","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"505e71bceab8cb1a"}},{"code_sha256_prefix":"0af4680021d038d8","entry":"QuantizeConv2d","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0af4680021d038d8"}},{"code_sha256_prefix":"0cefbcad32882f70","entry":"QuantizeLinear","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0cefbcad32882f70"}},{"code_sha256_prefix":"e071c97816d8e232","entry":"ViTIntermediate","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e071c97816d8e232"}},{"code_sha256_prefix":"b0f024c430a3ab21","entry":"ViTOutput","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b0f024c430a3ab21"}},{"code_sha256_prefix":"f4252cbcf11d1708","entry":"token_mixer","repo":"IMRL/BHViT","repo_kind":"official","path":"transformer/BHViT.py","file_url":"https://github.com/IMRL/BHViT/blob/HEAD/transformer/BHViT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f4252cbcf11d1708"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}