{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/does-bert-learn-as-humans-perceive","title":"Does BERT Learn as Humans Perceive? Understanding Linguistic Styles through Lexica","arxiv_id":"2109.02738","date":"2021-09-06","proceeding":"EMNLP 2021 11","authors":["Shirley Anugrah Hayati","Dongyeop Kang","Lyle Ungar"],"abstract":"People convey their intention and attitude through linguistic styles of the text that they write. In this study, we investigate lexicon usages across styles throughout two lenses: human perception and machine word importance, since words differ in the strength of the stylistic cues that they provide. To collect labels of human perception, we curate a new dataset, Hummingbird, on top of benchmarking style datasets. We have crowd workers highlight the representative words in the text that makes them think the text has the following styles: politeness, sentiment, offensiveness, and five emotion types. We then compare these human word labels with word importance derived from a popular fine-tuned style classifier like BERT. Our results show that the BERT often finds content words not relevant to the target style as important words used in style prediction, but humans do not perceive the same way even though for some styles (e.g., positive sentiment and joy) human- and machine-identified words share significant overlap for some styles.","url_abs":"https://arxiv.org/abs/2109.02738v2","url_pdf":"https://arxiv.org/pdf/2109.02738v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"does-bert-learn-as-humans-perceive","repo_url":"https://github.com/sweetpeach/hummingbird","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[{"slug":"hummingbird","name":"Hummingbird","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2109.02738","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.02738"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/sweetpeach/hummingbird","reach":null}],"summary":{"ran_draft_wrong":2,"unverified":2},"by_repo_kind":{"official":{"samples":4,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"fc3eba1ef1f4862c","entry":"match_bert_token_to_original","repo":"sweetpeach/hummingbird","repo_kind":"official","path":"code/extract_tokens_from_bert_data.py","file_url":"https://github.com/sweetpeach/hummingbird/blob/HEAD/code/extract_tokens_from_bert_data.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc3eba1ef1f4862c"}},{"code_sha256_prefix":"73fde0fb9d15db3f","entry":"remove_cls_and_bert","repo":"sweetpeach/hummingbird","repo_kind":"official","path":"code/extract_tokens_from_bert_data.py","file_url":"https://github.com/sweetpeach/hummingbird/blob/HEAD/code/extract_tokens_from_bert_data.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"73fde0fb9d15db3f"}},{"code_sha256_prefix":"cf25b0029a4d3c54","entry":"load_dataset","repo":"sweetpeach/hummingbird","repo_kind":"official","path":"code/model/training.py","file_url":"https://github.com/sweetpeach/hummingbird/blob/HEAD/code/model/training.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cf25b0029a4d3c54"}},{"code_sha256_prefix":"22b07bf6d6332fdf","entry":"make_new_attr_score","repo":"sweetpeach/hummingbird","repo_kind":"official","path":"code/extract_tokens_from_bert_data.py","file_url":"https://github.com/sweetpeach/hummingbird/blob/HEAD/code/extract_tokens_from_bert_data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"22b07bf6d6332fdf"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}