{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/perturbed-masking-parameter-free-probing-for","title":"Perturbed Masking: Parameter-free Probing for Analyzing and Interpreting BERT","arxiv_id":"2004.14786","date":"2020-04-30","proceeding":"ACL 2020 6","authors":["Zhiyong Wu","Yun Chen","Ben Kao","Qun Liu"],"abstract":"By introducing a small set of additional parameters, a probe learns to solve specific linguistic tasks (e.g., dependency parsing) in a supervised manner using feature representations (e.g., contextualized embeddings). The effectiveness of such probing tasks is taken as evidence that the pre-trained model encodes linguistic knowledge. However, this approach of evaluating a language model is undermined by the uncertainty of the amount of knowledge that is learned by the probe itself. Complementary to those works, we propose a parameter-free probing technique for analyzing pre-trained language models (e.g., BERT). Our method does not require direct supervision from the probing tasks, nor do we introduce additional parameters to the probing process. Our experiments on BERT show that syntactic trees recovered from BERT using our method are significantly better than linguistically-uninformed baselines. We further feed the empirically induced dependency structures into a downstream sentiment classification task and find its improvement compatible with or even superior to a human-designed dependency schema.","url_abs":"https://arxiv.org/abs/2004.14786v3","url_pdf":"https://arxiv.org/pdf/2004.14786v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"perturbed-masking-parameter-free-probing-for","repo_url":"https://github.com/LividWo/Perturbed-Masking","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"dependency-parsing","task_name":"Dependency Parsing"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"sentiment-analysis","task_name":"Sentiment Analysis"},{"task_slug":"sentiment-classification","task_name":"Sentiment Classification"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2004.14786","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.14786"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/LividWo/Perturbed-Masking","reach":null}],"summary":{"ran_draft_wrong":1,"ran_fixture":1,"ran":1,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"645fe561e567310e","entry":"_run_strip_accents","repo":"LividWo/Perturbed-Masking","repo_kind":"official","path":"dependency/get_matrix_for_dep_probe.py","file_url":"https://github.com/LividWo/Perturbed-Masking/blob/HEAD/dependency/get_matrix_for_dep_probe.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"645fe561e567310e"}},{"code_sha256_prefix":"1d4925223ff85ecb","entry":"get_all_subword_id","repo":"LividWo/Perturbed-Masking","repo_kind":"official","path":"dependency/get_matrix_for_dep_probe.py","file_url":"https://github.com/LividWo/Perturbed-Masking/blob/HEAD/dependency/get_matrix_for_dep_probe.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1d4925223ff85ecb"}},{"code_sha256_prefix":"08a30d08267abd7e","entry":"match_tokenized_to_untokenized","repo":"LividWo/Perturbed-Masking","repo_kind":"official","path":"dependency/get_matrix_for_dep_probe.py","file_url":"https://github.com/LividWo/Perturbed-Masking/blob/HEAD/dependency/get_matrix_for_dep_probe.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"08a30d08267abd7e"}},{"code_sha256_prefix":"dc18fb9f8368d38b","entry":"get_dep_matrix","repo":"LividWo/Perturbed-Masking","repo_kind":"official","path":"dependency/get_matrix_for_dep_probe.py","file_url":"https://github.com/LividWo/Perturbed-Masking/blob/HEAD/dependency/get_matrix_for_dep_probe.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dc18fb9f8368d38b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}