{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/what-you-can-cram-into-a-single-vector","title":"What you can cram into a single vector: Probing sentence embeddings for linguistic properties","arxiv_id":"1805.01070","date":"2018-05-03","proceeding":null,"authors":["Alexis Conneau","German Kruszewski","Guillaume Lample","Loïc Barrault","Marco Baroni"],"abstract":"Although much effort has recently been devoted to training high-quality\nsentence embeddings, we still have a poor understanding of what they are\ncapturing. \"Downstream\" tasks, often based on sentence classification, are\ncommonly used to evaluate the quality of sentence representations. The\ncomplexity of the tasks makes it however difficult to infer what kind of\ninformation is present in the representations. We introduce here 10 probing\ntasks designed to capture simple linguistic features of sentences, and we use\nthem to study embeddings generated by three different encoders trained in eight\ndistinct ways, uncovering intriguing properties of both encoders and training\nmethods.","url_abs":"http://arxiv.org/abs/1805.01070v2","url_pdf":"http://arxiv.org/pdf/1805.01070v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"what-you-can-cram-into-a-single-vector","repo_url":"https://github.com/facebookresearch/SentEval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"what-you-can-cram-into-a-single-vector","repo_url":"https://github.com/UKPLab/linspector-web","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"what-you-can-cram-into-a-single-vector","repo_url":"https://github.com/facebookresearch/InferSent","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"what-you-can-cram-into-a-single-vector","repo_url":"https://github.com/greenparachute/probing-with-noise","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-2-Clause"}},{"paper_slug":"what-you-can-cram-into-a-single-vector","repo_url":"https://github.com/maexe/linspector-web","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"what-you-can-cram-into-a-single-vector","repo_url":"https://github.com/sid7954/NLP-Toolkit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"sentence-classification","task_name":"Sentence Classification"},{"task_slug":"sentence-embeddings","task_name":"Sentence Embeddings"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1805.01070","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1805.01070"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/InferSent","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/greenparachute/probing-with-noise","reach":{"status":"ok","spdx":"BSD-2-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/SentEval","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sid7954/NLP-Toolkit","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/maexe/linspector-web","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/UKPLab/linspector-web","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_honours":1,"ran_draft_wrong":2},"by_repo_kind":{},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"ff910817bb9f0411","entry":"batcher","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"ff910817bb9f0411"}},{"code_sha256_prefix":"af7388308de7ccb4","entry":"create_dictionary","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"af7388308de7ccb4"}},{"code_sha256_prefix":"19d5c2686259a0af","entry":"get_wordvec","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"19d5c2686259a0af"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}