{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/protoformer-embedding-prototypes-for-1","title":"Protoformer: Embedding Prototypes for Transformers","arxiv_id":"2206.12710","date":"2022-06-25","proceeding":"PAKDD 2022: Advances in Knowledge Discovery and Data Mining 2022 5","authors":["Ashkan Farhangi","Ning Sui","Nan Hua","Haiyan Bai","Arthur Huang","Zhishan Guo"],"abstract":"Transformers have been widely applied in text classification. Unfortunately, real-world data contain anomalies and noisy labels that cause challenges for state-of-art Transformers. This paper proposes Protoformer, a novel self-learning framework for Transformers that can leverage problematic samples for text classification. Protoformer features a selection mechanism for embedding samples that allows us to efficiently extract and utilize anomalies prototypes and difficult class prototypes. We demonstrated such capabilities on datasets with diverse textual structures (e.g., Twitter, IMDB, ArXiv). We also applied the framework to several models. The results indicate that Protoformer can improve current Transformers in various empirical settings.","url_abs":"https://arxiv.org/abs/2206.12710v1","url_pdf":"https://arxiv.org/pdf/2206.12710v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"protoformer-embedding-prototypes-for-1","repo_url":"https://github.com/ashfarhangi/Protoformer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"GPL-3.0"}},{"paper_slug":"protoformer-embedding-prototypes-for-1","repo_url":"https://github.com/EthanCoder24/Protofomer-NLP","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"protoformer-embedding-prototypes-for-1","repo_url":"https://github.com/GitF82/NLP-Embeddings-Protoformer-Paper","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"protoformer-embedding-prototypes-for-1","repo_url":"https://github.com/codelion/adaptive-classifier/blob/main/src/adaptive_classifier/memory.py","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"classification-1","task_name":"Classification"},{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"learning-with-noisy-labels","task_name":"Learning with noisy labels"},{"task_slug":"self-learning","task_name":"Self-Learning"},{"task_slug":"sentiment-classification","task_name":"Sentiment Classification"},{"task_slug":"text-classification","task_name":"Text Classification"}],"methods":[{"method_slug":"self-learning","method_name":"Self-Learning"}],"datasets_introduced":[{"slug":"arxiv-10","name":"arXiv-10","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/text-classification-on-arxiv-10","task":"Text Classification","dataset":"arXiv-10","model":"Protoformer","rank_in_archive_order":1,"of":4,"metrics":{"Accuracy":"0.794"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2206.12710","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}