{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/fau-facial-expressions-valence-and-arousal-a","title":"Multitask Emotion Recognition with Incomplete Labels","arxiv_id":"2002.03557","date":"2020-02-10","proceeding":null,"authors":["Didan Deng","Zhaokang Chen","Bertram E. Shi"],"abstract":"We train a unified model to perform three tasks: facial action unit detection, expression classification, and valence-arousal estimation. We address two main challenges of learning the three tasks. First, most existing datasets are highly imbalanced. Second, most existing datasets do not contain labels for all three tasks. To tackle the first challenge, we apply data balancing techniques to experimental datasets. To tackle the second challenge, we propose an algorithm for the multitask model to learn from missing (incomplete) labels. This algorithm has two steps. We first train a teacher model to perform all three tasks, where each instance is trained by the ground truth label of its corresponding task. Secondly, we refer to the outputs of the teacher model as the soft labels. We use the soft labels and the ground truth to train the student model. We find that most of the student models outperform their teacher model on all the three tasks. Finally, we use model ensembling to boost performance further on the three tasks.","url_abs":"https://arxiv.org/abs/2002.03557v2","url_pdf":"https://arxiv.org/pdf/2002.03557v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"fau-facial-expressions-valence-and-arousal-a","repo_url":"https://github.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"fau-facial-expressions-valence-and-arousal-a","repo_url":"https://github.com/HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"fau-facial-expressions-valence-and-arousal-a","repo_url":"https://github.com/wtomin/A-Multitask-Solution-for-FAU-EXPR-VA","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"action-unit-detection","task_name":"Action Unit Detection"},{"task_slug":"arousal-estimation","task_name":"Arousal Estimation"},{"task_slug":"emotion-recognition","task_name":"Emotion Recognition"},{"task_slug":"facial-action-unit-detection","task_name":"Facial Action Unit Detection"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2002.03557","atlas_url":"https://app.syntology.ai/?focus=2002.03557","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2002.03557"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wtomin/A-Multitask-Solution-for-FAU-EXPR-VA","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_violates":1,"ran":5,"ran_honours":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1},"listed":{"samples":6,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b8e95809ca2c17c9","entry":"sigmoid","repo":"HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels","repo_kind":"listed","path":"Multitask-CNN-RNN/val.py","file_url":"https://github.com/HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels/blob/HEAD/Multitask-CNN-RNN/val.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":2,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b8e95809ca2c17c9"}},{"code_sha256_prefix":"a6365a0e5e75becf","entry":"IRLbl","repo":"HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels","repo_kind":"listed","path":"Multitask-CNN-RNN/torchsampler/imbalanced_ML.py","file_url":"https://github.com/HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels/blob/HEAD/Multitask-CNN-RNN/torchsampler/imbalanced_ML.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a6365a0e5e75becf"}},{"code_sha256_prefix":"67ee8f792913dc4d","entry":"MeanIR","repo":"HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels","repo_kind":"listed","path":"Multitask-CNN-RNN/torchsampler/imbalanced_ML.py","file_url":"https://github.com/HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels/blob/HEAD/Multitask-CNN-RNN/torchsampler/imbalanced_ML.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"67ee8f792913dc4d"}},{"code_sha256_prefix":"9892ec4891095438","entry":"accuracy","repo":"HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels","repo_kind":"listed","path":"Multitask-CNN-RNN/utils/model_utils.py","file_url":"https://github.com/HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels/blob/HEAD/Multitask-CNN-RNN/utils/model_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9892ec4891095438"}},{"code_sha256_prefix":"377d492e7d719b31","entry":"averaged_accuracy","repo":"HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels","repo_kind":"listed","path":"Multitask-CNN-RNN/utils/model_utils.py","file_url":"https://github.com/HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels/blob/HEAD/Multitask-CNN-RNN/utils/model_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"377d492e7d719b31"}},{"code_sha256_prefix":"f068ad448aee186f","entry":"averaged_f1_score","repo":"HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels","repo_kind":"listed","path":"Multitask-CNN-RNN/utils/model_utils.py","file_url":"https://github.com/HKUST-NISL/Multitask-Emotion-Recognition-with-Incomplete-Labels/blob/HEAD/Multitask-CNN-RNN/utils/model_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f068ad448aee186f"}},{"code_sha256_prefix":"3a0dedf12c75778e","entry":"parse_txt","repo":"wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels","repo_kind":"official","path":"emotion_demo.py","file_url":"https://github.com/wtomin/Multitask-Emotion-Recognition-with-Incomplete-Labels/blob/HEAD/emotion_demo.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3a0dedf12c75778e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}