{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/harnessing-the-power-of-multi-task","title":"Harnessing the Power of Multi-Task Pretraining for Ground-Truth Level Natural Language Explanations","arxiv_id":"2212.04231","date":"2022-12-08","proceeding":null,"authors":["Björn Plüster","Jakob Ambsdorf","Lukas Braach","Jae Hee Lee","Stefan Wermter"],"abstract":"Natural language explanations promise to offer intuitively understandable explanations of a neural network's decision process in complex vision-language tasks, as pursued in recent VL-NLE models. While current models offer impressive performance on task accuracy and explanation plausibility, they suffer from a range of issues: Some models feature a modular design where the explanation generation module is poorly integrated with a separate module for task-answer prediction, employ backbone models trained on limited sets of tasks, or incorporate ad hoc solutions to increase performance on single datasets. We propose to evade these limitations by applying recent advances in large-scale multi-task pretraining of generative Transformer models to the problem of VL-NLE tasks. Our approach outperforms recent models by a large margin, with human annotators preferring the generated explanations over the ground truth in two out of three evaluated datasets. As a novel challenge in VL-NLE research, we propose the problem of multi-task VL-NLE and show that jointly training on multiple tasks can increase the explanation quality. We discuss the ethical implications of high-quality NLE generation and other issues in recent VL-NLE research.","url_abs":"https://arxiv.org/abs/2212.04231v2","url_pdf":"https://arxiv.org/pdf/2212.04231v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"harnessing-the-power-of-multi-task","repo_url":"https://github.com/ofa-x/ofa-x","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"explanation-generation","task_name":"Explanation Generation"},{"task_slug":"visual-entailment","task_name":"Visual Entailment"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"hoc","method_name":"HOC"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/explanation-generation-on-vcr","task":"Explanation Generation","dataset":"VCR","model":"OFA-X-MT","rank_in_archive_order":1,"of":2,"metrics":{"Human Explanation Rating":"77.3"},"uses_additional_data":false},{"leaderboard":"/sota/explanation-generation-on-vcr","task":"Explanation Generation","dataset":"VCR","model":"OFA-X","rank_in_archive_order":2,"of":2,"metrics":{"Human Explanation Rating":"68.9"},"uses_additional_data":false},{"leaderboard":"/sota/explanation-generation-on-vqa-x","task":"Explanation Generation","dataset":"VQA-X","model":"OFA-X","rank_in_archive_order":1,"of":2,"metrics":{"Human Explanation Rating":"89.5"},"uses_additional_data":false},{"leaderboard":"/sota/explanation-generation-on-vqa-x","task":"Explanation Generation","dataset":"VQA-X","model":"OFA-X-MT","rank_in_archive_order":2,"of":2,"metrics":{"Human Explanation Rating":"87.8"},"uses_additional_data":false},{"leaderboard":"/sota/explanation-generation-on-e-snli-ve","task":"Explanation Generation","dataset":"e-SNLI-VE","model":"OFA-X","rank_in_archive_order":1,"of":2,"metrics":{"Human Explanation Rating":"85.7"},"uses_additional_data":false},{"leaderboard":"/sota/explanation-generation-on-e-snli-ve","task":"Explanation Generation","dataset":"e-SNLI-VE","model":"OFA-X-MT","rank_in_archive_order":2,"of":2,"metrics":{"Human Explanation Rating":"80.4"},"uses_additional_data":false},{"leaderboard":"/sota/visual-entailment-on-e-snli-ve","task":"Visual Entailment","dataset":"e-SNLI-VE","model":"OFA-X","rank_in_archive_order":1,"of":2,"metrics":{"Accuracy":"80.9"},"uses_additional_data":false},{"leaderboard":"/sota/visual-entailment-on-e-snli-ve","task":"Visual Entailment","dataset":"e-SNLI-VE","model":"OFA-X-MT","rank_in_archive_order":2,"of":2,"metrics":{"Accuracy":"78.9"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vcr-q-a-test","task":"Visual Question Answering (VQA)","dataset":"VCR (Q-A) test","model":"OFA-X","rank_in_archive_order":10,"of":11,"metrics":{"Accuracy":"71.2"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vcr-q-a-test","task":"Visual Question Answering (VQA)","dataset":"VCR (Q-A) test","model":"OFA-X-MT","rank_in_archive_order":11,"of":11,"metrics":{"Accuracy":"62"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-x","task":"Visual Question Answering (VQA)","dataset":"VQA-X","model":"OFA-X-MT","rank_in_archive_order":1,"of":2,"metrics":{"Accuracy":"92.6"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-x","task":"Visual Question Answering (VQA)","dataset":"VQA-X","model":"OFA-X","rank_in_archive_order":2,"of":2,"metrics":{"Accuracy":"91.2"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2212.04231","atlas_url":"https://app.syntology.ai/?focus=2212.04231","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.04231"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ofa-x/ofa-x","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_honours":1,"ran":1,"unverified":6},"by_repo_kind":{"official":{"samples":8,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"357d3a4f0be679d9","entry":"apply_half","repo":"ofa-x/ofa-x","repo_kind":"official","path":"OFA/evaluate.py","file_url":"https://github.com/ofa-x/ofa-x/blob/HEAD/OFA/evaluate.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"357d3a4f0be679d9"}},{"code_sha256_prefix":"3a3448cb08438214","entry":"label_smoothed_nll_loss","repo":"ofa-x/ofa-x","repo_kind":"official","path":"OFA/criterions/label_smoothed_cross_entropy_expl.py","file_url":"https://github.com/ofa-x/ofa-x/blob/HEAD/OFA/criterions/label_smoothed_cross_entropy_expl.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3a3448cb08438214"}},{"code_sha256_prefix":"e1397e09cb4c249d","entry":"construct_rdrop_sample","repo":"ofa-x/ofa-x","repo_kind":"official","path":"OFA/criterions/label_smoothed_encouraging_loss.py","file_url":"https://github.com/ofa-x/ofa-x/blob/HEAD/OFA/criterions/label_smoothed_encouraging_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e1397e09cb4c249d"}},{"code_sha256_prefix":"01478032e19f9e68","entry":"custom_to_pil","repo":"ofa-x/ofa-x","repo_kind":"official","path":"OFA/criterions/clip_scst_loss.py","file_url":"https://github.com/ofa-x/ofa-x/blob/HEAD/OFA/criterions/clip_scst_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"01478032e19f9e68"}},{"code_sha256_prefix":"e7939f8552fd8729","entry":"kl_loss","repo":"ofa-x/ofa-x","repo_kind":"official","path":"OFA/criterions/label_smoothed_encouraging_loss.py","file_url":"https://github.com/ofa-x/ofa-x/blob/HEAD/OFA/criterions/label_smoothed_encouraging_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e7939f8552fd8729"}},{"code_sha256_prefix":"cbd8615e3eba731b","entry":"label_smoothed_nll_loss","repo":"ofa-x/ofa-x","repo_kind":"official","path":"OFA/criterions/label_smoothed_encouraging_loss.py","file_url":"https://github.com/ofa-x/ofa-x/blob/HEAD/OFA/criterions/label_smoothed_encouraging_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cbd8615e3eba731b"}},{"code_sha256_prefix":"cdbc222d0057fb31","entry":"label_smoothed_nll_loss","repo":"ofa-x/ofa-x","repo_kind":"official","path":"OFA/criterions/label_smoothed_cross_entropy.py","file_url":"https://github.com/ofa-x/ofa-x/blob/HEAD/OFA/criterions/label_smoothed_cross_entropy.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cdbc222d0057fb31"}},{"code_sha256_prefix":"b0d5d75f27b8ce4f","entry":"scst_loss","repo":"ofa-x/ofa-x","repo_kind":"official","path":"OFA/criterions/clip_scst_loss.py","file_url":"https://github.com/ofa-x/ofa-x/blob/HEAD/OFA/criterions/clip_scst_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b0d5d75f27b8ce4f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}