{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/self-supervised-visual-preference-alignment","title":"Self-Supervised Visual Preference Alignment","arxiv_id":"2404.10501","date":"2024-04-16","proceeding":null,"authors":["Ke Zhu","Zheng Ge","Liang Zhao","Xiangyu Zhang"],"abstract":"This paper makes the first attempt towards unsupervised preference alignment in Vision-Language Models (VLMs). We generate chosen and rejected responses with regard to the original and augmented image pairs, and conduct preference alignment with direct preference optimization. It is based on a core idea: properly designed augmentation to the image input will induce VLM to generate false but hard negative responses, which helps the model to learn from and produce more robust and powerful answers. The whole pipeline no longer hinges on supervision from GPT-4 or human involvement during alignment, and is highly efficient with few lines of code. With only 8k randomly sampled unsupervised data, it achieves 90\\% relative score to GPT-4 on complex reasoning in LLaVA-Bench, and improves LLaVA-7B/13B by 6.7\\%/5.6\\% score on complex multi-modal benchmark MM-Vet. Visualizations shows its improved ability to align with user-intentions. A series of ablations are firmly conducted to reveal the latent mechanism of the approach, which also indicates its potential towards further scaling. Code are available in https://github.com/Kevinz-code/SeVa.","url_abs":"https://arxiv.org/abs/2404.10501v2","url_pdf":"https://arxiv.org/pdf/2404.10501v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"self-supervised-visual-preference-alignment","repo_url":"https://github.com/Kevinz-code/SeVa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"GPL-3.0"}}],"tasks":[{"task_slug":null,"task_name":"8k"},{"task_slug":null,"task_name":"MM-Vet"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"SeVa-13B","rank_in_archive_order":105,"of":231,"metrics":{"GPT-4 score":"41.0"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"SeVa-7B","rank_in_archive_order":136,"of":231,"metrics":{"GPT-4 score":"37.2"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2404.10501","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.10501"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Kevinz-code/SeVa","reach":{"status":"ok","spdx":"GPL-3.0"}}],"summary":{"ran":3,"ran_draft_wrong":4,"ran_violates":1,"ran_honours":1,"unverified":2},"by_repo_kind":{"official":{"samples":11,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"03a90df0ec7b6c4f","entry":"convert_dict_to_tensor","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/pope_eval.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/pope_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"03a90df0ec7b6c4f"}},{"code_sha256_prefix":"2edb1a40e640ff20","entry":"eval_all","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/pope_calculate.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/pope_calculate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"2edb1a40e640ff20"}},{"code_sha256_prefix":"592b3c1a88f93d7c","entry":"expand2square","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/llava/mm_utils.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"592b3c1a88f93d7c"}},{"code_sha256_prefix":"c774723322f1f9c6","entry":"find_all_linear_names","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/train_dpo_ours.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/train_dpo_ours.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"c774723322f1f9c6"}},{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/llava/eval/model_vqa.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/llava/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"bae18947b56f2be1","entry":"is_none","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/llava/eval/model_vqa_mmbench.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/llava/eval/model_vqa_mmbench.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"bae18947b56f2be1"}},{"code_sha256_prefix":"9b3c1cb391672ccb","entry":"load_image","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/llava/eval/model_vqa_qbench.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/llava/eval/model_vqa_qbench.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"9b3c1cb391672ccb"}},{"code_sha256_prefix":"c3ee9d07c900dd55","entry":"load_image_from_base64","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/llava/mm_utils.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"c3ee9d07c900dd55"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/llava/eval/model_vqa.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/llava/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"616ffbdc154ed2d8","entry":"maybe_zero_3","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/train_dpo_ours.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/train_dpo_ours.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"616ffbdc154ed2d8"}},{"code_sha256_prefix":"344dff4791fd1381","entry":"process_images","repo":"Kevinz-code/SeVa","repo_kind":"official","path":"seva/llava/mm_utils.py","file_url":"https://github.com/Kevinz-code/SeVa/blob/HEAD/seva/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"344dff4791fd1381"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}