{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/cogen-learning-from-feedback-with-coupled","title":"CoGen: Learning from Feedback with Coupled Comprehension and Generation","arxiv_id":"2408.15992","date":"2024-08-28","proceeding":null,"authors":["Mustafa Omer Gul","Yoav Artzi"],"abstract":"Systems with both language comprehension and generation capabilities can benefit from the tight connection between the two. This work studies coupling comprehension and generation with focus on continually learning from interaction with users. We propose techniques to tightly integrate the two capabilities for both learning and inference. We situate our studies in two-player reference games, and deploy various models for thousands of interactions with human users, while learning from interaction feedback signals. We show dramatic improvements in performance over time, with comprehension-generation coupling leading to performance improvements up to 26% in absolute terms and up to 17% higher accuracies compared to a non-coupled system. Our analysis also shows coupling has substantial qualitative impact on the system's language, making it significantly more human-like.","url_abs":"https://arxiv.org/abs/2408.15992v1","url_pdf":"https://arxiv.org/pdf/2408.15992v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"cogen-learning-from-feedback-with-coupled","repo_url":"https://github.com/lil-lab/cogen","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2408.15992","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.15992"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/lil-lab/cogen","reach":null}],"summary":{"ran_fixture":1,"ran_honours":1,"unverified":4},"by_repo_kind":{"official":{"samples":6,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"3391626ba57002a1","entry":"filter_targets","repo":"lil-lab/cogen","repo_kind":"official","path":"models/joint_inference.py","file_url":"https://github.com/lil-lab/cogen/blob/HEAD/models/joint_inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3391626ba57002a1"}},{"code_sha256_prefix":"5f621459c8e0f985","entry":"process_images","repo":"lil-lab/cogen","repo_kind":"official","path":"models/joint_inference.py","file_url":"https://github.com/lil-lab/cogen/blob/HEAD/models/joint_inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5f621459c8e0f985"}},{"code_sha256_prefix":"b6243816583a3cac","entry":"IdeficsJointInferenceModel","repo":"lil-lab/cogen","repo_kind":"official","path":"models/joint_inference.py","file_url":"https://github.com/lil-lab/cogen/blob/HEAD/models/joint_inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b6243816583a3cac"}},{"code_sha256_prefix":"85f44b9c0224bae7","entry":"construct_listener_full_prompt","repo":"lil-lab/cogen","repo_kind":"official","path":"models/joint_inference.py","file_url":"https://github.com/lil-lab/cogen/blob/HEAD/models/joint_inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"85f44b9c0224bae7"}},{"code_sha256_prefix":"fc616f122d15486d","entry":"get_listener_generation_prompts","repo":"lil-lab/cogen","repo_kind":"official","path":"models/joint_inference.py","file_url":"https://github.com/lil-lab/cogen/blob/HEAD/models/joint_inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc616f122d15486d"}},{"code_sha256_prefix":"d1b01eb9e322a6d5","entry":"process_idefics_listener_generation_input","repo":"lil-lab/cogen","repo_kind":"official","path":"models/joint_inference.py","file_url":"https://github.com/lil-lab/cogen/blob/HEAD/models/joint_inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d1b01eb9e322a6d5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}