{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/from-pixels-to-graphs-open-vocabulary-scene","title":"From Pixels to Graphs: Open-Vocabulary Scene Graph Generation with Vision-Language Models","arxiv_id":"2404.00906","date":"2024-04-01","proceeding":"CVPR 2024 1","authors":["Rongjie Li","Songyang Zhang","Dahua Lin","Kai Chen","Xuming He"],"abstract":"Scene graph generation (SGG) aims to parse a visual scene into an intermediate graph representation for downstream reasoning tasks. Despite recent advancements, existing methods struggle to generate scene graphs with novel visual relation concepts. To address this challenge, we introduce a new open-vocabulary SGG framework based on sequence generation. Our framework leverages vision-language pre-trained models (VLM) by incorporating an image-to-graph generation paradigm. Specifically, we generate scene graph sequences via image-to-text generation with VLM and then construct scene graphs from these sequences. By doing so, we harness the strong capabilities of VLM for open-vocabulary SGG and seamlessly integrate explicit relational modeling for enhancing the VL tasks. Experimental results demonstrate that our design not only achieves superior performance with an open vocabulary but also enhances downstream vision-language task performance through explicit relation modeling knowledge.","url_abs":"https://arxiv.org/abs/2404.00906v3","url_pdf":"https://arxiv.org/pdf/2404.00906v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"from-pixels-to-graphs-open-vocabulary-scene","repo_url":"https://github.com/shtuplus/pix2grp_cvpr2024","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"tasks":[{"task_slug":"graph-generation","task_name":"Graph Generation"},{"task_slug":"image-to-text","task_name":"Image to text"},{"task_slug":null,"task_name":"Relation"},{"task_slug":"scene-graph-generation","task_name":"Scene Graph Generation"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2404.00906","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.00906"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shtuplus/pix2grp_cvpr2024","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"ran":7,"unverified":2},"by_repo_kind":{"official":{"samples":9,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"56f02812d66a0d11","entry":"generate_caption","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"app/caption.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/app/caption.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"56f02812d66a0d11"}},{"code_sha256_prefix":"92f2e16ee0d3a24d","entry":"get_concat_v","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"app/dataset_browser.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/app/dataset_browser.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"92f2e16ee0d3a24d"}},{"code_sha256_prefix":"13a43a7d815937e6","entry":"read_img","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"app/calculate_coco_features.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/app/calculate_coco_features.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"13a43a7d815937e6"}},{"code_sha256_prefix":"926bb38f979a61ef","entry":"resize_img","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"app/utils.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/app/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"926bb38f979a61ef"}},{"code_sha256_prefix":"f809f2ecf34c1c99","entry":"resize_img_w","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"app/dataset_browser.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/app/dataset_browser.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"f809f2ecf34c1c99"}},{"code_sha256_prefix":"3d4bbb5ca53fd24a","entry":"sample_dataset","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"app/dataset_browser.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/app/dataset_browser.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"3d4bbb5ca53fd24a"}},{"code_sha256_prefix":"cb33571427334815","entry":"tile","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"lavis/models/base_model.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"cb33571427334815"}},{"code_sha256_prefix":"0ec9fc2025c16f65","entry":"all_gather_with_grad","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"lavis/models/base_model.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"0ec9fc2025c16f65"}},{"code_sha256_prefix":"3e12ea8c09674aa8","entry":"compute_gradcam_batch","repo":"shtuplus/pix2grp_cvpr2024","repo_kind":"official","path":"app/multimodal_search.py","file_url":"https://github.com/shtuplus/pix2grp_cvpr2024/blob/HEAD/app/multimodal_search.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"3e12ea8c09674aa8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}