{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/psalm-pixelwise-segmentation-with-large-multi","title":"PSALM: Pixelwise SegmentAtion with Large Multi-Modal Model","arxiv_id":"2403.14598","date":"2024-03-21","proceeding":null,"authors":["Zheng Zhang","Yeyao Ma","Enming Zhang","Xiang Bai"],"abstract":"PSALM is a powerful extension of the Large Multi-modal Model (LMM) to address the segmentation task challenges. To overcome the limitation of the LMM being limited to textual output, PSALM incorporates a mask decoder and a well-designed input schema to handle a variety of segmentation tasks. This schema includes images, task instructions, conditional prompts, and mask tokens, which enable the model to generate and classify segmentation masks effectively. The flexible design of PSALM supports joint training across multiple datasets and tasks, leading to improved performance and task generalization. PSALM achieves superior results on several benchmarks, such as RefCOCO/RefCOCO+/RefCOCOg, COCO Panoptic Segmentation, and COCO-Interactive, and further exhibits zero-shot capabilities on unseen tasks, such as open-vocabulary segmentation, generalized referring expression segmentation and video object segmentation, making a significant step towards a GPT moment in computer vision. Through extensive experiments, PSALM demonstrates its potential to transform the domain of image segmentation, leveraging the robust visual understanding capabilities of LMMs as seen in natural language processing. Code and models are available at https://github.com/zamling/PSALM.","url_abs":"https://arxiv.org/abs/2403.14598v1","url_pdf":"https://arxiv.org/pdf/2403.14598v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"psalm-pixelwise-segmentation-with-large-multi","repo_url":"https://github.com/zamling/psalm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"generalized-referring-expression-segmentation","task_name":"Generalized Referring Expression Segmentation"},{"task_slug":"image-segmentation","task_name":"Image Segmentation"},{"task_slug":"panoptic-segmentation","task_name":"Panoptic Segmentation"},{"task_slug":"referring-expression","task_name":"Referring Expression"},{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"video-object-segmentation","task_name":"Video Object Segmentation"},{"task_slug":"video-semantic-segmentation","task_name":"Video Semantic Segmentation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt","method_name":"GPT"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"PSALM","rank_in_archive_order":3,"of":37,"metrics":{"Overall IoU":"83.6"},"uses_additional_data":true}],"syntology":{"syntology_url":"https://syntology.ai/paper/2403.14598","atlas_url":"https://app.syntology.ai/?focus=2403.14598","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.14598"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zamling/psalm","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/zamling/PSALM","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":2,"unverified":5},"by_repo_kind":{"official":{"samples":7,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c3ee9d07c900dd55","entry":"load_image_from_base64","repo":"zamling/PSALM","repo_kind":"official","path":"psalm/mm_utils.py","file_url":"https://github.com/zamling/PSALM/blob/HEAD/psalm/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c3ee9d07c900dd55"}},{"code_sha256_prefix":"1df990c375318896","entry":"process_images","repo":"zamling/PSALM","repo_kind":"official","path":"psalm/mm_utils.py","file_url":"https://github.com/zamling/PSALM/blob/HEAD/psalm/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1df990c375318896"}},{"code_sha256_prefix":"6cf5142ee7139417","entry":"compute_metric","repo":"zamling/PSALM","repo_kind":"official","path":"psalm/eval/eval_davis.py","file_url":"https://github.com/zamling/PSALM/blob/HEAD/psalm/eval/eval_davis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6cf5142ee7139417"}},{"code_sha256_prefix":"136834d66336e6b9","entry":"intersectionAndUnionGPU","repo":"zamling/PSALM","repo_kind":"official","path":"psalm/eval/eval_davis.py","file_url":"https://github.com/zamling/PSALM/blob/HEAD/psalm/eval/eval_davis.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"136834d66336e6b9"}},{"code_sha256_prefix":"c61d9f251e8adf30","entry":"parse_outputs","repo":"zamling/PSALM","repo_kind":"official","path":"psalm/eval/eval_davis.py","file_url":"https://github.com/zamling/PSALM/blob/HEAD/psalm/eval/eval_davis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c61d9f251e8adf30"}},{"code_sha256_prefix":"37899f22fb191b37","entry":"pretty_print_semaphore","repo":"zamling/PSALM","repo_kind":"official","path":"psalm/utils.py","file_url":"https://github.com/zamling/PSALM/blob/HEAD/psalm/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37899f22fb191b37"}},{"code_sha256_prefix":"f9939a84b9a65279","entry":"violates_moderation","repo":"zamling/PSALM","repo_kind":"official","path":"psalm/utils.py","file_url":"https://github.com/zamling/PSALM/blob/HEAD/psalm/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f9939a84b9a65279"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}