{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ppmn-pixel-phrase-matching-network-for-one","title":"PPMN: Pixel-Phrase Matching Network for One-Stage Panoptic Narrative Grounding","arxiv_id":"2208.05647","date":"2022-08-11","proceeding":null,"authors":["Zihan Ding","Zi-han Ding","Tianrui Hui","Junshi Huang","Xiaoming Wei","Xiaolin Wei","Si Liu"],"abstract":"Panoptic Narrative Grounding (PNG) is an emerging task whose goal is to segment visual objects of things and stuff categories described by dense narrative captions of a still image. The previous two-stage approach first extracts segmentation region proposals by an off-the-shelf panoptic segmentation model, then conducts coarse region-phrase matching to ground the candidate regions for each noun phrase. However, the two-stage pipeline usually suffers from the performance limitation of low-quality proposals in the first stage and the loss of spatial details caused by region feature pooling, as well as complicated strategies designed for things and stuff categories separately. To alleviate these drawbacks, we propose a one-stage end-to-end Pixel-Phrase Matching Network (PPMN), which directly matches each phrase to its corresponding pixels instead of region proposals and outputs panoptic segmentation by simple combination. Thus, our model can exploit sufficient and finer cross-modal semantic correspondence from the supervision of densely annotated pixel-phrase pairs rather than sparse region-phrase pairs. In addition, we also propose a Language-Compatible Pixel Aggregation (LCPA) module to further enhance the discriminative ability of phrase features through multi-round refinement, which selects the most compatible pixels for each phrase to adaptively aggregate the corresponding visual context. Extensive experiments show that our method achieves new state-of-the-art performance on the PNG benchmark with 4.0 absolute Average Recall gains.","url_abs":"https://arxiv.org/abs/2208.05647v1","url_pdf":"https://arxiv.org/pdf/2208.05647v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ppmn-pixel-phrase-matching-network-for-one","repo_url":"https://github.com/dzh19990407/ppmn","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"panoptic-segmentation","task_name":"Panoptic Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-correspondence","task_name":"Semantic correspondence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2208.05647","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2208.05647"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dzh19990407/ppmn","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":4,"ran_honours":2,"unverified":11},"by_repo_kind":{"official":{"samples":17,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/modeling.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"cf9ffa02a42184af","entry":"whitespace_tokenize","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/tokenization.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/tokenization.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cf9ffa02a42184af"}},{"code_sha256_prefix":"5961b5a00aea8ba7","entry":"convert_sents_to_features","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/encoder_bert.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/encoder_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5961b5a00aea8ba7"}},{"code_sha256_prefix":"fdc64f4c72036ae4","entry":"gelu","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/modeling.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fdc64f4c72036ae4"}},{"code_sha256_prefix":"87e36ec29dea3d10","entry":"trivial_batch_collator","repo":"dzh19990407/ppmn","repo_kind":"official","path":"detectron2/extract_dataloader_from_detectron2.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/detectron2/extract_dataloader_from_detectron2.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"87e36ec29dea3d10"}},{"code_sha256_prefix":"5eff22fa0a651276","entry":"url_to_filename","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/file_utils.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/file_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5eff22fa0a651276"}},{"code_sha256_prefix":"51a4f4f944992c64","entry":"binary_cross_entropy","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/knet/cross_entropy_loss.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/knet/cross_entropy_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"51a4f4f944992c64"}},{"code_sha256_prefix":"df9f10631fa5d322","entry":"cached_path","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/file_utils.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"df9f10631fa5d322"}},{"code_sha256_prefix":"0183726dd361cb88","entry":"cross_entropy","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/knet/cross_entropy_loss.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/knet/cross_entropy_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0183726dd361cb88"}},{"code_sha256_prefix":"d75f96f03066d328","entry":"dice_loss","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/knet/dice_loss.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/knet/dice_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d75f96f03066d328"}},{"code_sha256_prefix":"6db16fe8f67e56b6","entry":"filename_to_url","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/file_utils.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6db16fe8f67e56b6"}},{"code_sha256_prefix":"b22836db23a6149f","entry":"load_ckpt","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/extract_fpn_with_ckpt_load_from_detectron2.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/extract_fpn_with_ckpt_load_from_detectron2.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b22836db23a6149f"}},{"code_sha256_prefix":"68c73cc8252a81c8","entry":"load_model","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/extract_fpn_with_ckpt_load_from_detectron2.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/extract_fpn_with_ckpt_load_from_detectron2.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"68c73cc8252a81c8"}},{"code_sha256_prefix":"7099305544368003","entry":"load_tf_weights_in_bert","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/modeling.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7099305544368003"}},{"code_sha256_prefix":"a4e183f24a5ad1a7","entry":"load_vocab","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/tokenization.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/tokenization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a4e183f24a5ad1a7"}},{"code_sha256_prefix":"b6513e2a117d99af","entry":"mask_cross_entropy","repo":"dzh19990407/ppmn","repo_kind":"official","path":"models/knet/cross_entropy_loss.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/models/knet/cross_entropy_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b6513e2a117d99af"}},{"code_sha256_prefix":"55d186d867d67cb7","entry":"upsample_eval","repo":"dzh19990407/ppmn","repo_kind":"official","path":"train_net.py","file_url":"https://github.com/dzh19990407/ppmn/blob/HEAD/train_net.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"55d186d867d67cb7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}