{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/segmenting-moving-objects-via-an-object","title":"Segmenting Moving Objects via an Object-Centric Layered Representation","arxiv_id":"2207.02206","date":"2022-07-05","proceeding":null,"authors":["Junyu Xie","Weidi Xie","Andrew Zisserman"],"abstract":"The objective of this paper is a model that is able to discover, track and segment multiple moving objects in a video. We make four contributions: First, we introduce an object-centric segmentation model with a depth-ordered layer representation. This is implemented using a variant of the transformer architecture that ingests optical flow, where each query vector specifies an object and its layer for the entire video. The model can effectively discover multiple moving objects and handle mutual occlusions; Second, we introduce a scalable pipeline for generating multi-object synthetic training data via layer compositions, that is used to train the proposed model, significantly reducing the requirements for labour-intensive annotations, and supporting Sim2Real generalisation; Third, we conduct thorough ablation studies, showing that the model is able to learn object permanence and temporal shape consistency, and is able to predict amodal segmentation masks; Fourth, we evaluate our model, trained only on synthetic data, on standard video segmentation benchmarks, DAVIS, MoCA, SegTrack, FBMS-59, and achieve state-of-the-art performance among existing methods that do not rely on any manual annotations. With test-time adaptation, we observe further performance boosts.","url_abs":"https://arxiv.org/abs/2207.02206v2","url_pdf":"https://arxiv.org/pdf/2207.02206v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"segmenting-moving-objects-via-an-object","repo_url":"https://github.com/Jyxarthur/OCLR_model","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"motion-segmentation","task_name":"Motion Segmentation"},{"task_slug":"object","task_name":"Object"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"test-time-adaptation","task_name":"Test-time Adaptation"},{"task_slug":"unsupervised-object-segmentation","task_name":"Unsupervised Object Segmentation"},{"task_slug":"video-segmentation","task_name":"Video Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/unsupervised-object-segmentation-on-davis","task":"Unsupervised Object Segmentation","dataset":"DAVIS 2016","model":"OCLR","rank_in_archive_order":4,"of":9,"metrics":{"J score":"72.1"},"uses_additional_data":true},{"leaderboard":"/sota/unsupervised-object-segmentation-on-fbms-59","task":"Unsupervised Object Segmentation","dataset":"FBMS-59","model":"OCLR","rank_in_archive_order":3,"of":7,"metrics":{"mIoU":"65.4"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-object-segmentation-on-segtrack","task":"Unsupervised Object Segmentation","dataset":"SegTrack-v2","model":"OCLR","rank_in_archive_order":3,"of":8,"metrics":{"mIoU":"67.6"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2207.02206","atlas_url":"https://app.syntology.ai/?focus=2207.02206","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.02206"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Jyxarthur/OCLR_model","reach":null}],"summary":{"ran":11,"ran_draft_wrong":4,"ran_honours":1,"unverified":5},"by_repo_kind":{"official":{"samples":21,"ran":16,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"867da52dc8c76b0f","entry":"AttnMap","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"867da52dc8c76b0f"}},{"code_sha256_prefix":"75d889c61752310c","entry":"ConvBlock","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"75d889c61752310c"}},{"code_sha256_prefix":"8378236c795fdc48","entry":"EncoderLayer","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8378236c795fdc48"}},{"code_sha256_prefix":"a447981fffd42b39","entry":"MultiHeadAttention","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a447981fffd42b39"}},{"code_sha256_prefix":"55e3d071afe4d46b","entry":"Norm","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"55e3d071afe4d46b"}},{"code_sha256_prefix":"f66e673c9146f36d","entry":"OrderingHead","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f66e673c9146f36d"}},{"code_sha256_prefix":"47038bfa9f36a207","entry":"SoftPositionTimeEmbed","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"47038bfa9f36a207"}},{"code_sha256_prefix":"fcbab3ccaf3b573d","entry":"SoftTimeEmbed","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fcbab3ccaf3b573d"}},{"code_sha256_prefix":"d4e6b98e360c24bf","entry":"TransDecoder","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d4e6b98e360c24bf"}},{"code_sha256_prefix":"1c0dce82d6701d1a","entry":"UNet_decoder","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1c0dce82d6701d1a"}},{"code_sha256_prefix":"5a52ccb38fd0d98e","entry":"UNet_encoder","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5a52ccb38fd0d98e"}},{"code_sha256_prefix":"a6a5c2077b398c5d","entry":"_in_projection_packed","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a6a5c2077b398c5d"}},{"code_sha256_prefix":"86341ad6b5c2c7a4","entry":"attention","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"86341ad6b5c2c7a4"}},{"code_sha256_prefix":"6cb4b60cb90aeb1d","entry":"attn_mask","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6cb4b60cb90aeb1d"}},{"code_sha256_prefix":"584cc60931efa04b","entry":"build_spacetime_grid","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"584cc60931efa04b"}},{"code_sha256_prefix":"7a79f5c1eb6acc1b","entry":"build_time_grid","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7a79f5c1eb6acc1b"}},{"code_sha256_prefix":"34a9dcdc4c7bfe30","entry":"BTNTransformerDec","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"34a9dcdc4c7bfe30"}},{"code_sha256_prefix":"2d11b5a36034202a","entry":"BTNTransformerEnc","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2d11b5a36034202a"}},{"code_sha256_prefix":"137850413fc262a6","entry":"DecoderLayer","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"137850413fc262a6"}},{"code_sha256_prefix":"09bb9b075e9f690d","entry":"OCLR","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"09bb9b075e9f690d"}},{"code_sha256_prefix":"1eceb7d7a75f267f","entry":"TransEncoder","repo":"Jyxarthur/OCLR_model","repo_kind":"official","path":"models/oclr.py","file_url":"https://github.com/Jyxarthur/OCLR_model/blob/HEAD/models/oclr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1eceb7d7a75f267f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}