{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unified-image-and-video-saliency-modeling","title":"Unified Image and Video Saliency Modeling","arxiv_id":"2003.05477","date":"2020-03-11","proceeding":"ECCV 2020 8","authors":["Richard Droste","Jianbo Jiao","J. Alison Noble"],"abstract":"Visual saliency modeling for images and videos is treated as two independent tasks in recent computer vision literature. While image saliency modeling is a well-studied problem and progress on benchmarks like SALICON and MIT300 is slowing, video saliency models have shown rapid gains on the recent DHF1K benchmark. Here, we take a step back and ask: Can image and video saliency modeling be approached via a unified model, with mutual benefit? We identify different sources of domain shift between image and video saliency data and between different video saliency datasets as a key challenge for effective joint modelling. To address this we propose four novel domain adaptation techniques - Domain-Adaptive Priors, Domain-Adaptive Fusion, Domain-Adaptive Smoothing and Bypass-RNN - in addition to an improved formulation of learned Gaussian priors. We integrate these techniques into a simple and lightweight encoder-RNN-decoder-style network, UNISAL, and train it jointly with image and video saliency data. We evaluate our method on the video saliency datasets DHF1K, Hollywood-2 and UCF-Sports, and the image saliency datasets SALICON and MIT300. With one set of parameters, UNISAL achieves state-of-the-art performance on all video saliency datasets and is on par with the state-of-the-art for image saliency datasets, despite faster runtime and a 5 to 20-fold smaller model size compared to all competing deep methods. We provide retrospective analyses and ablation studies which confirm the importance of the domain shift modeling. The code is available at https://github.com/rdroste/unisal","url_abs":"https://arxiv.org/abs/2003.05477v3","url_pdf":"https://arxiv.org/pdf/2003.05477v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unified-image-and-video-saliency-modeling","repo_url":"https://github.com/rdroste/unisal","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"unified-image-and-video-saliency-modeling","repo_url":"https://github.com/lcc157/code","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"domain-adaptation","task_name":"Domain Adaptation"},{"task_slug":"saliency-prediction","task_name":"Saliency Prediction"},{"task_slug":"video-saliency-detection","task_name":"Video Saliency Detection"}],"methods":[{"method_slug":"3d-convolution","method_name":"3D Convolution"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/video-saliency-detection-on-msu-video","task":"Video Saliency Detection","dataset":"MSU Video Saliency Prediction","model":"UNISAL (videos)","rank_in_archive_order":4,"of":14,"metrics":{"AUC-J":"0.858","CC":"0.707","FPS":"70.46","KLDiv":"0.536","NSS":"2.03","SIM":"0.609"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2003.05477","atlas_url":"https://app.syntology.ai/?focus=2003.05477","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2003.05477"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rdroste/unisal","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lcc157/code","reach":null}],"summary":{"ran_draft_wrong":2,"ran":4,"ran_honours":1,"unverified":2},"by_repo_kind":{"official":{"samples":6,"ran":6,"repositories":1},"listed":{"samples":3,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"a0131fb70c267a9e","entry":"conv_1x1_bn","repo":"rdroste/unisal","repo_kind":"official","path":"unisal/models/MobileNetV2.py","file_url":"https://github.com/rdroste/unisal/blob/HEAD/unisal/models/MobileNetV2.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a0131fb70c267a9e"}},{"code_sha256_prefix":"e087cbfa1e53486d","entry":"conv_bn","repo":"rdroste/unisal","repo_kind":"official","path":"unisal/models/MobileNetV2.py","file_url":"https://github.com/rdroste/unisal/blob/HEAD/unisal/models/MobileNetV2.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e087cbfa1e53486d"}},{"code_sha256_prefix":"5b2e8323d628096d","entry":"get_kwargs_dict","repo":"rdroste/unisal","repo_kind":"official","path":"unisal/utils.py","file_url":"https://github.com/rdroste/unisal/blob/HEAD/unisal/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5b2e8323d628096d"}},{"code_sha256_prefix":"2812f253b8a66480","entry":"get_kwargs_names","repo":"rdroste/unisal","repo_kind":"official","path":"unisal/utils.py","file_url":"https://github.com/rdroste/unisal/blob/HEAD/unisal/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2812f253b8a66480"}},{"code_sha256_prefix":"c101e7bffe5bcda9","entry":"get_optimal_out_size","repo":"rdroste/unisal","repo_kind":"official","path":"unisal/data.py","file_url":"https://github.com/rdroste/unisal/blob/HEAD/unisal/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c101e7bffe5bcda9"}},{"code_sha256_prefix":"11091601dda5e015","entry":"load_module","repo":"rdroste/unisal","repo_kind":"official","path":"unisal/utils.py","file_url":"https://github.com/rdroste/unisal/blob/HEAD/unisal/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"11091601dda5e015"}},{"code_sha256_prefix":"2859db1f0821894e","entry":"normalize_map","repo":"lcc157/code","repo_kind":"listed","path":"unisal/salience_metrics.py","file_url":"https://github.com/lcc157/code/blob/HEAD/unisal/salience_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2859db1f0821894e"}},{"code_sha256_prefix":"386eb8dd9e132cce","entry":"auc_judd","repo":"lcc157/code","repo_kind":"listed","path":"unisal/salience_metrics.py","file_url":"https://github.com/lcc157/code/blob/HEAD/unisal/salience_metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"386eb8dd9e132cce"}},{"code_sha256_prefix":"e64aa209f895e3d2","entry":"auc_shuff_acl","repo":"lcc157/code","repo_kind":"listed","path":"unisal/salience_metrics.py","file_url":"https://github.com/lcc157/code/blob/HEAD/unisal/salience_metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e64aa209f895e3d2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}