{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sound-localization-from-motion-jointly","title":"Sound Localization from Motion: Jointly Learning Sound Direction and Camera Rotation","arxiv_id":"2303.11329","date":"2023-03-20","proceeding":"ICCV 2023 1","authors":["Ziyang Chen","Shengyi Qian","Andrew Owens"],"abstract":"The images and sounds that we perceive undergo subtle but geometrically consistent changes as we rotate our heads. In this paper, we use these cues to solve a problem we call Sound Localization from Motion (SLfM): jointly estimating camera rotation and localizing sound sources. We learn to solve these tasks solely through self-supervision. A visual model predicts camera rotation from a pair of images, while an audio model predicts the direction of sound sources from binaural sounds. We train these models to generate predictions that agree with one another. At test time, the models can be deployed independently. To obtain a feature representation that is well-suited to solving this challenging problem, we also propose a method for learning an audio-visual representation through cross-view binauralization: estimating binaural sound from one view, given images and sound from another. Our model can successfully estimate accurate rotations on both real and synthetic scenes, and localize sound sources with accuracy competitive with state-of-the-art self-supervised approaches. Project site: https://ificl.github.io/SLfM/","url_abs":"https://arxiv.org/abs/2303.11329v2","url_pdf":"https://arxiv.org/pdf/2303.11329v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sound-localization-from-motion-jointly","repo_url":"https://github.com/IFICL/SLfM","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"sound-localization-from-motion-jointly","repo_url":"https://github.com/wskingdom/crowd-hat","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"test","method_name":"Test"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2303.11329","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.11329"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wskingdom/crowd-hat","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/IFICL/SLfM","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"unverified":10},"by_repo_kind":{"official":{"samples":11,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d18d05278cbd104f","entry":"unet_upconv","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/models/audio_unet.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/models/audio_unet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d18d05278cbd104f"}},{"code_sha256_prefix":"7c5f6563574c70dd","entry":"conv2d","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/models/audio_unet.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/models/audio_unet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7c5f6563574c70dd"}},{"code_sha256_prefix":"e895d03c674bf8a1","entry":"init_args","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/config/config.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/config/config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e895d03c674bf8a1"}},{"code_sha256_prefix":"5c50f59339937d40","entry":"logit2angle","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/evaluation/evaluate_angle.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/evaluation/evaluate_angle.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5c50f59339937d40"}},{"code_sha256_prefix":"e2c10d8abac5a4d3","entry":"predict","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/main_audio.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/main_audio.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e2c10d8abac5a4d3"}},{"code_sha256_prefix":"8c280d3682a41f71","entry":"predict","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/main_camera.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/main_camera.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8c280d3682a41f71"}},{"code_sha256_prefix":"8831db672f635424","entry":"predict","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/evaluation/functs.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/evaluation/functs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8831db672f635424"}},{"code_sha256_prefix":"de1947a2864a7738","entry":"rot2theta","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/evaluation/evaluate_angle.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/evaluation/evaluate_angle.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"de1947a2864a7738"}},{"code_sha256_prefix":"27b1140af5e3661d","entry":"unet_conv","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/models/audio_unet.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/models/audio_unet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"27b1140af5e3661d"}},{"code_sha256_prefix":"86a4e8ccd2e033b5","entry":"validation","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/main_audio.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/main_audio.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"86a4e8ccd2e033b5"}},{"code_sha256_prefix":"e001d7497056ed09","entry":"validation","repo":"IFICL/SLfM","repo_kind":"official","path":"slfm/main_camera.py","file_url":"https://github.com/IFICL/SLfM/blob/HEAD/slfm/main_camera.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e001d7497056ed09"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}