{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/end-to-end-learning-of-visual-representations","title":"End-to-End Learning of Visual Representations from Uncurated Instructional Videos","arxiv_id":"1912.06430","date":"2019-12-13","proceeding":"CVPR 2020 6","authors":["Antoine Miech","Jean-Baptiste Alayrac","Lucas Smaira","Ivan Laptev","Josef Sivic","Andrew Zisserman"],"abstract":"Annotating videos is cumbersome, expensive and not scalable. Yet, many strong video models still rely on manually annotated data. With the recent introduction of the HowTo100M dataset, narrated videos now offer the possibility of learning video representations without manual supervision. In this work we propose a new learning approach, MIL-NCE, capable of addressing misalignments inherent to narrated videos. With this approach we are able to learn strong video representations from scratch, without the need for any manual annotation. We evaluate our representations on a wide range of four downstream tasks over eight datasets: action recognition (HMDB-51, UCF-101, Kinetics-700), text-to-video retrieval (YouCook2, MSR-VTT), action localization (YouTube-8M Segments, CrossTask) and action segmentation (COIN). Our method outperforms all published self-supervised approaches for these tasks as well as several fully supervised baselines.","url_abs":"https://arxiv.org/abs/1912.06430v4","url_pdf":"https://arxiv.org/pdf/1912.06430v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"end-to-end-learning-of-visual-representations","repo_url":"https://github.com/antoine77340/MIL-NCE_HowTo100M","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"end-to-end-learning-of-visual-representations","repo_url":"https://github.com/antoine77340/S3D_HowTo100M","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"end-to-end-learning-of-visual-representations","repo_url":"https://github.com/antoine77340/milnce_howto100m","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"end-to-end-learning-of-visual-representations","repo_url":"https://github.com/linjieli222/hero_video_feature_extractor","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"action-localization","task_name":"Action Localization"},{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"},{"task_slug":"action-segmentation","task_name":"Action Segmentation"},{"task_slug":"long-video-retrieval-background-removed","task_name":"Long Video Retrieval (Background Removed)"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"text-to-video-retrieval","task_name":"Text to Video Retrieval"},{"task_slug":"video-retrieval","task_name":"Video Retrieval"},{"task_slug":"zero-shot-video-retrieval","task_name":"Zero-Shot Video Retrieval"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/action-recognition-on-rareact","task":"Action Recognition","dataset":"RareAct","model":"HT100M S3D","rank_in_archive_order":3,"of":3,"metrics":{"mWAP":"30.5"},"uses_additional_data":false},{"leaderboard":"/sota/action-segmentation-on-coin","task":"Action Segmentation","dataset":"COIN","model":"MIL-NCE","rank_in_archive_order":7,"of":9,"metrics":{"Frame accuracy":"61.0"},"uses_additional_data":false},{"leaderboard":"/sota/action-segmentation-on-coin","task":"Action Segmentation","dataset":"COIN","model":"CBT","rank_in_archive_order":9,"of":9,"metrics":{"Frame accuracy":"53.9"},"uses_additional_data":false},{"leaderboard":"/sota/long-video-retrieval-background-removed-on","task":"Long Video Retrieval (Background Removed)","dataset":"YouCook2","model":"MIL-NCE","rank_in_archive_order":6,"of":6,"metrics":{"Cap. Avg. R@1":"43.1","Cap. Avg. R@10":"79.1","Cap. Avg. R@5":"68.6"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-video-retrieval-on-msr-vtt","task":"Zero-Shot Video Retrieval","dataset":"MSR-VTT","model":"MIL-NCE","rank_in_archive_order":37,"of":41,"metrics":{"text-to-video Mean Rank":"29.5","text-to-video R@1":"9.9","text-to-video R@10":"32.4","text-to-video R@5":"24.0"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-video-retrieval-on-youcook2","task":"Zero-Shot Video Retrieval","dataset":"YouCook2","model":"MIL-NCE","rank_in_archive_order":7,"of":9,"metrics":{"text-to-video Mean Rank":"10","text-to-video R@1":"15.1","text-to-video R@10":"51.2","text-to-video R@5":"38.0"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1912.06430","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1912.06430"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/antoine77340/S3D_HowTo100M","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/antoine77340/milnce_howto100m","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/antoine77340/MIL-NCE_HowTo100M","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/linjieli222/hero_video_feature_extractor","reach":null}],"summary":{"ran":1,"ran_draft_wrong":1,"ran_fixture":1,"unverified":2},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1},"listed":{"samples":4,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d59b4b3c004c31ff","entry":"compute_metrics","repo":"antoine77340/milnce_howto100m","repo_kind":"listed","path":"metrics.py","file_url":"https://github.com/antoine77340/milnce_howto100m/blob/HEAD/metrics.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d59b4b3c004c31ff"}},{"code_sha256_prefix":"4d241892c1ca731a","entry":"get_last_checkpoint","repo":"antoine77340/MIL-NCE_HowTo100M","repo_kind":"official","path":"main_distributed.py","file_url":"https://github.com/antoine77340/MIL-NCE_HowTo100M/blob/HEAD/main_distributed.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4d241892c1ca731a"}},{"code_sha256_prefix":"45cc344e60f640a1","entry":"get_padding_shape","repo":"antoine77340/milnce_howto100m","repo_kind":"listed","path":"s3dg.py","file_url":"https://github.com/antoine77340/milnce_howto100m/blob/HEAD/s3dg.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"45cc344e60f640a1"}},{"code_sha256_prefix":"265b876728ccc704","entry":"get_args","repo":"antoine77340/milnce_howto100m","repo_kind":"listed","path":"args.py","file_url":"https://github.com/antoine77340/milnce_howto100m/blob/HEAD/args.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"265b876728ccc704"}},{"code_sha256_prefix":"a9c14b9fb71dbe7c","entry":"get_cosine_schedule_with_warmup","repo":"antoine77340/milnce_howto100m","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/antoine77340/milnce_howto100m/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a9c14b9fb71dbe7c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}