{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/cripp-vqa-counterfactual-reasoning-about","title":"CRIPP-VQA: Counterfactual Reasoning about Implicit Physical Properties via Video Question Answering","arxiv_id":"2211.03779","date":"2022-11-07","proceeding":null,"authors":["Maitreya Patel","Tejas Gokhale","Chitta Baral","Yezhou Yang"],"abstract":"Videos often capture objects, their visible properties, their motion, and the interactions between different objects. Objects also have physical properties such as mass, which the imaging pipeline is unable to directly capture. However, these properties can be estimated by utilizing cues from relative object motion and the dynamics introduced by collisions. In this paper, we introduce CRIPP-VQA, a new video question answering dataset for reasoning about the implicit physical properties of objects in a scene. CRIPP-VQA contains videos of objects in motion, annotated with questions that involve counterfactual reasoning about the effect of actions, questions about planning in order to reach a goal, and descriptive questions about visible properties of objects. The CRIPP-VQA test set enables evaluation under several out-of-distribution settings -- videos with objects with masses, coefficients of friction, and initial velocities that are not observed in the training distribution. Our experiments reveal a surprising and significant performance gap in terms of answering questions about implicit properties (the focus of this paper) and explicit properties of objects (the focus of prior work).","url_abs":"https://arxiv.org/abs/2211.03779v1","url_pdf":"https://arxiv.org/pdf/2211.03779v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"cripp-vqa-counterfactual-reasoning-about","repo_url":"https://github.com/maitreyapatel/cripp-vqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"add-po","task_name":"Add - PO"},{"task_slug":"add-pq","task_name":"Add - PQ"},{"task_slug":"counterfactual-planning","task_name":"Counterfactual Planning"},{"task_slug":"counterfactual-reasoning","task_name":"Counterfactual Reasoning"},{"task_slug":"descriptive","task_name":"Descriptive"},{"task_slug":"friction","task_name":"Friction"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"remove-po","task_name":"Remove - PO"},{"task_slug":"remove-pq","task_name":"Remove - PQ"},{"task_slug":"replace-po","task_name":"Replace - PO"},{"task_slug":"replace-pq","task_name":"Replace - PQ"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"},{"task_slug":null,"task_name":"counterfactual"}],"methods":[{"method_slug":"test","method_name":"Test"}],"datasets_introduced":[{"slug":"cripp-vqa","name":"CRIPP-VQA","full_name":"Counterfactual Reasoning about Implicit Physical Properties via Video Question Answering"}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2211.03779","atlas_url":"https://app.syntology.ai/?focus=2211.03779","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.03779"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/thaolmk54/hcrn-videoqa","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/maitreyapatel/cripp-vqa","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/Maitreyapatel/CRIPP-VQA","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":4,"ran":1,"unverified":2},"by_repo_kind":{"official":{"samples":1,"ran":0,"repositories":1},"found_in_text":{"samples":6,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c3b215555357b9d1","entry":"conv3x3x3","repo":"thaolmk54/hcrn-videoqa","repo_kind":"found_in_text","path":"preprocess/models/pre_act_resnet.py","file_url":"https://github.com/thaolmk54/hcrn-videoqa/blob/HEAD/preprocess/models/pre_act_resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c3b215555357b9d1"}},{"code_sha256_prefix":"43c0211ff8dbca5f","entry":"downsample_basic_block","repo":"thaolmk54/hcrn-videoqa","repo_kind":"found_in_text","path":"preprocess/models/pre_act_resnet.py","file_url":"https://github.com/thaolmk54/hcrn-videoqa/blob/HEAD/preprocess/models/pre_act_resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"43c0211ff8dbca5f"}},{"code_sha256_prefix":"f1dd0a40f7d1ade3","entry":"get_fine_tuning_parameters","repo":"thaolmk54/hcrn-videoqa","repo_kind":"found_in_text","path":"preprocess/models/densenet.py","file_url":"https://github.com/thaolmk54/hcrn-videoqa/blob/HEAD/preprocess/models/densenet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f1dd0a40f7d1ade3"}},{"code_sha256_prefix":"e6dbe2d3b6f0cc87","entry":"invert_dict","repo":"thaolmk54/hcrn-videoqa","repo_kind":"found_in_text","path":"DataLoader.py","file_url":"https://github.com/thaolmk54/hcrn-videoqa/blob/HEAD/DataLoader.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e6dbe2d3b6f0cc87"}},{"code_sha256_prefix":"6d12da501ae235a1","entry":"todevice","repo":"thaolmk54/hcrn-videoqa","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/thaolmk54/hcrn-videoqa/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6d12da501ae235a1"}},{"code_sha256_prefix":"4eb168a439e7658b","entry":"load_vocab","repo":"thaolmk54/hcrn-videoqa","repo_kind":"found_in_text","path":"DataLoader.py","file_url":"https://github.com/thaolmk54/hcrn-videoqa/blob/HEAD/DataLoader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4eb168a439e7658b"}},{"code_sha256_prefix":"74e9503d785fb1c2","entry":"main","repo":"Maitreyapatel/CRIPP-VQA","repo_kind":"official","path":"evaluation/eval_regular_qa.py","file_url":"https://github.com/Maitreyapatel/CRIPP-VQA/blob/HEAD/evaluation/eval_regular_qa.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"74e9503d785fb1c2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}