{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unbiased-scene-graph-generation-from-biased","title":"Unbiased Scene Graph Generation from Biased Training","arxiv_id":"2002.11949","date":"2020-02-27","proceeding":"CVPR 2020 6","authors":["Kaihua Tang","Yulei Niu","Jianqiang Huang","Jiaxin Shi","Hanwang Zhang"],"abstract":"Today's scene graph generation (SGG) task is still far from practical, mainly due to the severe training bias, e.g., collapsing diverse \"human walk on / sit on / lay on beach\" into \"human on beach\". Given such SGG, the down-stream tasks such as VQA can hardly infer better scene structures than merely a bag of objects. However, debiasing in SGG is not trivial because traditional debiasing methods cannot distinguish between the good and bad bias, e.g., good context prior (e.g., \"person read book\" rather than \"eat\") and bad long-tailed bias (e.g., \"near\" dominating \"behind / in front of\"). In this paper, we present a novel SGG framework based on causal inference but not the conventional likelihood. We first build a causal graph for SGG, and perform traditional biased training with the graph. Then, we propose to draw the counterfactual causality from the trained graph to infer the effect from the bad bias, which should be removed. In particular, we use Total Direct Effect (TDE) as the proposed final predicate score for unbiased SGG. Note that our framework is agnostic to any SGG model and thus can be widely applied in the community who seeks unbiased predictions. By using the proposed Scene Graph Diagnosis toolkit on the SGG benchmark Visual Genome and several prevailing models, we observed significant improvements over the previous state-of-the-art methods.","url_abs":"https://arxiv.org/abs/2002.11949v3","url_pdf":"https://arxiv.org/pdf/2002.11949v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unbiased-scene-graph-generation-from-biased","repo_url":"https://github.com/KaihuaTang/Scene-Graph-Benchmark.pytorch","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"unbiased-scene-graph-generation-from-biased","repo_url":"https://github.com/Karim-53/SGG","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"unbiased-scene-graph-generation-from-biased","repo_url":"https://github.com/coldmanck/recovering-unbiased-scene-graphs","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"unbiased-scene-graph-generation-from-biased","repo_url":"https://github.com/ihaeyong/unbiased-sgg","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"unbiased-scene-graph-generation-from-biased","repo_url":"https://github.com/jaleedkhan/jSGG","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"unbiased-scene-graph-generation-from-biased","repo_url":"https://github.com/zacharie12/zacharie","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"causal-inference","task_name":"Causal Inference"},{"task_slug":"graph-generation","task_name":"Graph Generation"},{"task_slug":"scene-graph-generation","task_name":"Scene Graph Generation"},{"task_slug":"unbiased-scene-graph-generation","task_name":"Unbiased Scene Graph Generation"},{"task_slug":null,"task_name":"counterfactual"}],"methods":[{"method_slug":"causal-inference","method_name":"Causal inference"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/scene-graph-generation-on-visual-genome","task":"Scene Graph Generation","dataset":"Visual Genome","model":"Causal-TDE","rank_in_archive_order":3,"of":19,"metrics":{"Recall@50":"31.93","mean Recall @20":"6.9"},"uses_additional_data":false},{"leaderboard":"/sota/unbiased-scene-graph-generation-on-visual","task":"Unbiased Scene Graph Generation","dataset":"Visual Genome","model":"TDE (VCTree-ResNeXt-101-FPN backbone; PredCls mode)","rank_in_archive_order":9,"of":31,"metrics":{"F@100":"36.9","mR@20":"19.2","ng-mR@20":"20.9"},"uses_additional_data":false},{"leaderboard":"/sota/unbiased-scene-graph-generation-on-visual","task":"Unbiased Scene Graph Generation","dataset":"Visual Genome","model":"TDE (MOTIFS-ResNeXt-101-FPN backbone; PredCls mode)","rank_in_archive_order":10,"of":31,"metrics":{"F@100":"37.2","mR@20":"17.4","ng-mR@20":"18.7"},"uses_additional_data":false},{"leaderboard":"/sota/unbiased-scene-graph-generation-on-visual","task":"Unbiased Scene Graph Generation","dataset":"Visual Genome","model":"TDE (VCTree-ResNeXt-101-FPN backbone; SGCls mode)","rank_in_archive_order":16,"of":31,"metrics":{"F@100":"18.6","mR@20":"11.2","ng-mR@20":"12.4"},"uses_additional_data":false},{"leaderboard":"/sota/unbiased-scene-graph-generation-on-visual","task":"Unbiased Scene Graph Generation","dataset":"Visual Genome","model":"TDE (MOTIFS-ResNeXt-101-FPN backbone; SGCls mode)","rank_in_archive_order":19,"of":31,"metrics":{"F@100":"19.9","mR@20":"9.9","ng-mR@20":"10.7"},"uses_additional_data":false},{"leaderboard":"/sota/unbiased-scene-graph-generation-on-visual","task":"Unbiased Scene Graph Generation","dataset":"Visual Genome","model":"TDE (VCTree-ResNeXt-101-FPN backbone; SGDet mode)","rank_in_archive_order":23,"of":31,"metrics":{"F@100":"15.1","mR@20":"6.8","ng-mR@20":"7.8"},"uses_additional_data":false},{"leaderboard":"/sota/unbiased-scene-graph-generation-on-visual","task":"Unbiased Scene Graph Generation","dataset":"Visual Genome","model":"TDE (MOTIFS-ResNeXt-101-FPN backbone; SGDet mode)","rank_in_archive_order":24,"of":31,"metrics":{"F@100":"13.2","mR@20":"9.7","ng-mR@20":"7.4"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2002.11949","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2002.11949"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jaleedkhan/jSGG","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ihaeyong/unbiased-sgg","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/coldmanck/recovering-unbiased-scene-graphs","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Karim-53/SGG","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/KaihuaTang/Scene-Graph-Benchmark.pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zacharie12/zacharie","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_draft_wrong":2,"ran_fixture":1,"unverified":3},"by_repo_kind":{"official":{"samples":3,"ran":0,"repositories":1},"listed":{"samples":2,"ran":2,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"fbc88f9f38569eea","entry":"build_roi_relation_head","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"fbc88f9f38569eea"}},{"code_sha256_prefix":"32a2dca6019370be","entry":"build_roi_relation_head","repo":"ihaeyong/unbiased-sgg","repo_kind":"listed","path":"maskrcnn_benchmark/modeling/roi_heads/relation_head/relation_head.py","file_url":"https://github.com/ihaeyong/unbiased-sgg/blob/HEAD/maskrcnn_benchmark/modeling/roi_heads/relation_head/relation_head.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"32a2dca6019370be"}},{"code_sha256_prefix":"6811cd526ab3e1a7","entry":"get_center_delta","repo":"coldmanck/recovering-unbiased-scene-graphs","repo_kind":"listed","path":"tools/relation_train_net.py","file_url":"https://github.com/coldmanck/recovering-unbiased-scene-graphs/blob/HEAD/tools/relation_train_net.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6811cd526ab3e1a7"}},{"code_sha256_prefix":"2d943c0c96e61f16","entry":"entropy_loss","repo":"KaihuaTang/Scene-Graph-Benchmark.pytorch","repo_kind":"official","path":"maskrcnn_benchmark/layers/entropy_loss.py","file_url":"https://github.com/KaihuaTang/Scene-Graph-Benchmark.pytorch/blob/HEAD/maskrcnn_benchmark/layers/entropy_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"2d943c0c96e61f16"}},{"code_sha256_prefix":"2902bf4410253ff7","entry":"interpolate","repo":"KaihuaTang/Scene-Graph-Benchmark.pytorch","repo_kind":"official","path":"maskrcnn_benchmark/layers/misc.py","file_url":"https://github.com/KaihuaTang/Scene-Graph-Benchmark.pytorch/blob/HEAD/maskrcnn_benchmark/layers/misc.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"2902bf4410253ff7"}},{"code_sha256_prefix":"382942f3ada73679","entry":"kl_div_loss","repo":"KaihuaTang/Scene-Graph-Benchmark.pytorch","repo_kind":"official","path":"maskrcnn_benchmark/layers/kl_div_loss.py","file_url":"https://github.com/KaihuaTang/Scene-Graph-Benchmark.pytorch/blob/HEAD/maskrcnn_benchmark/layers/kl_div_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"382942f3ada73679"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}