{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/demi-discriminative-estimator-of-mutual-1","title":"DEMI: Discriminative Estimator of Mutual Information","arxiv_id":"2010.01766","date":"2020-10-05","proceeding":null,"authors":["Ruizhi Liao","Daniel Moyer","Polina Golland","William M. Wells"],"abstract":"Estimating mutual information between continuous random variables is often intractable and extremely challenging for high-dimensional data. Recent progress has leveraged neural networks to optimize variational lower bounds on mutual information. Although showing promise for this difficult problem, the variational methods have been theoretically and empirically proven to have serious statistical limitations: 1) many methods struggle to produce accurate estimates when the underlying mutual information is either low or high; 2) the resulting estimators may suffer from high variance. Our approach is based on training a classifier that provides the probability that a data sample pair is drawn from the joint distribution rather than from the product of its marginal distributions. Moreover, we establish a direct connection between mutual information and the average log odds estimate produced by the classifier on a test set, leading to a simple and accurate estimator of mutual information. We show theoretically that our method and other variational approaches are equivalent when they achieve their optimum, while our method sidesteps the variational bound. Empirical results demonstrate high accuracy of our approach and the advantages of our estimator in the context of representation learning. Our demo is available at https://github.com/RayRuizhiLiao/demi_mi_estimator.","url_abs":"https://arxiv.org/abs/2010.01766v2","url_pdf":"https://arxiv.org/pdf/2010.01766v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"demi-discriminative-estimator-of-mutual-1","repo_url":"https://github.com/RayRuizhiLiao/demi_mi_estimator","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"representation-learning","task_name":"Representation Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2010.01766","atlas_url":"https://app.syntology.ai/?focus=2010.01766","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.01766"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/RayRuizhiLiao/demi_mi_estimator","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":6},"by_repo_kind":{"official":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"1823c869cee40eca","entry":"cov_to_mi","repo":"RayRuizhiLiao/demi_mi_estimator","repo_kind":"official","path":"gaussian_sampler.py","file_url":"https://github.com/RayRuizhiLiao/demi_mi_estimator/blob/HEAD/gaussian_sampler.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1823c869cee40eca"}},{"code_sha256_prefix":"c617d6e85a7fb500","entry":"gaussian_generator","repo":"RayRuizhiLiao/demi_mi_estimator","repo_kind":"official","path":"gaussian_sampler.py","file_url":"https://github.com/RayRuizhiLiao/demi_mi_estimator/blob/HEAD/gaussian_sampler.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c617d6e85a7fb500"}},{"code_sha256_prefix":"8a456261317b11a7","entry":"load_and_cache_examples","repo":"RayRuizhiLiao/demi_mi_estimator","repo_kind":"official","path":"utils.py","file_url":"https://github.com/RayRuizhiLiao/demi_mi_estimator/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8a456261317b11a7"}},{"code_sha256_prefix":"6049f731f4511b38","entry":"make_mlp","repo":"RayRuizhiLiao/demi_mi_estimator","repo_kind":"official","path":"utils.py","file_url":"https://github.com/RayRuizhiLiao/demi_mi_estimator/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6049f731f4511b38"}},{"code_sha256_prefix":"970d707aca516fbd","entry":"mi_to_rho","repo":"RayRuizhiLiao/demi_mi_estimator","repo_kind":"official","path":"gaussian_sampler.py","file_url":"https://github.com/RayRuizhiLiao/demi_mi_estimator/blob/HEAD/gaussian_sampler.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"970d707aca516fbd"}},{"code_sha256_prefix":"9e2b8d22d3029fb9","entry":"permute_samples","repo":"RayRuizhiLiao/demi_mi_estimator","repo_kind":"official","path":"utils.py","file_url":"https://github.com/RayRuizhiLiao/demi_mi_estimator/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9e2b8d22d3029fb9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}