{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/minde-mutual-information-neural-diffusion","title":"MINDE: Mutual Information Neural Diffusion Estimation","arxiv_id":"2310.09031","date":"2023-10-13","proceeding":null,"authors":["Giulio Franzese","Mustapha Bounoua","Pietro Michiardi"],"abstract":"In this work we present a new method for the estimation of Mutual Information (MI) between random variables. Our approach is based on an original interpretation of the Girsanov theorem, which allows us to use score-based diffusion models to estimate the Kullback Leibler divergence between two densities as a difference between their score functions. As a by-product, our method also enables the estimation of the entropy of random variables. Armed with such building blocks, we present a general recipe to measure MI, which unfolds in two directions: one uses conditional diffusion process, whereas the other uses joint diffusion processes that allow simultaneous modelling of two random variables. Our results, which derive from a thorough experimental protocol over all the variants of our approach, indicate that our method is more accurate than the main alternatives from the literature, especially for challenging distributions. Furthermore, our methods pass MI self-consistency tests, including data processing and additivity under independence, which instead are a pain-point of existing methods.","url_abs":"https://arxiv.org/abs/2310.09031v2","url_pdf":"https://arxiv.org/pdf/2310.09031v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"minde-mutual-information-neural-diffusion","repo_url":"https://github.com/MustaphaBounoua/minde","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok"}}],"tasks":[{"task_slug":"mutual-information-estimation","task_name":"Mutual Information Estimation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2310.09031","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.09031"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MustaphaBounoua/minde","reach":{"status":"ok"}}],"summary":{"ran_honours":2,"ran":10,"ran_violates":2,"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"official":{"samples":17,"ran":15,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":17,"samples":[{"code_sha256_prefix":"7f1040f5e3991d5e","entry":"identity","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/models/mlp.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/models/mlp.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7f1040f5e3991d5e"}},{"code_sha256_prefix":"664629edefb519ca","entry":"compute_negative_ln_prob","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/baseline/doe.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/baseline/doe.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"664629edefb519ca"}},{"code_sha256_prefix":"217d8f080fa03d5a","entry":"concat_vect","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/libs/util.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/libs/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"217d8f080fa03d5a"}},{"code_sha256_prefix":"b14c2df44bc7cc53","entry":"deconcat","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/libs/util.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/libs/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b14c2df44bc7cc53"}},{"code_sha256_prefix":"d1ef6b8cb9a28a53","entry":"default","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/models/mlp.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/models/mlp.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d1ef6b8cb9a28a53"}},{"code_sha256_prefix":"608e364a9d2376a3","entry":"exists","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/models/mlp.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/models/mlp.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"608e364a9d2376a3"}},{"code_sha256_prefix":"c92c27c924b517e8","entry":"get_2d_sincos_pos_embed","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/models/transformer.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/models/transformer.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c92c27c924b517e8"}},{"code_sha256_prefix":"b467609613cd30c7","entry":"get_data_loader","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/baseline/eval_doe.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/baseline/eval_doe.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b467609613cd30c7"}},{"code_sha256_prefix":"7d7a7d691ec8ba36","entry":"get_ln_score_function","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/baseline/doe.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/baseline/doe.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7d7a7d691ec8ba36"}},{"code_sha256_prefix":"58f73aefe072a299","entry":"get_normalizing_constant","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/libs/importance.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/libs/importance.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"58f73aefe072a299"}},{"code_sha256_prefix":"3386981c26ab7a77","entry":"log_normal","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/libs/importance.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/libs/importance.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3386981c26ab7a77"}},{"code_sha256_prefix":"03310bba324ae4fb","entry":"modulate","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/models/transformer.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/models/transformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"03310bba324ae4fb"}},{"code_sha256_prefix":"40742aca28e365db","entry":"pop_elem_i","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/libs/util.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/libs/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"40742aca28e365db"}},{"code_sha256_prefix":"d5b6945ebc4a6b15","entry":"sample_vp_truncated_q","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/libs/importance.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/libs/importance.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d5b6945ebc4a6b15"}},{"code_sha256_prefix":"10cb8c435da6f038","entry":"train_model","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/baseline/eval_doe.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/baseline/eval_doe.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"10cb8c435da6f038"}},{"code_sha256_prefix":"665d8a4e8f673a4c","entry":"get_2d_sincos_pos_embed_from_grid","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/models/transformer.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/models/transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"665d8a4e8f673a4c"}},{"code_sha256_prefix":"0faaf4c77cd5b5e7","entry":"test_model","repo":"MustaphaBounoua/minde","repo_kind":"official","path":"src/baseline/eval_doe.py","file_url":"https://github.com/MustaphaBounoua/minde/blob/HEAD/src/baseline/eval_doe.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0faaf4c77cd5b5e7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}