{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ffb-a-fair-fairness-benchmark-for-in","title":"FFB: A Fair Fairness Benchmark for In-Processing Group Fairness Methods","arxiv_id":"2306.09468","date":"2023-06-15","proceeding":null,"authors":["Xiaotian Han","Jianfeng Chi","Yu Chen","Qifan Wang","Han Zhao","Na Zou","Xia Hu"],"abstract":"This paper introduces the Fair Fairness Benchmark (\\textsf{FFB}), a benchmarking framework for in-processing group fairness methods. Ensuring fairness in machine learning is important for ethical compliance. However, there exist challenges in comparing and developing fairness methods due to inconsistencies in experimental settings, lack of accessible algorithmic implementations, and limited extensibility of current fairness packages and tools. To address these issues, we introduce an open-source standardized benchmark for evaluating in-processing group fairness methods and provide a comprehensive analysis of state-of-the-art methods to ensure different notions of group fairness. This work offers the following key contributions: the provision of flexible, extensible, minimalistic, and research-oriented open-source code; the establishment of unified fairness method benchmarking pipelines; and extensive benchmarking, which yields key insights from $\\mathbf{45,079}$ experiments, $\\mathbf{14,428}$ GPU hours. We believe that our work will significantly facilitate the growth and development of the fairness research community.","url_abs":"https://arxiv.org/abs/2306.09468v2","url_pdf":"https://arxiv.org/pdf/2306.09468v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ffb-a-fair-fairness-benchmark-for-in","repo_url":"https://github.com/ahxt/fair_fairness_benchmark","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"fairness","task_name":"Fairness"},{"task_slug":null,"task_name":"GPU"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2306.09468","atlas_url":"https://app.syntology.ai/?focus=2306.09468","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.09468"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ahxt/fair_fairness_benchmark","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":9},"by_repo_kind":{"official":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f0a20ecb976a4b4e","entry":"load_adult_data","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/dataset.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f0a20ecb976a4b4e"}},{"code_sha256_prefix":"831fffb30d05aae1","entry":"load_german_data","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/dataset.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"831fffb30d05aae1"}},{"code_sha256_prefix":"6efc0a57d5e4e72d","entry":"load_jigsaw_text_data","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/dataset.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6efc0a57d5e4e72d"}},{"code_sha256_prefix":"e44410ab58a2a6a1","entry":"train_epoch","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/ffb_tabular_adv_gr.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/ffb_tabular_adv_gr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e44410ab58a2a6a1"}},{"code_sha256_prefix":"7403ecdd83befa98","entry":"train_epoch","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/ffb_tabular_diffdp.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/ffb_tabular_diffdp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7403ecdd83befa98"}},{"code_sha256_prefix":"ec995a87fb3aa08f","entry":"train_step","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/ffb_image_diffdp.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/ffb_image_diffdp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ec995a87fb3aa08f"}},{"code_sha256_prefix":"a6a63d55993a1803","entry":"train_step","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/ffb_image_diffeodd.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/ffb_image_diffeodd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a6a63d55993a1803"}},{"code_sha256_prefix":"fdec6b87d8ff457b","entry":"train_step","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/ffb_image_erm.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/ffb_image_erm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fdec6b87d8ff457b"}},{"code_sha256_prefix":"4313a3b250fe0838","entry":"train_step","repo":"ahxt/fair_fairness_benchmark","repo_kind":"official","path":"src/ffb_tabular_adv_gr.py","file_url":"https://github.com/ahxt/fair_fairness_benchmark/blob/HEAD/src/ffb_tabular_adv_gr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4313a3b250fe0838"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}