{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2605-03824","title":"Reproducing Complex Set-Compositional Information Retrieval","arxiv_id":"2605.03824","date":"2026-05-05","proceeding":null,"authors":["Vincent Degenhart","Dewi Timman","Arjen P. de Vries","Faegheh Hasibi","Mohanna Hoveyda"],"abstract":"Complex information needs may involve set-compositional queries using conjunction, disjunction, and exclusion, yet it remains unclear whether current retrieval paradigms genuinely satisfy such constraints or exploit `semantic shortcuts'. We conduct a reproducibility study to benchmark major retrieval families and reasoning-targeted methods on QUEST and QUEST+Variants, and introduce LIMIT+, a controlled benchmark where relevance depends on arbitrary attribute predicates and constraint satisfaction, and less on pretrained knowledge. Our findings show that (i) on QUEST, the best neural retrievers achieve an effectiveness that is more than double what can be achieved with BM25 (Recall@100 ${>}$0.41 vs.\\ 0.20), but reasoning-targeted methods like ReasonIR and Search-R1 do not outperform general-purpose retrievers uniformly; (ii) on LIMIT+, gains fail to transfer, where the strongest QUEST method collapses from Recall@100${\\approx}$0.42 to below 0.02, while classic lexical retrieval gains to ${\\sim}$0.96. Lastly, (iii) stratifying by compositional depth reveals a consistent degradation across all methods, where algebraic sparse and lexical methods show more stable performance while dense approaches collapse. We release code and LIMIT+ data generation scripts to support future reproducibility and controlled evaluation.","url_abs":"https://arxiv.org/abs/2605.03824","url_pdf":"https://arxiv.org/pdf/2605.03824","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2605.03824","atlas_url":"https://app.syntology.ai/?focus=2605.03824","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2605.03824"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/informagi/Complex-Set-Compositional-IR","reach":null}],"summary":{"ran":21,"unverified":2},"by_repo_kind":{"found_in_text":{"samples":23,"ran":21,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":23,"samples":[{"code_sha256_prefix":"f09376c27ba31691","entry":"auto_tune_batch_size","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/baselines/contriever/contriever.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/baselines/contriever/contriever.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f09376c27ba31691"}},{"code_sha256_prefix":"fc4a9040a404c25d","entry":"build_limit_plus_qrels","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/figure_constraint_depth.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/figure_constraint_depth.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc4a9040a404c25d"}},{"code_sha256_prefix":"afd079a38a6018fe","entry":"check_has_ids","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/preprocess_qrels.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/preprocess_qrels.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"afd079a38a6018fe"}},{"code_sha256_prefix":"cb72de2a79da99b8","entry":"content_hash","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/baselines/contriever/contriever.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/baselines/contriever/contriever.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cb72de2a79da99b8"}},{"code_sha256_prefix":"c5d20d2b2403e0c0","entry":"convert_example","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/data_generation_utils/convert_quest_variants_to_standard.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/data_generation_utils/convert_quest_variants_to_standard.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c5d20d2b2403e0c0"}},{"code_sha256_prefix":"593136fac381e340","entry":"convert_run_to_trec_jsonl","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/prepare_logicol_for_trec.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/prepare_logicol_for_trec.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"593136fac381e340"}},{"code_sha256_prefix":"db03e4f4d7414a30","entry":"create_qrels_trec","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/prepare_logicol_for_trec.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/prepare_logicol_for_trec.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"db03e4f4d7414a30"}},{"code_sha256_prefix":"f3eaa6a2a6acbc45","entry":"estimate_tokens","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/cost_estimates/estimate_reranking_costs.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/cost_estimates/estimate_reranking_costs.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f3eaa6a2a6acbc45"}},{"code_sha256_prefix":"e15795cac8bdd049","entry":"get_openrouter_pricing","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/cost_estimates/estimate_reranking_costs.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/cost_estimates/estimate_reranking_costs.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e15795cac8bdd049"}},{"code_sha256_prefix":"b94cf68d51b65c8b","entry":"load_corpus_mappings","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/figure_constraint_depth.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/figure_constraint_depth.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b94cf68d51b65c8b"}},{"code_sha256_prefix":"a5175d986e184179","entry":"load_jsonl","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/cost_estimates/estimate_reranking_costs.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/cost_estimates/estimate_reranking_costs.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a5175d986e184179"}},{"code_sha256_prefix":"3769c42b408e1d80","entry":"load_qrels","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/evaluate.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3769c42b408e1d80"}},{"code_sha256_prefix":"46b5d2f743b4de45","entry":"load_qrels_from_test_file","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/evaluate.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"46b5d2f743b4de45"}},{"code_sha256_prefix":"d8f8a63c59cd87f0","entry":"load_qrels_jsonl","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/evaluate_jsonl.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/evaluate_jsonl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d8f8a63c59cd87f0"}},{"code_sha256_prefix":"30566d92e38958fc","entry":"load_qrels_trec","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/figure_constraint_depth.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/figure_constraint_depth.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"30566d92e38958fc"}},{"code_sha256_prefix":"889285baef13a6a3","entry":"load_query_ids","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/preprocess_run.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/preprocess_run.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"889285baef13a6a3"}},{"code_sha256_prefix":"8775a0078734eb98","entry":"load_run_jsonl","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/evaluate_jsonl.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/evaluate_jsonl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8775a0078734eb98"}},{"code_sha256_prefix":"d99a1c56838fe7cc","entry":"mean_pooling","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/baselines/contriever/contriever.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/baselines/contriever/contriever.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d99a1c56838fe7cc"}},{"code_sha256_prefix":"19e979c48a1cbdc5","entry":"parse_jsonl_line","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/preprocess_run.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/preprocess_run.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"19e979c48a1cbdc5"}},{"code_sha256_prefix":"ac467553ebf1f5a6","entry":"parse_reasonir_jsonl_line","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/preprocess_run.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/preprocess_run.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ac467553ebf1f5a6"}},{"code_sha256_prefix":"d3229d742e877177","entry":"recall_at_k","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/evaluate_jsonl.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/evaluate_jsonl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d3229d742e877177"}},{"code_sha256_prefix":"8dcdd8a024bf06d6","entry":"load_qrels_jsonl","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/evaluation/evaluate.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/evaluation/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8dcdd8a024bf06d6"}},{"code_sha256_prefix":"c6977a53bbef7a4c","entry":"run_estimation","repo":"informagi/Complex-Set-Compositional-IR","repo_kind":"found_in_text","path":"code/cost_estimates/compare_model_costs.py","file_url":"https://github.com/informagi/Complex-Set-Compositional-IR/blob/HEAD/code/cost_estimates/compare_model_costs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c6977a53bbef7a4c"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}