{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/newsinterview-a-dataset-and-a-playground-to","title":"NewsInterview: a Dataset and a Playground to Evaluate LLMs' Ground Gap via Informational Interviews","arxiv_id":"2411.13779","date":"2024-11-21","proceeding":null,"authors":["Michael Lu","Hyundong Justin Cho","Weiyan Shi","Jonathan May","Alexander Spangher"],"abstract":"Large Language Models (LLMs) have demonstrated impressive capabilities in generating coherent text but often struggle with grounding language and strategic dialogue. To address this gap, we focus on journalistic interviews, a domain rich in grounding communication and abundant in data. We curate a dataset of 40,000 two-person informational interviews from NPR and CNN, and reveal that LLMs are significantly less likely than human interviewers to use acknowledgements and to pivot to higher-level questions. Realizing that a fundamental deficit exists in multi-turn planning and strategic thinking, we develop a realistic simulated environment, incorporating source personas and persuasive elements, in order to facilitate the development of agents with longer-horizon rewards. Our experiments show that while source LLMs mimic human behavior in information sharing, interviewer LLMs struggle with recognizing when questions are answered and engaging persuasively, leading to suboptimal information extraction across model size and capability. These findings underscore the need for enhancing LLMs' strategic dialogue capabilities.","url_abs":"https://arxiv.org/abs/2411.13779v1","url_pdf":"https://arxiv.org/pdf/2411.13779v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"newsinterview-a-dataset-and-a-playground-to","repo_url":"https://github.com/alex2awesome/news-interview-question-generation","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2411.13779","atlas_url":"https://app.syntology.ai/?focus=2411.13779","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.13779"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/alex2awesome/news-interview-question-generation","reach":{"status":"ok"}}],"summary":{"unverified":12},"by_repo_kind":{"official":{"samples":12,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"0bc47936273feb09","entry":"LLM_question_gen_prompt_loader","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"LLM_question_generation.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/LLM_question_generation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0bc47936273feb09"}},{"code_sha256_prefix":"366ee473e7d2d52e","entry":"OUTLINE_LLM_QGen_prompt_loader","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"LLM_question_generation.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/LLM_question_generation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"366ee473e7d2d52e"}},{"code_sha256_prefix":"ba8480c986cf277b","entry":"count_qa_pairs","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"data_processing/create_task_dataset.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/data_processing/create_task_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ba8480c986cf277b"}},{"code_sha256_prefix":"e3670effbb518800","entry":"downsample_task_dataset","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"data_processing/create_task_dataset.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/data_processing/create_task_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e3670effbb518800"}},{"code_sha256_prefix":"ceb7f5482c32ac95","entry":"ensure_valid_json","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"chain_of_thought/split_outline.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/chain_of_thought/split_outline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ceb7f5482c32ac95"}},{"code_sha256_prefix":"4d18565e184c4383","entry":"extract_interviewer_questions","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"data_processing/gpt_classify_all_questions.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/data_processing/gpt_classify_all_questions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4d18565e184c4383"}},{"code_sha256_prefix":"8677e6c7cef658b5","entry":"extract_interviewer_questions","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"data_processing/vllm_classify_all_questions.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/data_processing/vllm_classify_all_questions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8677e6c7cef658b5"}},{"code_sha256_prefix":"4a69fb1616f573d8","entry":"get_classify_all_questions_taxonomy_prompt","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"prompts.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/prompts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4a69fb1616f573d8"}},{"code_sha256_prefix":"7e16649719a6765e","entry":"get_classify_taxonomy_prompt","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"prompts.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/prompts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7e16649719a6765e"}},{"code_sha256_prefix":"68658280a96d53a0","entry":"get_consistency_eval_prompt","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"prompts.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/prompts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"68658280a96d53a0"}},{"code_sha256_prefix":"bd9b8e1dc1b004f1","entry":"get_outline_parts","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"chain_of_thought/outline_only_question_gen.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/chain_of_thought/outline_only_question_gen.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bd9b8e1dc1b004f1"}},{"code_sha256_prefix":"d4f7ffbe4d1b5250","entry":"initialize_tokenizer","repo":"alex2awesome/news-interview-question-generation","repo_kind":"official","path":"helper_functions.py","file_url":"https://github.com/alex2awesome/news-interview-question-generation/blob/HEAD/helper_functions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d4f7ffbe4d1b5250"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}