{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2606-29067","title":"ThinkProbe: Beyond Accuracy -- Structural Profiling of Open-Ended LLM Reasoning Traces via Non-Generative Thought Graphs","arxiv_id":"2606.29067","date":"2026-06-27","proceeding":null,"authors":["Mohamed Amine Kerkouri","Simon D. Hernandez","Marouane Tliba","Yann Dauxais","Maha Ben-Fares","Pierre Holat"],"abstract":"We present ThinkProbe, a framework for structural analysis of LLM reasoning traces. ThinkProbe converts each trace into a Thought Graph a directed graph with cycles, 8 node types, and 6 edge types and derives a 19-metric five-dimensional cognitive profile (5D-CP: Breadth, Depth, Structure, Metacognitive, Efficiency) through a fully non-generative pipeline combining rule-based segmentation and discriminative semantic linking. Applied to 4{,}200 traces from 7 native reasoning models across 200 open-ended questions and 10 cognitive domains, ThinkProbe reveals that reasoning structure is a stable, model-level property: between-model variance exceeds between-domain variance by up to fourfold across four of five cognitive dimensions, with Structure showing genuine sensitivity to question domain, exposing qualitatively distinct cognitive profiles invisible to accuracy-based evaluation.","url_abs":"https://arxiv.org/abs/2606.29067","url_pdf":"https://arxiv.org/pdf/2606.29067","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2606.29067","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2606.29067"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/kmamine/ThinkProb","reach":null}],"summary":{"ran":3,"ran_honours":3,"ran_draft_wrong":6,"ran_fixture":1,"unverified":7},"by_repo_kind":{"found_in_text":{"samples":20,"ran":13,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":20,"samples":[{"code_sha256_prefix":"f5d60d0025f25163","entry":"Edge","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f5d60d0025f25163"}},{"code_sha256_prefix":"c649deae79f41bd9","entry":"ThoughtUnit","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c649deae79f41bd9"}},{"code_sha256_prefix":"4b58280bd7bfc19a","entry":"_RawSpan","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4b58280bd7bfc19a"}},{"code_sha256_prefix":"94360b7199f2fa20","entry":"_approx_tokens","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"94360b7199f2fa20"}},{"code_sha256_prefix":"411e7ee564acf253","entry":"_cosine","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"411e7ee564acf253"}},{"code_sha256_prefix":"e101e443fc5c7881","entry":"_cross_segment_edges","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e101e443fc5c7881"}},{"code_sha256_prefix":"a30e7ab91fa3d7b9","entry":"_entailment_probs","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a30e7ab91fa3d7b9"}},{"code_sha256_prefix":"1f582917ce733756","entry":"_pass2_soft_boundaries","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1f582917ce733756"}},{"code_sha256_prefix":"15a5bd6b0752b51e","entry":"_pass5_nli_refinement","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"15a5bd6b0752b51e"}},{"code_sha256_prefix":"22fe95b38c3374d7","entry":"_semantic_trajectory","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"22fe95b38c3374d7"}},{"code_sha256_prefix":"790e007e6f21a18c","entry":"_soft_boundaries","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"790e007e6f21a18c"}},{"code_sha256_prefix":"90dbe169c5050279","entry":"_split_sentences","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"90dbe169c5050279"}},{"code_sha256_prefix":"9f37ef4dfadd43da","entry":"_structural_split","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9f37ef4dfadd43da"}},{"code_sha256_prefix":"65db80452857f13f","entry":"BoundaryClass","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"65db80452857f13f"}},{"code_sha256_prefix":"d72d957573720f1e","entry":"EdgeType","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d72d957573720f1e"}},{"code_sha256_prefix":"5fe8fe905dcea343","entry":"NodeFamily","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5fe8fe905dcea343"}},{"code_sha256_prefix":"1e2c729aedfe6062","entry":"NodeType","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1e2c729aedfe6062"}},{"code_sha256_prefix":"7c11b5e745766ffe","entry":"_detect_supplementary_class","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7c11b5e745766ffe"}},{"code_sha256_prefix":"07de6172c81f99e7","entry":"_pass1_supplementary","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"07de6172c81f99e7"}},{"code_sha256_prefix":"dfd81d5834829083","entry":"segment","repo":"kmamine/ThinkProb","repo_kind":"found_in_text","path":"src/thinkbench/extract/segmenter.py","file_url":"https://github.com/kmamine/ThinkProb/blob/HEAD/src/thinkbench/extract/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dfd81d5834829083"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":{"kind":"leaderboard_placements","source":"Syntology's leaderboard-shaped extractor over the paper's own arXiv-HTML tables: a model pointed at a cell, the number was read from that cell and checked against the board's metric, dataset, split and scale, and an independent check accepted the entry; not reviewed by the paper's authors or the archive's editors","extractor_model":"global.anthropic.claude-sonnet-4-5-20250929-v1:0","verifier_model":null,"prompt_sha":"fa63d4bb9d755694","coverage":{"sentence":"Syntology has checked 6,264 of the 9,581 papers on this site that are newer than the archive; results from the others appear after they are checked.","papers_newer_than_archive":9581,"papers_checked":6264},"entries":[],"not_placed":{"boards":0,"rejected_by_independent_check":0,"refused_by_a_rule":0,"check_did_not_answer":0,"proposed_without_a_cell":0,"declined_by_site":0}}}