{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/locllm-exploiting-generalizable-human","title":"LocLLM: Exploiting Generalizable Human Keypoint Localization via Large Language Model","arxiv_id":"2406.04659","date":"2024-06-07","proceeding":"CVPR 2024 1","authors":["Dongkai Wang","Shiyu Xuan","Shiliang Zhang"],"abstract":"The capacity of existing human keypoint localization models is limited by keypoint priors provided by the training data. To alleviate this restriction and pursue more general model, this work studies keypoint localization from a different perspective by reasoning locations based on keypiont clues in text descriptions. We propose LocLLM, the first Large-Language Model (LLM) based keypoint localization model that takes images and text instructions as inputs and outputs the desired keypoint coordinates. LocLLM leverages the strong reasoning capability of LLM and clues of keypoint type, location, and relationship in textual descriptions for keypoint localization. To effectively tune LocLLM, we construct localization-based instruction conversations to connect keypoint description with corresponding coordinates in input image, and fine-tune the whole model in a parameter-efficient training pipeline. LocLLM shows remarkable performance on standard 2D/3D keypoint localization benchmarks. Moreover, incorporating language clues into the localization makes LocLLM show superior flexibility and generalizable capability in cross dataset keypoint localization, and even detecting novel type of keypoints unseen during training.","url_abs":"https://arxiv.org/abs/2406.04659v1","url_pdf":"https://arxiv.org/pdf/2406.04659v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"locllm-exploiting-generalizable-human","repo_url":"https://github.com/kennethwdk/LocLLM","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.04659","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.04659"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/kennethwdk/LocLLM","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":2,"ran_violates":1,"ran":4,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"official":{"samples":12,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"83abf1ce90db7100","entry":"affine_transform","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"datasets/h36m.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/datasets/h36m.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"83abf1ce90db7100"}},{"code_sha256_prefix":"4cb732f513d69dfd","entry":"disabled_train","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"models/locllm.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/models/locllm.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4cb732f513d69dfd"}},{"code_sha256_prefix":"88428d9a8f104e23","entry":"flip_joints_3d","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"datasets/h36m.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/datasets/h36m.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"88428d9a8f104e23"}},{"code_sha256_prefix":"f8c01e13c130a2a9","entry":"get_3rd_point","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"datasets/h36m.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/datasets/h36m.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f8c01e13c130a2a9"}},{"code_sha256_prefix":"70696932e5ece0fb","entry":"get_affine_transform","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"datasets/coco.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/datasets/coco.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"DEP_MISSING","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70696932e5ece0fb"}},{"code_sha256_prefix":"e7af7e6c543635d7","entry":"lora_state_dict","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"models/lora.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/models/lora.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7af7e6c543635d7"}},{"code_sha256_prefix":"e7fcb6d9ac9deaf4","entry":"named_apply","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"models/dino.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/models/dino.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7fcb6d9ac9deaf4"}},{"code_sha256_prefix":"74b80552a8a3db3c","entry":"transform_preds","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"datasets/coco.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/datasets/coco.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"74b80552a8a3db3c"}},{"code_sha256_prefix":"5d16d4d4fc573ac7","entry":"add_residual","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"models/dino_layers/block.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/models/dino_layers/block.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5d16d4d4fc573ac7"}},{"code_sha256_prefix":"85f7ffc01945bb72","entry":"drop_add_residual_stochastic_depth","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"models/dino_layers/block.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/models/dino_layers/block.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"85f7ffc01945bb72"}},{"code_sha256_prefix":"d49a474bc36c5234","entry":"fliplr_joints","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"datasets/coco.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/datasets/coco.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d49a474bc36c5234"}},{"code_sha256_prefix":"5be3610fa1fee19e","entry":"get_branges_scales","repo":"kennethwdk/LocLLM","repo_kind":"official","path":"models/dino_layers/block.py","file_url":"https://github.com/kennethwdk/LocLLM/blob/HEAD/models/dino_layers/block.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5be3610fa1fee19e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}