{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/frontiers-in-intelligent-colonoscopy","title":"Frontiers in Intelligent Colonoscopy","arxiv_id":"2410.17241","date":"2024-10-22","proceeding":null,"authors":["Ge-Peng Ji","Jingyi Liu","Peng Xu","Nick Barnes","Fahad Shahbaz Khan","Salman Khan","Deng-Ping Fan"],"abstract":"Colonoscopy is currently one of the most sensitive screening methods for colorectal cancer. This study investigates the frontiers of intelligent colonoscopy techniques and their prospective implications for multimodal medical applications. With this goal, we begin by assessing the current data-centric and model-centric landscapes through four tasks for colonoscopic scene perception, including classification, detection, segmentation, and vision-language understanding. This assessment enables us to identify domain-specific challenges and reveals that multimodal research in colonoscopy remains open for further exploration. To embrace the coming multimodal era, we establish three foundational initiatives: a large-scale multimodal instruction tuning dataset ColonINST, a colonoscopy-designed multimodal language model ColonGPT, and a multimodal benchmark. To facilitate ongoing monitoring of this rapidly evolving field, we provide a public website for the latest updates: https://github.com/ai4colonoscopy/IntelliScope.","url_abs":"https://arxiv.org/abs/2410.17241v2","url_pdf":"https://arxiv.org/pdf/2410.17241v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"frontiers-in-intelligent-colonoscopy","repo_url":"https://github.com/ai4colonoscopy/intelliscope","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"referring-expression-comprehension","task_name":"Referring Expression Comprehension"},{"task_slug":"referring-expression-generation","task_name":"Referring expression generation"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[{"slug":"coloninst-v1","name":"ColonINST-v1","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/image-classification-on-coloninst-v1-seen","task":"Image Classification","dataset":"ColonINST-v1 (Seen)","model":"ColonGPT (w/ LoRA, w/o extra data)","rank_in_archive_order":1,"of":17,"metrics":{"Accuray":"94.06"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-coloninst-v1-unseen","task":"Image Classification","dataset":"ColonINST-v1 (Unseen)","model":"ColonGPT (w/ LoRA, w/o extra data)","rank_in_archive_order":1,"of":17,"metrics":{"Accuray":"83.24"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-generation-on-coloninst","task":"Referring expression generation","dataset":"ColonINST-v1 (Seen)","model":"ColonGPT (w/ LoRA, w/o extra data)","rank_in_archive_order":1,"of":17,"metrics":{"Accuray":"99.96"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-generation-on-coloninst-1","task":"Referring expression generation","dataset":"ColonINST-v1 (Unseen)","model":"ColonGPT (w/ LoRA, w/o extra data)","rank_in_archive_order":1,"of":17,"metrics":{"Accuray":"80.18"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.17241","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.17241"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ai4colonoscopy/intelliscope","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":6},"by_repo_kind":{"official":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9f86dd0b9bd9a73e","entry":"build_vision_projector","repo":"ai4colonoscopy/intelliscope","repo_kind":"official","path":"colongpt/model/multimodal_projector/builder.py","file_url":"https://github.com/ai4colonoscopy/intelliscope/blob/HEAD/colongpt/model/multimodal_projector/builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9f86dd0b9bd9a73e"}},{"code_sha256_prefix":"6c1039e6346f9d13","entry":"build_vision_tower","repo":"ai4colonoscopy/intelliscope","repo_kind":"official","path":"colongpt/model/multimodal_encoder/builder.py","file_url":"https://github.com/ai4colonoscopy/intelliscope/blob/HEAD/colongpt/model/multimodal_encoder/builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6c1039e6346f9d13"}},{"code_sha256_prefix":"e8365288ac1484fa","entry":"interpolate_eva","repo":"ai4colonoscopy/intelliscope","repo_kind":"official","path":"colongpt/model/multimodal_encoder/interpolator.py","file_url":"https://github.com/ai4colonoscopy/intelliscope/blob/HEAD/colongpt/model/multimodal_encoder/interpolator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e8365288ac1484fa"}},{"code_sha256_prefix":"5f51e549b1086405","entry":"interpolate_pos_encoding","repo":"ai4colonoscopy/intelliscope","repo_kind":"official","path":"colongpt/model/multimodal_encoder/interpolator.py","file_url":"https://github.com/ai4colonoscopy/intelliscope/blob/HEAD/colongpt/model/multimodal_encoder/interpolator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5f51e549b1086405"}},{"code_sha256_prefix":"d4f3b60d80240330","entry":"interpolate_pos_encoding_clip","repo":"ai4colonoscopy/intelliscope","repo_kind":"official","path":"colongpt/model/multimodal_encoder/interpolator.py","file_url":"https://github.com/ai4colonoscopy/intelliscope/blob/HEAD/colongpt/model/multimodal_encoder/interpolator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d4f3b60d80240330"}},{"code_sha256_prefix":"16c61629faac8429","entry":"load_pretrained_model","repo":"ai4colonoscopy/intelliscope","repo_kind":"official","path":"colongpt/model/builder.py","file_url":"https://github.com/ai4colonoscopy/intelliscope/blob/HEAD/colongpt/model/builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"16c61629faac8429"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}