{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/language-models-are-implicitly-continuous","title":"Language Models Are Implicitly Continuous","arxiv_id":"2504.03933","date":"2025-04-04","proceeding":null,"authors":["Samuele Marro","Davide Evangelista","X. Angelo Huang","Emanuele La Malfa","Michele Lombardi","Michael Wooldridge"],"abstract":"Language is typically modelled with discrete sequences. However, the most successful approaches to language modelling, namely neural networks, are continuous and smooth function approximators. In this work, we show that Transformer-based language models implicitly learn to represent sentences as continuous-time functions defined over a continuous input space. This phenomenon occurs in most state-of-the-art Large Language Models (LLMs), including Llama2, Llama3, Phi3, Gemma, Gemma2, and Mistral, and suggests that LLMs reason about language in ways that fundamentally differ from humans. Our work formally extends Transformers to capture the nuances of time and space continuity in both input and output space. Our results challenge the traditional interpretation of how LLMs understand language, with several linguistic and engineering implications.","url_abs":"https://arxiv.org/abs/2504.03933v1","url_pdf":"https://arxiv.org/pdf/2504.03933v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"language-models-are-implicitly-continuous","repo_url":"https://github.com/samuelemarro/continuous-llm-experiments","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2504.03933","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.03933"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/samuelemarro/continuous-llm-experiments","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":2,"unverified":14},"by_repo_kind":{"official":{"samples":16,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":16,"samples":[{"code_sha256_prefix":"0dd0c5bdb6aa92c9","entry":"get_models","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/pr_slow_ci_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/pr_slow_ci_models.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"0dd0c5bdb6aa92c9"}},{"code_sha256_prefix":"17e5d8c36cc6bce8","entry":"parse_commit_message","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/pr_slow_ci_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/pr_slow_ci_models.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"17e5d8c36cc6bce8"}},{"code_sha256_prefix":"ab306165fa74656c","entry":"build_tip_message","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/deprecate_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/deprecate_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ab306165fa74656c"}},{"code_sha256_prefix":"54b7273383fcfc2d","entry":"extract_model_info","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/deprecate_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/deprecate_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"54b7273383fcfc2d"}},{"code_sha256_prefix":"b176efb26f72e507","entry":"get_architectures_from_config_class","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/create_dummy_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/create_dummy_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b176efb26f72e507"}},{"code_sha256_prefix":"a4f9d3e88bf20c25","entry":"get_clean_message_list","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"src/transformers/agents/llm_engine.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/src/transformers/agents/llm_engine.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"a4f9d3e88bf20c25"}},{"code_sha256_prefix":"3d999b95009c7537","entry":"get_config_class_from_processor_class","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/create_dummy_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/create_dummy_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"3d999b95009c7537"}},{"code_sha256_prefix":"f8faa1cf13e05f5e","entry":"get_continuity_mask","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"continuity.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/continuity.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"f8faa1cf13e05f5e"}},{"code_sha256_prefix":"938fc7c2422ef91d","entry":"get_list_of_repo_model_paths","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/models_to_deprecate.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/models_to_deprecate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"938fc7c2422ef91d"}},{"code_sha256_prefix":"051d790d9b197d9d","entry":"get_model_doc_path","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/deprecate_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/deprecate_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"051d790d9b197d9d"}},{"code_sha256_prefix":"93ccbb68485b6350","entry":"get_module_source_from_name","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/diff_model_converter.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/diff_model_converter.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"93ccbb68485b6350"}},{"code_sha256_prefix":"0f78c7037b35a137","entry":"get_new_python_files_between_commits","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/pr_slow_ci_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/pr_slow_ci_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"0f78c7037b35a137"}},{"code_sha256_prefix":"eb515b8353ac85a9","entry":"get_outputs_from_continuous_inputs","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"continuity.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/continuity.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"eb515b8353ac85a9"}},{"code_sha256_prefix":"0c0b81b081919d8e","entry":"get_processor_types_from_config_class","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils/create_dummy_models.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils/create_dummy_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"0c0b81b081919d8e"}},{"code_sha256_prefix":"db8a84a93fd23422","entry":"get_tracked_tokens","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"db8a84a93fd23422"}},{"code_sha256_prefix":"2f02a58663c23730","entry":"reorder","repo":"samuelemarro/continuous-llm-experiments","repo_kind":"official","path":"utils.py","file_url":"https://github.com/samuelemarro/continuous-llm-experiments/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"2f02a58663c23730"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}