{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/don-t-pay-attention-to-the-noise-learning","title":"Don't Pay Attention to the Noise: Learning Self-supervised Representations of Light Curves with a Denoising Time Series Transformer","arxiv_id":"2207.02777","date":"2022-07-06","proceeding":null,"authors":["Mario Morvan","Nikolaos Nikolaou","Kai Hou Yip","Ingo Waldmann"],"abstract":"Astrophysical light curves are particularly challenging data objects due to the intensity and variety of noise contaminating them. Yet, despite the astronomical volumes of light curves available, the majority of algorithms used to process them are still operating on a per-sample basis. To remedy this, we propose a simple Transformer model -- called Denoising Time Series Transformer (DTST) -- and show that it excels at removing the noise and outliers in datasets of time series when trained with a masked objective, even when no clean targets are available. Moreover, the use of self-attention enables rich and illustrative queries into the learned representations. We present experiments on real stellar light curves from the Transiting Exoplanet Space Satellite (TESS), showing advantages of our approach compared to traditional denoising techniques.","url_abs":"https://arxiv.org/abs/2207.02777v1","url_pdf":"https://arxiv.org/pdf/2207.02777v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"don-t-pay-attention-to-the-noise-learning","repo_url":"https://github.com/mariomorvan/denoising-time-series-transformer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"time-series-1","task_name":"Time Series"},{"task_slug":"time-series","task_name":"Time Series Analysis"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2207.02777","atlas_url":"https://app.syntology.ai/?focus=2207.02777","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.02777"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mariomorvan/denoising-time-series-transformer","reach":null}],"summary":{"ran":10,"ran_honours":2,"ran_draft_wrong":4,"unverified":2},"by_repo_kind":{"official":{"samples":18,"ran":16,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":18,"samples":[{"code_sha256_prefix":"42f0ae0ee495e5d7","entry":"BatchNorm","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"42f0ae0ee495e5d7"}},{"code_sha256_prefix":"043d857ed2708db5","entry":"FixedPosEmbedding","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"043d857ed2708db5"}},{"code_sha256_prefix":"7347e64e7d97d46a","entry":"IQRLoss","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"7347e64e7d97d46a"}},{"code_sha256_prefix":"7c4b76725171d44c","entry":"LearnedPosEmbedding","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"7c4b76725171d44c"}},{"code_sha256_prefix":"f4d0858dcf8f10e0","entry":"MaskedHuberLoss","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"f4d0858dcf8f10e0"}},{"code_sha256_prefix":"ac251bc6c8ffdbfe","entry":"MaskedL1Loss","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ac251bc6c8ffdbfe"}},{"code_sha256_prefix":"3a7ede8147c5dd0e","entry":"MaskedMSELoss","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"3a7ede8147c5dd0e"}},{"code_sha256_prefix":"150c70c525ad502d","entry":"PosEmbedding","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"150c70c525ad502d"}},{"code_sha256_prefix":"e583815367c0ebbc","entry":"TransformerEncoder","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e583815367c0ebbc"}},{"code_sha256_prefix":"82ac7fddfe86c023","entry":"TransformerEncoderLayer","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"82ac7fddfe86c023"}},{"code_sha256_prefix":"1b6804b7cf69d1b7","entry":"detrend","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"1b6804b7cf69d1b7"}},{"code_sha256_prefix":"4372f26d99b2b73d","entry":"estimate_noise","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"4372f26d99b2b73d"}},{"code_sha256_prefix":"ffb7ff09cfc9a52a","entry":"inverse_standardise_batch","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ffb7ff09cfc9a52a"}},{"code_sha256_prefix":"9f5d0dc785c490d3","entry":"iqr","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"9f5d0dc785c490d3"}},{"code_sha256_prefix":"ebed1f18f80743bc","entry":"naniqr","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ebed1f18f80743bc"}},{"code_sha256_prefix":"1bfe235be7e974ba","entry":"nanstd","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"1bfe235be7e974ba"}},{"code_sha256_prefix":"ab58a7f246202a08","entry":"LitImputer","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ab58a7f246202a08"}},{"code_sha256_prefix":"d02ec62220ae4d91","entry":"_MaskedLoss","repo":"mariomorvan/denoising-time-series-transformer","repo_kind":"official","path":"models/models.py","file_url":"https://github.com/mariomorvan/denoising-time-series-transformer/blob/HEAD/models/models.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"d02ec62220ae4d91"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}