{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/gimme-signals-discriminative-signal-encoding","title":"Gimme Signals: Discriminative signal encoding for multimodal activity recognition","arxiv_id":"2003.06156","date":"2020-03-13","proceeding":null,"authors":["Raphael Memmesheimer","Nick Theisen","Dietrich Paulus"],"abstract":"We present a simple, yet effective and flexible method for action recognition supporting multiple sensor modalities. Multivariate signal sequences are encoded in an image and are then classified using a recently proposed EfficientNet CNN architecture. Our focus was to find an approach that generalizes well across different sensor modalities without specific adaptions while still achieving good results. We apply our method to 4 action recognition datasets containing skeleton sequences, inertial and motion capturing measurements as well as \\wifi fingerprints that range up to 120 action classes. Our method defines the current best CNN-based approach on the NTU RGB+D 120 dataset, lifts the state of the art on the ARIL Wi-Fi dataset by +6.78%, improves the UTD-MHAD inertial baseline by +14.4%, the UTD-MHAD skeleton baseline by 1.13% and achieves 96.11% on the Simitate motion capturing data (80/20 split). We further demonstrate experiments on both, modality fusion on a signal level and signal reduction to prevent the representation from overloading.","url_abs":"https://arxiv.org/abs/2003.06156v2","url_pdf":"https://arxiv.org/pdf/2003.06156v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"gimme-signals-discriminative-signal-encoding","repo_url":"https://github.com/airglow/gimme_signals_action_recognition","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"gimme-signals-discriminative-signal-encoding","repo_url":"https://github.com/raphaelmemmesheimer/gimme_signals_action_recognition","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"},{"task_slug":"activity-recognition","task_name":"Activity Recognition"},{"task_slug":"multimodal-activity-recognition","task_name":"Multimodal Activity Recognition"},{"task_slug":"skeleton-based-action-recognition","task_name":"Skeleton Based Action Recognition"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"depthwise-convolution","method_name":"Depthwise Convolution"},{"method_slug":"depthwise-separable-convolution","method_name":"Depthwise Separable Convolution"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"efficientnet","method_name":"EfficientNet"},{"method_slug":"inverted-residual-block","method_name":"Inverted Residual Block"},{"method_slug":"pointwise-convolution","method_name":"Pointwise Convolution"},{"method_slug":"rmsprop","method_name":"RMSProp"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"squeeze-and-excitation-block","method_name":"Squeeze-and-Excitation Block"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/action-recognition-in-videos-on-ntu-rgbd-120","task":"Action Recognition","dataset":"NTU RGB+D 120","model":"Gimme Signals (AIS)","rank_in_archive_order":18,"of":21,"metrics":{"Accuracy (Cross-Setup)":"70.8","Accuracy (Cross-Subject)":"71.59"},"uses_additional_data":false},{"leaderboard":"/sota/multimodal-activity-recognition-on-utd-mhad","task":"Multimodal Activity Recognition","dataset":"UTD-MHAD","model":"Gimme Signals (Skeleton, AIS)","rank_in_archive_order":2,"of":5,"metrics":{"Accuracy (CS)":"93.33"},"uses_additional_data":false},{"leaderboard":"/sota/skeleton-based-action-recognition-on-ntu-rgbd-1","task":"Skeleton Based Action Recognition","dataset":"NTU RGB+D 120","model":"Gimme Signals (Skeleton, AIS)","rank_in_archive_order":66,"of":83,"metrics":{"Accuracy (Cross-Setup)":"71.6%","Accuracy (Cross-Subject)":"70.8%"},"uses_additional_data":false}],"syntology":{"atlas_url":null,"mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}