We present a novel machine learning based algorithm extending the interaction space around mobile devices. The technique uses only the RGB camera now commonplace on off-the-shelf mobile devices. Our algorithm robustly recognizes a wide range of in-air gestures, supporting user variation, and varying lighting conditions. We demonstrate that our algorithm runs in real-time on unmodified mobile devices, including resource-constrained smartphones and smartwatches. Our goal is not to replace the touchscreen as primary input device, but rather to augment and enrich the existing interaction vocabulary using gestures. While touch input works well for many scenarios, we demonstrate numerous interaction tasks such as mode switches, application and task management, menu selection and certain types of navigation, where such input can be either complemented or better served by in-air gestures. This removes screen real-estate issues on small touchscreens, and allows input to be expanded to the 3D space around the device. We present results for recognition accuracy (93% test and 98% train), impact of memory footprint and other model parameters. Finally, we report results from preliminary user evaluations, discuss advantages and limitations and conclude with directions for future work.
%0 Conference Paper
%1 song2014inair
%A Song, Jie
%A Sörös, Gábor
%A Pece, Fabrizio
%A Fanello, Sean Ryan
%A Izadi, Shahram
%A Keskin, Cem
%A Hilliges, Otmar
%B Proceedings of the 27th Annual ACM Symposium on User Interface Software and Technology
%C New York, NY, USA
%D 2014
%I Association for Computing Machinery (ACM)
%K gesture-recognition hci mobile-computing mobile-gesture mobile-interaction random-forest real
%P 319-329
%R 10.1145/2642918.2647373
%T In-Air Gestures around Unmodified Mobile Devices
%U https://doi.org/10.1145/2642918.2647373
%X We present a novel machine learning based algorithm extending the interaction space around mobile devices. The technique uses only the RGB camera now commonplace on off-the-shelf mobile devices. Our algorithm robustly recognizes a wide range of in-air gestures, supporting user variation, and varying lighting conditions. We demonstrate that our algorithm runs in real-time on unmodified mobile devices, including resource-constrained smartphones and smartwatches. Our goal is not to replace the touchscreen as primary input device, but rather to augment and enrich the existing interaction vocabulary using gestures. While touch input works well for many scenarios, we demonstrate numerous interaction tasks such as mode switches, application and task management, menu selection and certain types of navigation, where such input can be either complemented or better served by in-air gestures. This removes screen real-estate issues on small touchscreens, and allows input to be expanded to the 3D space around the device. We present results for recognition accuracy (93% test and 98% train), impact of memory footprint and other model parameters. Finally, we report results from preliminary user evaluations, discuss advantages and limitations and conclude with directions for future work.
%@ 9781450330695
@inproceedings{song2014inair,
abstract = {We present a novel machine learning based algorithm extending the interaction space around mobile devices. The technique uses only the RGB camera now commonplace on off-the-shelf mobile devices. Our algorithm robustly recognizes a wide range of in-air gestures, supporting user variation, and varying lighting conditions. We demonstrate that our algorithm runs in real-time on unmodified mobile devices, including resource-constrained smartphones and smartwatches. Our goal is not to replace the touchscreen as primary input device, but rather to augment and enrich the existing interaction vocabulary using gestures. While touch input works well for many scenarios, we demonstrate numerous interaction tasks such as mode switches, application and task management, menu selection and certain types of navigation, where such input can be either complemented or better served by in-air gestures. This removes screen real-estate issues on small touchscreens, and allows input to be expanded to the 3D space around the device. We present results for recognition accuracy (93% test and 98% train), impact of memory footprint and other model parameters. Finally, we report results from preliminary user evaluations, discuss advantages and limitations and conclude with directions for future work.},
added-at = {2019-11-14T08:00:27.000+0100},
address = {New York, NY, USA},
author = {Song, Jie and Sörös, Gábor and Pece, Fabrizio and Fanello, Sean Ryan and Izadi, Shahram and Keskin, Cem and Hilliges, Otmar},
biburl = {https://www.bibsonomy.org/bibtex/2b1349b99a414e4fb62b62855f6a585f0/jpmor},
booktitle = {Proceedings of the 27th Annual ACM Symposium on User Interface Software and Technology},
doi = {10.1145/2642918.2647373},
interhash = {2e7a3978a336bd9f7cbb27bdf662e866},
intrahash = {b1349b99a414e4fb62b62855f6a585f0},
isbn = {9781450330695},
keywords = {gesture-recognition hci mobile-computing mobile-gesture mobile-interaction random-forest real},
language = {English},
location = {Honolulu, Hawaii, USA},
numpages = {11},
pages = {319-329},
publisher = {Association for Computing Machinery (ACM)},
school = {Eidgenössische Technische Hochschule Zürich (ETH Zurich)},
series = {UIST ’14},
timestamp = {2020-10-07T13:36:50.000+0200},
title = {In-Air Gestures around Unmodified Mobile Devices},
url = {https://doi.org/10.1145/2642918.2647373},
year = 2014
}