There is a large variety of objects and appliances in human environments, such as stoves, coffee dispensers, juice extractors, and so on. It is challenging for a roboticist to program a robot for each of these object types and for each of their instantiations. In this work, we present a novel approach to manipulation planning based on the idea that many household objects share similarly-operated object parts. We formulate the manipulation planning as a structured prediction problem and design a deep learning model that can handle large noise in the manipulation demonstrations and learns features from three different modalities: point-clouds, language and trajectory. In order to collect a large number of manipulation demonstrations for different objects, we developed a new crowd-sourcing platform called Robobarista. We test our model on our dataset consisting of 116 objects with 249 parts along with 250 language instructions, for which there are 1225 crowd-sourced manipulation demonstrations. We further show that our robot can even manipulate objects it has never seen before.
Description
Robobarista: Object Part Based Transfer of Manipulation Trajectories from Crowd-Sourcing in 3D Pointclouds | SpringerLink
%0 Book Section
%1 Sung2018
%A Sung, Jaeyong
%A Jin, Seok Hyun
%A Saxena, Ashutosh
%B Robotics Research: Volume 2
%C Cham
%D 2018
%E Bicchi, Antonio
%E Burgard, Wolfram
%I Springer International Publishing
%K TODO design paper robotics
%P 701--720
%R 10.1007/978-3-319-60916-4_40
%T Robobarista: Object Part Based Transfer of Manipulation Trajectories from Crowd-Sourcing in 3D Pointclouds
%U https://doi.org/10.1007/978-3-319-60916-4_40
%X There is a large variety of objects and appliances in human environments, such as stoves, coffee dispensers, juice extractors, and so on. It is challenging for a roboticist to program a robot for each of these object types and for each of their instantiations. In this work, we present a novel approach to manipulation planning based on the idea that many household objects share similarly-operated object parts. We formulate the manipulation planning as a structured prediction problem and design a deep learning model that can handle large noise in the manipulation demonstrations and learns features from three different modalities: point-clouds, language and trajectory. In order to collect a large number of manipulation demonstrations for different objects, we developed a new crowd-sourcing platform called Robobarista. We test our model on our dataset consisting of 116 objects with 249 parts along with 250 language instructions, for which there are 1225 crowd-sourced manipulation demonstrations. We further show that our robot can even manipulate objects it has never seen before.
%@ 978-3-319-60916-4
@inbook{Sung2018,
abstract = {There is a large variety of objects and appliances in human environments, such as stoves, coffee dispensers, juice extractors, and so on. It is challenging for a roboticist to program a robot for each of these object types and for each of their instantiations. In this work, we present a novel approach to manipulation planning based on the idea that many household objects share similarly-operated object parts. We formulate the manipulation planning as a structured prediction problem and design a deep learning model that can handle large noise in the manipulation demonstrations and learns features from three different modalities: point-clouds, language and trajectory. In order to collect a large number of manipulation demonstrations for different objects, we developed a new crowd-sourcing platform called Robobarista. We test our model on our dataset consisting of 116 objects with 249 parts along with 250 language instructions, for which there are 1225 crowd-sourced manipulation demonstrations. We further show that our robot can even manipulate objects it has never seen before.},
added-at = {2018-05-02T19:59:58.000+0200},
address = {Cham},
author = {Sung, Jaeyong and Jin, Seok Hyun and Saxena, Ashutosh},
biburl = {https://www.bibsonomy.org/bibtex/2fe284aca953f978d89e262b781eb2c03/achakraborty},
booktitle = {Robotics Research: Volume 2},
description = {Robobarista: Object Part Based Transfer of Manipulation Trajectories from Crowd-Sourcing in 3D Pointclouds | SpringerLink},
doi = {10.1007/978-3-319-60916-4_40},
editor = {Bicchi, Antonio and Burgard, Wolfram},
interhash = {67a37a3a97940cf14918af818fd50108},
intrahash = {fe284aca953f978d89e262b781eb2c03},
isbn = {978-3-319-60916-4},
keywords = {TODO design paper robotics},
pages = {701--720},
publisher = {Springer International Publishing},
timestamp = {2018-05-02T19:59:58.000+0200},
title = {Robobarista: Object Part Based Transfer of Manipulation Trajectories from Crowd-Sourcing in 3D Pointclouds},
url = {https://doi.org/10.1007/978-3-319-60916-4_40},
year = 2018
}