%0 Conference Paper %1 YaoSze14 %A Yao, H. %A Szepesvári, Cs. %A Pires, B.A. %A Zhang, X. %B IEEE ADPRL %D 2014 %K Decision Markov Processes, RL, abstraction, approximation, control control, factored function learning, linear model-based models, planning, pseudo-MDPs reinforcement %P 189--197 %T Pseudo-MDPs and Factored Linear Action Models %X In this paper we introduce the concept of pseudo-MDPs to develop abstractions. Pseudo-MDPs relax the requirement that the transition kernel has to be a probability kernel. We show that the new framework captures many existing abstractions. We also introduce the concept of factored linear action models; a special case. Again, the relation of factored linear action models and existing works are discussed. We use the general framework to develop a theory for bounding the suboptimality of policies derived from pseudo-MDPs. Specializing the framework, we recover existing results. We give a least-squares approach and a constrained optimization approach of learning the factored linear model as well as efficient computation methods. We demonstrate that the constrained optimization approach gives better performance than the least-squares approach with normalization.

@inproceedings{YaoSze14, abstract = {In this paper we introduce the concept of pseudo-MDPs to develop abstractions. Pseudo-MDPs relax the requirement that the transition kernel has to be a probability kernel. We show that the new framework captures many existing abstractions. We also introduce the concept of factored linear action models; a special case. Again, the relation of factored linear action models and existing works are discussed. We use the general framework to develop a theory for bounding the suboptimality of policies derived from pseudo-MDPs. Specializing the framework, we recover existing results. We give a least-squares approach and a constrained optimization approach of learning the factored linear model as well as efficient computation methods. We demonstrate that the constrained optimization approach gives better performance than the least-squares approach with normalization. }, added-at = {2020-03-17T03:03:01.000+0100}, author = {Yao, H. and Szepesv{\'a}ri, {Cs}. and Pires, B.A. and Zhang, X.}, biburl = {https://www.bibsonomy.org/bibtex/27571ce0aed906af79b5c01b9b4030ce2/csaba}, booktitle = {IEEE ADPRL}, date-added = {2014-10-11 19:26:42 -0600}, date-modified = {2016-05-09 08:36:59 +0000}, interhash = {e0b23b76f61735e84053d94dcf94f78e}, intrahash = {7571ce0aed906af79b5c01b9b4030ce2}, keywords = {Decision Markov Processes, RL, abstraction, approximation, control control, factored function learning, linear model-based models, planning, pseudo-MDPs reinforcement}, month = {October}, pages = {189--197}, pdf = {papers/ieee_adprl2014.pdf}, timestamp = {2020-03-17T03:03:01.000+0100}, title = {Pseudo-MDPs and Factored Linear Action Models}, year = 2014 }