%0 Journal Article %1 nobrega2013predicting %A Nóbrega, Caio Santos Bezerra %A Marinho, Leandro Balby %B Proceedings of KDMiLe - Symposium on Knowledge Discovery, Mining and Learning, ISSN 2318-1060 %D 2013 %K factorization learning matrix myown rate recommender %T Predicting the Learning Rate of Gradient Descent for Accelerating Matrix Factorization %U http://www.dsc.ufcg.edu.br/~lbmarinho/homepage/pub/nobrega_kdmile_2013.pdf %X Matrix Factorization (MF) is the predominant technique in recommender systems. The model parameters are usually learned by means of numerical methods, such as gradient descent. The learning rate of gradient descent is typically set to lower values in order to ensure that the algorithm does not miss a local optimum. As a consequence, the algorithm may take several iterations to converge, which ends up increasing the computational costs of the training phase. Ideally, one wants to find the learning rate that leads to a local optimum in one iteration, but that is very difficult to achieve given the high complexity of the search space. However, if in one shot one gets close enough to the local optimum, one will need only a few iterations until convergence, hence saving iterations and speeding up the learning process. Starting with an exploratory analysis on several recommender systems datasets, we observed that there is an overall linear relationship between the learning rate and the number of iterations needed until convergence. Another key observation is that this relationship holds across the different datasets, with only minor variations. From this, we propose to use simple linear regression models for predicting, for an unknown dataset, which learning rate leads to the minimum number of iterations. We show that, for some datasets, we can reduce the number of iterations up to 50% when compared to the traditional approach.

@article{nobrega2013predicting, abstract = {Matrix Factorization (MF) is the predominant technique in recommender systems. The model parameters are usually learned by means of numerical methods, such as gradient descent. The learning rate of gradient descent is typically set to lower values in order to ensure that the algorithm does not miss a local optimum. As a consequence, the algorithm may take several iterations to converge, which ends up increasing the computational costs of the training phase. Ideally, one wants to find the learning rate that leads to a local optimum in one iteration, but that is very difficult to achieve given the high complexity of the search space. However, if in one shot one gets close enough to the local optimum, one will need only a few iterations until convergence, hence saving iterations and speeding up the learning process. Starting with an exploratory analysis on several recommender systems datasets, we observed that there is an overall linear relationship between the learning rate and the number of iterations needed until convergence. Another key observation is that this relationship holds across the different datasets, with only minor variations. From this, we propose to use simple linear regression models for predicting, for an unknown dataset, which learning rate leads to the minimum number of iterations. We show that, for some datasets, we can reduce the number of iterations up to 50% when compared to the traditional approach.}, added-at = {2013-09-13T19:58:20.000+0200}, author = {Nóbrega, Caio Santos Bezerra and Marinho, Leandro Balby}, biburl = {https://www.bibsonomy.org/bibtex/2edecbca2bce28bf809520dd81f170946/lbalby}, booktitle = {Proceedings of KDMiLe - Symposium on Knowledge Discovery, Mining and Learning, ISSN 2318-1060}, interhash = {da3361017c2e46e8c49b1698dd2055eb}, intrahash = {edecbca2bce28bf809520dd81f170946}, keywords = {factorization learning matrix myown rate recommender}, timestamp = {2014-11-02T13:53:23.000+0100}, title = {Predicting the Learning Rate of Gradient Descent for Accelerating Matrix Factorization}, url = {http://www.dsc.ufcg.edu.br/~lbmarinho/homepage/pub/nobrega_kdmile_2013.pdf}, year = 2013 }