%0 Generic %1 kobak2018optimal %A Kobak, Dmitry %A Lomond, Jonathan %A Sanchez, Benoit %D 2018 %K High dimensionality %T Optimal ridge penalty for real-world high-dimensional data can be zero or negative due to the implicit ridge regularization %U http://arxiv.org/abs/1805.10939 %X A conventional wisdom in statistical learning is that large models require strong regularization to prevent overfitting. Here we show that this rule can be violated by linear regression in the underdetermined $np$ situation under realistic conditions. Using simulations and real-life high-dimensional data sets, we demonstrate that an explicit positive ridge penalty can fail to provide any improvement over the minimum-norm least squares estimator. Moreover, the optimal value of ridge penalty in this situation can be negative. This happens when the high-variance directions in the predictor space can predict the response variable, which is often the case in the real-world high-dimensional data. In this regime, low-variance directions provide an implicit ridge regularization and can make any further positive ridge penalty detrimental. We prove that augmenting any linear model with random covariates and using minimum-norm estimator is asymptotically equivalent to adding the ridge penalty. We use a spiked covariance model as an analytically tractable example and prove that the optimal ridge penalty in this case is negative when $np$.

@misc{kobak2018optimal, abstract = {A conventional wisdom in statistical learning is that large models require strong regularization to prevent overfitting. Here we show that this rule can be violated by linear regression in the underdetermined $n\ll p$ situation under realistic conditions. Using simulations and real-life high-dimensional data sets, we demonstrate that an explicit positive ridge penalty can fail to provide any improvement over the minimum-norm least squares estimator. Moreover, the optimal value of ridge penalty in this situation can be negative. This happens when the high-variance directions in the predictor space can predict the response variable, which is often the case in the real-world high-dimensional data. In this regime, low-variance directions provide an implicit ridge regularization and can make any further positive ridge penalty detrimental. We prove that augmenting any linear model with random covariates and using minimum-norm estimator is asymptotically equivalent to adding the ridge penalty. We use a spiked covariance model as an analytically tractable example and prove that the optimal ridge penalty in this case is negative when $n\ll p$.}, added-at = {2022-01-03T04:58:10.000+0100}, author = {Kobak, Dmitry and Lomond, Jonathan and Sanchez, Benoit}, biburl = {https://www.bibsonomy.org/bibtex/2b736a3cbd27ea011ed0f0e628c4de0a3/sshariat}, description = {Optimal ridge penalty for real-world high-dimensional data can be zero or negative due to the implicit ridge regularization}, interhash = {0e858122ed8d5f09c9a8fe7fb566e3a0}, intrahash = {b736a3cbd27ea011ed0f0e628c4de0a3}, keywords = {High dimensionality}, note = {cite arxiv:1805.10939}, timestamp = {2022-01-03T04:58:10.000+0100}, title = {Optimal ridge penalty for real-world high-dimensional data can be zero or negative due to the implicit ridge regularization}, url = {http://arxiv.org/abs/1805.10939}, year = 2018 }