Generalised linear models for multi-class classification problems are one of
the fundamental building blocks of modern machine learning tasks. In this
manuscript, we characterise the learning of a mixture of $K$ Gaussians with
generic means and covariances via empirical risk minimisation (ERM) with any
convex loss and regularisation. In particular, we prove exact asymptotics
characterising the ERM estimator in high-dimensions, extending several previous
results about Gaussian mixture classification in the literature. We exemplify
our result in two tasks of interest in statistical learning: a) classification
for a mixture with sparse means, where we study the efficiency of $\ell_1$
penalty with respect to $\ell_2$; b) max-margin multi-class classification,
where we characterise the phase transition on the existence of the multi-class
logistic maximum likelihood estimator for $K>2$. Finally, we discuss how our
theory can be applied beyond the scope of synthetic data, showing that in
different cases Gaussian mixtures capture closely the learning curve of
classification tasks in real data sets.
Description
[2106.03791] Learning Gaussian Mixtures with Generalised Linear Models: Precise Asymptotics in High-dimensions
%0 Journal Article
%1 loureiro2021learning
%A Loureiro, Bruno
%A Sicuro, Gabriele
%A Gerbelot, Cédric
%A Pacco, Alessandro
%A Krzakala, Florent
%A Zdeborová, Lenka
%D 2021
%K bayesian mixture-models readings
%T Learning Gaussian Mixtures with Generalised Linear Models: Precise
Asymptotics in High-dimensions
%U http://arxiv.org/abs/2106.03791
%X Generalised linear models for multi-class classification problems are one of
the fundamental building blocks of modern machine learning tasks. In this
manuscript, we characterise the learning of a mixture of $K$ Gaussians with
generic means and covariances via empirical risk minimisation (ERM) with any
convex loss and regularisation. In particular, we prove exact asymptotics
characterising the ERM estimator in high-dimensions, extending several previous
results about Gaussian mixture classification in the literature. We exemplify
our result in two tasks of interest in statistical learning: a) classification
for a mixture with sparse means, where we study the efficiency of $\ell_1$
penalty with respect to $\ell_2$; b) max-margin multi-class classification,
where we characterise the phase transition on the existence of the multi-class
logistic maximum likelihood estimator for $K>2$. Finally, we discuss how our
theory can be applied beyond the scope of synthetic data, showing that in
different cases Gaussian mixtures capture closely the learning curve of
classification tasks in real data sets.
@article{loureiro2021learning,
abstract = {Generalised linear models for multi-class classification problems are one of
the fundamental building blocks of modern machine learning tasks. In this
manuscript, we characterise the learning of a mixture of $K$ Gaussians with
generic means and covariances via empirical risk minimisation (ERM) with any
convex loss and regularisation. In particular, we prove exact asymptotics
characterising the ERM estimator in high-dimensions, extending several previous
results about Gaussian mixture classification in the literature. We exemplify
our result in two tasks of interest in statistical learning: a) classification
for a mixture with sparse means, where we study the efficiency of $\ell_1$
penalty with respect to $\ell_2$; b) max-margin multi-class classification,
where we characterise the phase transition on the existence of the multi-class
logistic maximum likelihood estimator for $K>2$. Finally, we discuss how our
theory can be applied beyond the scope of synthetic data, showing that in
different cases Gaussian mixtures capture closely the learning curve of
classification tasks in real data sets.},
added-at = {2021-06-08T15:31:12.000+0200},
author = {Loureiro, Bruno and Sicuro, Gabriele and Gerbelot, Cédric and Pacco, Alessandro and Krzakala, Florent and Zdeborová, Lenka},
biburl = {https://www.bibsonomy.org/bibtex/2ebb4535049746e8841307454212b443c/kirk86},
description = {[2106.03791] Learning Gaussian Mixtures with Generalised Linear Models: Precise Asymptotics in High-dimensions},
interhash = {4e214b30aa0cd41ce447a9f5acad4e83},
intrahash = {ebb4535049746e8841307454212b443c},
keywords = {bayesian mixture-models readings},
note = {cite arxiv:2106.03791Comment: 12 pages + 34 pages of Appendix, 10 figures},
timestamp = {2021-06-08T15:31:12.000+0200},
title = {Learning Gaussian Mixtures with Generalised Linear Models: Precise
Asymptotics in High-dimensions},
url = {http://arxiv.org/abs/2106.03791},
year = 2021
}