Excessive reuse of test data has become commonplace in today's machine
learning workflows. Popular benchmarks, competitions, industrial scale tuning,
among other applications, all involve test data reuse beyond guidance by
statistical confidence bounds. Nonetheless, recent replication studies give
evidence that popular benchmarks continue to support progress despite years of
extensive reuse. We proffer a new explanation for the apparent longevity of
test data: Many proposed models are similar in their predictions and we prove
that this similarity mitigates overfitting. Specifically, we show empirically
that models proposed for the ImageNet ILSVRC benchmark agree in their
predictions well beyond what we can conclude from their accuracy levels alone.
Likewise, models created by large scale hyperparameter search enjoy high levels
of similarity. Motivated by these empirical observations, we give a
non-asymptotic generalization bound that takes similarity into account, leading
to meaningful confidence bounds in practical settings.
Description
[1905.12580] Model Similarity Mitigates Test Set Overuse
%0 Journal Article
%1 mania2019model
%A Mania, Horia
%A Miller, John
%A Schmidt, Ludwig
%A Hardt, Moritz
%A Recht, Benjamin
%D 2019
%K bounds deep-learning generalization learning machine-learning mathematics probability stats theory
%T Model Similarity Mitigates Test Set Overuse
%U http://arxiv.org/abs/1905.12580
%X Excessive reuse of test data has become commonplace in today's machine
learning workflows. Popular benchmarks, competitions, industrial scale tuning,
among other applications, all involve test data reuse beyond guidance by
statistical confidence bounds. Nonetheless, recent replication studies give
evidence that popular benchmarks continue to support progress despite years of
extensive reuse. We proffer a new explanation for the apparent longevity of
test data: Many proposed models are similar in their predictions and we prove
that this similarity mitigates overfitting. Specifically, we show empirically
that models proposed for the ImageNet ILSVRC benchmark agree in their
predictions well beyond what we can conclude from their accuracy levels alone.
Likewise, models created by large scale hyperparameter search enjoy high levels
of similarity. Motivated by these empirical observations, we give a
non-asymptotic generalization bound that takes similarity into account, leading
to meaningful confidence bounds in practical settings.
@article{mania2019model,
abstract = {Excessive reuse of test data has become commonplace in today's machine
learning workflows. Popular benchmarks, competitions, industrial scale tuning,
among other applications, all involve test data reuse beyond guidance by
statistical confidence bounds. Nonetheless, recent replication studies give
evidence that popular benchmarks continue to support progress despite years of
extensive reuse. We proffer a new explanation for the apparent longevity of
test data: Many proposed models are similar in their predictions and we prove
that this similarity mitigates overfitting. Specifically, we show empirically
that models proposed for the ImageNet ILSVRC benchmark agree in their
predictions well beyond what we can conclude from their accuracy levels alone.
Likewise, models created by large scale hyperparameter search enjoy high levels
of similarity. Motivated by these empirical observations, we give a
non-asymptotic generalization bound that takes similarity into account, leading
to meaningful confidence bounds in practical settings.},
added-at = {2019-05-31T16:18:51.000+0200},
author = {Mania, Horia and Miller, John and Schmidt, Ludwig and Hardt, Moritz and Recht, Benjamin},
biburl = {https://www.bibsonomy.org/bibtex/2019847494627ebda8a55c086ccb08e0e/kirk86},
description = {[1905.12580] Model Similarity Mitigates Test Set Overuse},
interhash = {7218bdcc61153dcc3508be67be6c5613},
intrahash = {019847494627ebda8a55c086ccb08e0e},
keywords = {bounds deep-learning generalization learning machine-learning mathematics probability stats theory},
note = {cite arxiv:1905.12580Comment: 18 pages, 7 figures},
timestamp = {2019-05-31T16:19:59.000+0200},
title = {Model Similarity Mitigates Test Set Overuse},
url = {http://arxiv.org/abs/1905.12580},
year = 2019
}