Detecting out-of-distribution examples is important for safety-critical machine learning applications such as detecting novel biological phenomena and self-driving cars. However, existing research mainly focuses on simple small-scale settings. To set the stage for more realistic out-of-distribution detection, we depart from small-scale settings and explore large-scale multiclass and multi-label settings with high-resolution images and thousands of classes. To make future work in real-world settings possible, we create new benchmarks for three large-scale settings. To test ImageNet multiclass anomaly detectors, we introduce the Species dataset containing over 700,000 images and over a thousand anomalous species. We leverage ImageNet-21K to evaluate PASCAL VOC and COCO multilabel anomaly detectors. Third, we introduce a new benchmark for anomaly segmentation by introducing a segmentation benchmark with road anomalies. We conduct extensive experiments in these more realistic settings for out-of-distribution detection and find that a surprisingly simple detector based on the maximum logit outperforms prior methods in all the large-scale multi-class, multi-label, and segmentation tasks, establishing a simple new baseline for future work.
%0 Conference Paper
%1 pmlr-v162-hendrycks22a_MLS
%A Hendrycks, Dan
%A Basart, Steven
%A Mazeika, Mantas
%A Zou, Andy
%A Kwon, Joseph
%A Mostajabi, Mohammadreza
%A Steinhardt, Jacob
%A Song, Dawn
%B Proceedings of the 39th International Conference on Machine Learning
%D 2022
%E Chaudhuri, Kamalika
%E Jegelka, Stefanie
%E Song, Le
%E Szepesvari, Csaba
%E Niu, Gang
%E Sabato, Sivan
%I PMLR
%K OOD_detection
%P 8759--8773
%T Scaling Out-of-Distribution Detection for Real-World Settings
%U https://proceedings.mlr.press/v162/hendrycks22a.html
%V 162
%X Detecting out-of-distribution examples is important for safety-critical machine learning applications such as detecting novel biological phenomena and self-driving cars. However, existing research mainly focuses on simple small-scale settings. To set the stage for more realistic out-of-distribution detection, we depart from small-scale settings and explore large-scale multiclass and multi-label settings with high-resolution images and thousands of classes. To make future work in real-world settings possible, we create new benchmarks for three large-scale settings. To test ImageNet multiclass anomaly detectors, we introduce the Species dataset containing over 700,000 images and over a thousand anomalous species. We leverage ImageNet-21K to evaluate PASCAL VOC and COCO multilabel anomaly detectors. Third, we introduce a new benchmark for anomaly segmentation by introducing a segmentation benchmark with road anomalies. We conduct extensive experiments in these more realistic settings for out-of-distribution detection and find that a surprisingly simple detector based on the maximum logit outperforms prior methods in all the large-scale multi-class, multi-label, and segmentation tasks, establishing a simple new baseline for future work.
@inproceedings{pmlr-v162-hendrycks22a_MLS,
abstract = {Detecting out-of-distribution examples is important for safety-critical machine learning applications such as detecting novel biological phenomena and self-driving cars. However, existing research mainly focuses on simple small-scale settings. To set the stage for more realistic out-of-distribution detection, we depart from small-scale settings and explore large-scale multiclass and multi-label settings with high-resolution images and thousands of classes. To make future work in real-world settings possible, we create new benchmarks for three large-scale settings. To test ImageNet multiclass anomaly detectors, we introduce the Species dataset containing over 700,000 images and over a thousand anomalous species. We leverage ImageNet-21K to evaluate PASCAL VOC and COCO multilabel anomaly detectors. Third, we introduce a new benchmark for anomaly segmentation by introducing a segmentation benchmark with road anomalies. We conduct extensive experiments in these more realistic settings for out-of-distribution detection and find that a surprisingly simple detector based on the maximum logit outperforms prior methods in all the large-scale multi-class, multi-label, and segmentation tasks, establishing a simple new baseline for future work.},
added-at = {2023-11-30T16:46:27.000+0100},
author = {Hendrycks, Dan and Basart, Steven and Mazeika, Mantas and Zou, Andy and Kwon, Joseph and Mostajabi, Mohammadreza and Steinhardt, Jacob and Song, Dawn},
biburl = {https://www.bibsonomy.org/bibtex/23eb20490beed55edccd6ce9bcdab2940/andolab},
booktitle = {Proceedings of the 39th International Conference on Machine Learning},
editor = {Chaudhuri, Kamalika and Jegelka, Stefanie and Song, Le and Szepesvari, Csaba and Niu, Gang and Sabato, Sivan},
interhash = {7e9709caac92c6851d6519d2a91f6310},
intrahash = {3eb20490beed55edccd6ce9bcdab2940},
keywords = {OOD_detection},
month = {17--23 Jul},
pages = {8759--8773},
pdf = {https://proceedings.mlr.press/v162/hendrycks22a/hendrycks22a.pdf},
publisher = {PMLR},
series = {Proceedings of Machine Learning Research},
timestamp = {2023-11-30T16:46:27.000+0100},
title = {Scaling Out-of-Distribution Detection for Real-World Settings},
url = {https://proceedings.mlr.press/v162/hendrycks22a.html},
volume = 162,
year = 2022
}