%0 Conference Paper %1 falis-etal-2021-cophe %A Falis, Matús %A Dong, Hang %A Birch, Alexandra %A Alex, Beatrice %B Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing %C Online and Punta Cana, Dominican Republic %D 2021 %I Association for Computational Linguistics %K evaluation_metrics hierarchical_relation hierarchy icd mimic multi-label_classification nlp ontologies ontology %P 907--912 %T CoPHE: A Count-Preserving Hierarchical Evaluation Metric in Large-Scale Multi-Label Text Classification %U https://aclanthology.org/2021.emnlp-main.69 %X Large-Scale Multi-Label Text Classification (LMTC) includes tasks with hierarchical label spaces, such as automatic assignment of ICD-9 codes to discharge summaries. Performance of models in prior art is evaluated with standard precision, recall, and F1 measures without regard for the rich hierarchical structure. In this work we argue for hierarchical evaluation of the predictions of neural LMTC models. With the example of the ICD-9 ontology we describe a structural issue in the representation of the structured label space in prior art, and propose an alternative representation based on the depth of the ontology. We propose a set of metrics for hierarchical evaluation using the depth-based representation. We compare the evaluation scores from the proposed metrics with previously used metrics on prior art LMTC models for ICD-9 coding in MIMIC-III. We also propose further avenues of research involving the proposed ontological representation.

@inproceedings{falis-etal-2021-cophe, abstract = {Large-Scale Multi-Label Text Classification (LMTC) includes tasks with hierarchical label spaces, such as automatic assignment of ICD-9 codes to discharge summaries. Performance of models in prior art is evaluated with standard precision, recall, and F1 measures without regard for the rich hierarchical structure. In this work we argue for hierarchical evaluation of the predictions of neural LMTC models. With the example of the ICD-9 ontology we describe a structural issue in the representation of the structured label space in prior art, and propose an alternative representation based on the depth of the ontology. We propose a set of metrics for hierarchical evaluation using the depth-based representation. We compare the evaluation scores from the proposed metrics with previously used metrics on prior art LMTC models for ICD-9 coding in MIMIC-III. We also propose further avenues of research involving the proposed ontological representation.}, added-at = {2021-11-09T15:00:33.000+0100}, address = {Online and Punta Cana, Dominican Republic}, author = {Falis, Mat{\'u}{\v{s}} and Dong, Hang and Birch, Alexandra and Alex, Beatrice}, biburl = {https://www.bibsonomy.org/bibtex/2baa06ca5034855d14cbe1b862538ad33/hangdong}, booktitle = {Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing}, description = {CoPHE: A Count-Preserving Hierarchical Evaluation Metric in Large-Scale Multi-Label Text Classification - ACL Anthology}, interhash = {79d1546b2b626a140d4033e6a04050a1}, intrahash = {baa06ca5034855d14cbe1b862538ad33}, keywords = {evaluation_metrics hierarchical_relation hierarchy icd mimic multi-label_classification nlp ontologies ontology}, month = nov, pages = {907--912}, publisher = {Association for Computational Linguistics}, timestamp = {2021-11-09T15:00:33.000+0100}, title = {{C}o{PHE}: A Count-Preserving Hierarchical Evaluation Metric in Large-Scale Multi-Label Text Classification}, url = {https://aclanthology.org/2021.emnlp-main.69}, year = 2021 }