%0 Journal Article %1 DBLP:conf/icadl/KabongoDA21 %A Kabongo, Salomon %A D'Souza, Jennifer %A Auer, Sören %D 2021 %J springer, International Conference on Asian Digital Libraries %K Information-extraction Knowledge-graphs Neural-machine-learning Scholarly-text-mining Table-mining leibnizailab myown %P 453–470 %R https://doi.org/10.1007/978-3-030-91669-5_35 %T Automated Mining of Leaderboards for Empirical AI Research %U https://link.springer.com/chapter/10.1007/978-3-030-91669-5_35 %V ICADL 2021: Towards Open and Trustworthy Digital Societies %X With the rapid growth of research publications, empowering scientists to keep an oversight over scientific progress is of paramount importance. In this regard, the leaderboards facet of information organization provides an overview on the state-of-the-art by aggregating empirical results from various studies addressing the same research challenge. Crowdsourcing efforts like PAPERSWITHCODE among others are devoted to the construction of leaderboards predominantly for various subdomains in Artificial Intelligence. Leaderboards provide machine-readable scholarly knowledge that has proven to be directly useful for scientists to keep track of research progress – their construction could be greatly expedited with automated text mining. This study presents a comprehensive approach for generating leaderboards for knowledge-graph-based scholarly information organization. Specifically, we investigate the problem of automated leaderboard construction using state-of-the-art transformer models, viz. Bert, SciBert, and XLNet. Our analysis reveals an optimal approach that significantly outperforms existing baselines for the task with evaluation scores above 90% in F1. This, in turn, offers new state-of-the-art results for leaderboard extraction. As a result, a vast share of empirical AI research can be organized in the next-generation digital libraries as knowledge graphs.

@article{DBLP:conf/icadl/KabongoDA21, abstract = {With the rapid growth of research publications, empowering scientists to keep an oversight over scientific progress is of paramount importance. In this regard, the leaderboards facet of information organization provides an overview on the state-of-the-art by aggregating empirical results from various studies addressing the same research challenge. Crowdsourcing efforts like PAPERSWITHCODE among others are devoted to the construction of leaderboards predominantly for various subdomains in Artificial Intelligence. Leaderboards provide machine-readable scholarly knowledge that has proven to be directly useful for scientists to keep track of research progress – their construction could be greatly expedited with automated text mining. This study presents a comprehensive approach for generating leaderboards for knowledge-graph-based scholarly information organization. Specifically, we investigate the problem of automated leaderboard construction using state-of-the-art transformer models, viz. Bert, SciBert, and XLNet. Our analysis reveals an optimal approach that significantly outperforms existing baselines for the task with evaluation scores above 90% in F1. This, in turn, offers new state-of-the-art results for leaderboard extraction. As a result, a vast share of empirical AI research can be organized in the next-generation digital libraries as knowledge graphs.}, added-at = {2022-03-23T11:33:08.000+0100}, author = {Kabongo, Salomon and D'Souza, Jennifer and Auer, S{\"{o}}ren}, bibsource = {dblp computer science bibliography, https://dblp.org}, biburl = {https://www.bibsonomy.org/bibtex/2b3246b5a93dbfff4315c185fe71c1060/ksalomon}, doi = {https://doi.org/10.1007/978-3-030-91669-5_35}, eprint = {2109.13089}, eprinttype = {arXiv}, interhash = {4da507f8e881375bd0d837ed935de3a6}, intrahash = {b3246b5a93dbfff4315c185fe71c1060}, journal = {springer, International Conference on Asian Digital Libraries}, keywords = {Information-extraction Knowledge-graphs Neural-machine-learning Scholarly-text-mining Table-mining leibnizailab myown}, month = {November 2021}, pages = {453–470}, timestamp = {2022-03-23T11:33:08.000+0100}, title = {Automated Mining of Leaderboards for Empirical {AI} Research}, url = {https://link.springer.com/chapter/10.1007/978-3-030-91669-5_35}, volume = {ICADL 2021: Towards Open and Trustworthy Digital Societies}, year = 2021 }