%0 Journal Article %1 omeliyanenko2023capskg %A Omeliyanenko, Janna %A Zehe, Albin %A Hotho, Andreas %A Schlör, Daniel %D 2023 %J International Semantic Web Conference ISWC 2023, to appear %K capskg kg knowledge knowledgegraph myown %T CapsKG: Enabling Continual Knowledge Integration in Language Models for Automatic Knowledge Graph Completion %X Automated completion of knowledge graphs is a popular topic in the Semantic Web community that aims to automatically and continuously integrate new appearing knowledge into knowledge graphs using artificial intelligence. Recently, approaches that leverage implicit knowledge from language models for this task have shown promising re- sults. However, by fine-tuning language models directly to the domain of knowledge graphs, models forget their original language representation and associated knowledge. An existing solution to address this issue is a trainable adapter, which is integrated into a frozen language model to extract the relevant knowledge without altering the model itself. How- ever, this constrains the generalizability to the specific extraction task and by design requires new and independent adapters to be trained for new knowledge extraction tasks. This effectively prevents the model from benefiting from existing knowledge incorporated in previously trained adapters. In this paper, we propose to combine the benefits of adapters for knowl- edge graph completion with the idea of integrating capsules, introduced in the field of continual learning. This allows the continuous integra- tion of knowledge into a joint model by sharing and reusing previously trained capsules. We find that our approach outperforms solutions using traditional adapters, while requiring notably fewer parameters for con- tinuous knowledge integration. Moreover, we show that this architecture benefits significantly from knowledge sharing in low-resource situations, outperforming adapter-based models on the task of link prediction.

@article{omeliyanenko2023capskg, abstract = {Automated completion of knowledge graphs is a popular topic in the Semantic Web community that aims to automatically and continuously integrate new appearing knowledge into knowledge graphs using artificial intelligence. Recently, approaches that leverage implicit knowledge from language models for this task have shown promising re- sults. However, by fine-tuning language models directly to the domain of knowledge graphs, models forget their original language representation and associated knowledge. An existing solution to address this issue is a trainable adapter, which is integrated into a frozen language model to extract the relevant knowledge without altering the model itself. How- ever, this constrains the generalizability to the specific extraction task and by design requires new and independent adapters to be trained for new knowledge extraction tasks. This effectively prevents the model from benefiting from existing knowledge incorporated in previously trained adapters. In this paper, we propose to combine the benefits of adapters for knowl- edge graph completion with the idea of integrating capsules, introduced in the field of continual learning. This allows the continuous integra- tion of knowledge into a joint model by sharing and reusing previously trained capsules. We find that our approach outperforms solutions using traditional adapters, while requiring notably fewer parameters for con- tinuous knowledge integration. Moreover, we show that this architecture benefits significantly from knowledge sharing in low-resource situations, outperforming adapter-based models on the task of link prediction.}, added-at = {2024-01-22T12:49:25.000+0100}, author = {Omeliyanenko, Janna and Zehe, Albin and Hotho, Andreas and Schlör, Daniel}, biburl = {https://www.bibsonomy.org/bibtex/2f7bfc8f3297a421da46b3107ba67211b/albinzehe}, description = {Accepted at ISWC 2023, to appear}, interhash = {8f031d3c320a2055a5c8fdeca152cc9b}, intrahash = {f7bfc8f3297a421da46b3107ba67211b}, journal = {International Semantic Web Conference ISWC 2023, to appear}, keywords = {capskg kg knowledge knowledgegraph myown}, timestamp = {2024-01-22T12:49:25.000+0100}, title = {CapsKG: Enabling Continual Knowledge Integration in Language Models for Automatic Knowledge Graph Completion}, year = 2023 }