The dominant graph neural networks (GNNs) overrely on the graph links, several serious performance problems with which have been witnessed already, e.g., suspended animation problem and over-smoothing problem. What’s more, the inherently inter-connected nature precludes parallelization within the graph, which becomes critical for large-sized graph, as memory constraints limit batching across the nodes. In this paper, we will introduce a new graph neural network, namely GRAPH-BERT (Graph based BERT), solely based on the attention mechanism without any graph convolution or aggregation operators. Instead of feeding GRAPH-BERT with the complete large input graph, we propose to train GRAPH-BERT with sampled linkless subgraphs within their local contexts. GRAPH-BERT can be learned effectively in a standalone mode. Meanwhile, a pre-trained GRAPHBERT can also be transferred to other application tasks directly or with necessary fine-tuning if any supervised label information or certain application oriented objective is available. We have tested the effectiveness of GRAPH-BERT on several graph benchmark datasets. Based the pretrained GRAPH-BERT with the node attribute reconstruction and structure recovery tasks, we further fine-tune GRAPH-BERT on node classification and graph clustering tasks specifically. The experimental results have demonstrated that GRAPHBERT can out-perform the existing GNNs in both the learning effectiveness and efficiency.
Zhang 等。 - 2020 - Graph-Bert Only Attention is Needed for Learning .pdf:files/26/Zhang 等。 - 2020 - Graph-Bert Only Attention is Needed for Learning .pdf:application/pdf
%0 Journal Article
%1 zhang_graph-bert_2020
%A Zhang, Jiawei
%A Zhang, Haopeng
%A Xia, Congying
%A Sun, Li
%D 2020
%J arXiv:2001.05140 cs, stat
%K attention pretrain ssl
%T Graph-Bert: Only Attention is Needed for Learning Graph Representations
%U http://arxiv.org/abs/2001.05140
%X The dominant graph neural networks (GNNs) overrely on the graph links, several serious performance problems with which have been witnessed already, e.g., suspended animation problem and over-smoothing problem. What’s more, the inherently inter-connected nature precludes parallelization within the graph, which becomes critical for large-sized graph, as memory constraints limit batching across the nodes. In this paper, we will introduce a new graph neural network, namely GRAPH-BERT (Graph based BERT), solely based on the attention mechanism without any graph convolution or aggregation operators. Instead of feeding GRAPH-BERT with the complete large input graph, we propose to train GRAPH-BERT with sampled linkless subgraphs within their local contexts. GRAPH-BERT can be learned effectively in a standalone mode. Meanwhile, a pre-trained GRAPHBERT can also be transferred to other application tasks directly or with necessary fine-tuning if any supervised label information or certain application oriented objective is available. We have tested the effectiveness of GRAPH-BERT on several graph benchmark datasets. Based the pretrained GRAPH-BERT with the node attribute reconstruction and structure recovery tasks, we further fine-tune GRAPH-BERT on node classification and graph clustering tasks specifically. The experimental results have demonstrated that GRAPHBERT can out-perform the existing GNNs in both the learning effectiveness and efficiency.
%Z Comment: 10 pages
@article{zhang_graph-bert_2020,
abstract = {The dominant graph neural networks (GNNs) overrely on the graph links, several serious performance problems with which have been witnessed already, e.g., suspended animation problem and over-smoothing problem. What’s more, the inherently inter-connected nature precludes parallelization within the graph, which becomes critical for large-sized graph, as memory constraints limit batching across the nodes. In this paper, we will introduce a new graph neural network, namely GRAPH-BERT (Graph based BERT), solely based on the attention mechanism without any graph convolution or aggregation operators. Instead of feeding GRAPH-BERT with the complete large input graph, we propose to train GRAPH-BERT with sampled linkless subgraphs within their local contexts. GRAPH-BERT can be learned effectively in a standalone mode. Meanwhile, a pre-trained GRAPHBERT can also be transferred to other application tasks directly or with necessary fine-tuning if any supervised label information or certain application oriented objective is available. We have tested the effectiveness of GRAPH-BERT on several graph benchmark datasets. Based the pretrained GRAPH-BERT with the node attribute reconstruction and structure recovery tasks, we further fine-tune GRAPH-BERT on node classification and graph clustering tasks specifically. The experimental results have demonstrated that GRAPHBERT can out-perform the existing GNNs in both the learning effectiveness and efficiency.},
added-at = {2021-04-08T07:18:22.000+0200},
annote = {Comment: 10 pages},
author = {Zhang, Jiawei and Zhang, Haopeng and Xia, Congying and Sun, Li},
biburl = {https://www.bibsonomy.org/bibtex/295ba6889c96b9a3dcbb2b46655a3079a/mengcao},
file = {Zhang 等。 - 2020 - Graph-Bert Only Attention is Needed for Learning .pdf:files/26/Zhang 等。 - 2020 - Graph-Bert Only Attention is Needed for Learning .pdf:application/pdf},
interhash = {bfd4dc9d73df1b6e1b05c763256e71e9},
intrahash = {95ba6889c96b9a3dcbb2b46655a3079a},
journal = {arXiv:2001.05140 [cs, stat]},
keywords = {attention pretrain ssl},
language = {en},
month = jan,
note = {arXiv: 2001.05140},
shorttitle = {Graph-{Bert}},
timestamp = {2021-04-25T13:21:04.000+0200},
title = {Graph-{Bert}: {Only} {Attention} is {Needed} for {Learning} {Graph} {Representations}},
url = {http://arxiv.org/abs/2001.05140},
urldate = {2020-11-28},
year = 2020
}