It is common practice to assess consistency of diagnostic ratings in terms of agreement beyond chance. To explore the interpretation of such a term we consider relevant statistical techniques such as Cohen's kappa and log-linear models for agreement on nominal ratings. We relate these approaches to a special latent class concept that decomposes observed ratings into a class of systematically consistent and a class of fortuitous ratings. This decomposition provides a common framework in which the specific premises of Cohen's kappa and of log-linear models can be identified and put into perspective. As a result it is shown that Cohen's kappa may be an inadequate and biased index of chance-corrected agreement in studies of intra-observer as well as inter-observer consistency. We suggest a more critical use and interpretation of measures gauging observer reliability by the amount of agreement beyond chance.
%0 Journal Article
%1 GuggenmoosHolzmann:Vonk:98
%A Guggenmoos-Holzmann, Irene
%A Vonk, Richard
%D 1998
%J Statistics in Medicine
%K annotation statistics kappa
%N 8
%P 797-812
%T Kappa-like indices of observer agreement viewed from a latent class perspective
%U http://www3.interscience.wiley.com/cgi-bin/abstract/3155/ABSTRACT?CRETRY=1&SRETRY=0
%V 17
%X It is common practice to assess consistency of diagnostic ratings in terms of agreement beyond chance. To explore the interpretation of such a term we consider relevant statistical techniques such as Cohen's kappa and log-linear models for agreement on nominal ratings. We relate these approaches to a special latent class concept that decomposes observed ratings into a class of systematically consistent and a class of fortuitous ratings. This decomposition provides a common framework in which the specific premises of Cohen's kappa and of log-linear models can be identified and put into perspective. As a result it is shown that Cohen's kappa may be an inadequate and biased index of chance-corrected agreement in studies of intra-observer as well as inter-observer consistency. We suggest a more critical use and interpretation of measures gauging observer reliability by the amount of agreement beyond chance.
@article{GuggenmoosHolzmann:Vonk:98,
abstract = {It is common practice to assess consistency of diagnostic ratings in terms of agreement beyond chance. To explore the interpretation of such a term we consider relevant statistical techniques such as Cohen's kappa and log-linear models for agreement on nominal ratings. We relate these approaches to a special latent class concept that decomposes observed ratings into a class of systematically consistent and a class of fortuitous ratings. This decomposition provides a common framework in which the specific premises of Cohen's kappa and of log-linear models can be identified and put into perspective. As a result it is shown that Cohen's kappa may be an inadequate and biased index of chance-corrected agreement in studies of intra-observer as well as inter-observer consistency. We suggest a more critical use and interpretation of measures gauging observer reliability by the amount of agreement beyond chance.},
added-at = {2007-01-31T04:55:37.000+0100},
author = {Guggenmoos-Holzmann, Irene and Vonk, Richard},
biburl = {https://www.bibsonomy.org/bibtex/222595770865145552a35c9a49bbcaae8/stefano},
interhash = {2545c1df03f785c1aa218a5758570213},
intrahash = {22595770865145552a35c9a49bbcaae8},
journal = {Statistics in Medicine},
keywords = {annotation statistics kappa},
number = 8,
pages = {797-812},
timestamp = {2007-01-31T04:55:37.000+0100},
title = {Kappa-like indices of observer agreement viewed from a latent class perspective},
url = {http://www3.interscience.wiley.com/cgi-bin/abstract/3155/ABSTRACT?CRETRY=1&SRETRY=0},
volume = 17,
year = 1998
}