We propose a new framework for reasoning about information in complex
systems. Our foundation is based on a variational extension of Shannon's
information theory that takes into account the modeling power and computational
constraints of the observer. The resulting predictive
$\mathcalV$-information encompasses mutual information and other notions of
informativeness such as the coefficient of determination. Unlike Shannon's
mutual information and in violation of the data processing inequality,
$V$-information can be created through computation. This is
consistent with deep neural networks extracting hierarchies of progressively
more informative features in representation learning. Additionally, we show
that by incorporating computational constraints, $V$-information can
be reliably estimated from data even in high dimensions with PAC-style
guarantees. Empirically, we demonstrate predictive $V$-information is
more effective than mutual information for structure learning and fair
representation learning.
Description
[2002.10689] A Theory of Usable Information Under Computational Constraints
%0 Journal Article
%1 xu2020theory
%A Xu, Yilun
%A Zhao, Shengjia
%A Song, Jiaming
%A Stewart, Russell
%A Ermon, Stefano
%D 2020
%K compression computation constrains information readings theory
%T A Theory of Usable Information Under Computational Constraints
%U http://arxiv.org/abs/2002.10689
%X We propose a new framework for reasoning about information in complex
systems. Our foundation is based on a variational extension of Shannon's
information theory that takes into account the modeling power and computational
constraints of the observer. The resulting predictive
$\mathcalV$-information encompasses mutual information and other notions of
informativeness such as the coefficient of determination. Unlike Shannon's
mutual information and in violation of the data processing inequality,
$V$-information can be created through computation. This is
consistent with deep neural networks extracting hierarchies of progressively
more informative features in representation learning. Additionally, we show
that by incorporating computational constraints, $V$-information can
be reliably estimated from data even in high dimensions with PAC-style
guarantees. Empirically, we demonstrate predictive $V$-information is
more effective than mutual information for structure learning and fair
representation learning.
@article{xu2020theory,
abstract = {We propose a new framework for reasoning about information in complex
systems. Our foundation is based on a variational extension of Shannon's
information theory that takes into account the modeling power and computational
constraints of the observer. The resulting \emph{predictive
$\mathcal{V}$-information} encompasses mutual information and other notions of
informativeness such as the coefficient of determination. Unlike Shannon's
mutual information and in violation of the data processing inequality,
$\mathcal{V}$-information can be created through computation. This is
consistent with deep neural networks extracting hierarchies of progressively
more informative features in representation learning. Additionally, we show
that by incorporating computational constraints, $\mathcal{V}$-information can
be reliably estimated from data even in high dimensions with PAC-style
guarantees. Empirically, we demonstrate predictive $\mathcal{V}$-information is
more effective than mutual information for structure learning and fair
representation learning.},
added-at = {2020-05-03T11:11:45.000+0200},
author = {Xu, Yilun and Zhao, Shengjia and Song, Jiaming and Stewart, Russell and Ermon, Stefano},
biburl = {https://www.bibsonomy.org/bibtex/24ab645c98679ef43fe0ccc42b747eb6a/kirk86},
description = {[2002.10689] A Theory of Usable Information Under Computational Constraints},
interhash = {93811e81791e342c728f4cb40990fff8},
intrahash = {4ab645c98679ef43fe0ccc42b747eb6a},
keywords = {compression computation constrains information readings theory},
note = {cite arxiv:2002.10689Comment: ICLR 2020 (Talk)},
timestamp = {2020-05-03T11:12:27.000+0200},
title = {A Theory of Usable Information Under Computational Constraints},
url = {http://arxiv.org/abs/2002.10689},
year = 2020
}