The information bottleneck method TeX Export

@inproceedings{citeulike:466027, abstract = {We define the relevant information in a signal \$x\in X\$ as being the information that this signal provides about another signal \$y\in \Y\$. Examples include the information that face images provide about the names of the people portrayed, or the information that speech sounds provide about the words spoken. Understanding the signal \$x\$ requires more than just predicting \$y\$, it also requires specifying which features of \$\X\$ play a role in the prediction. We formalize this problem as that of finding a short code for \$\X\$ that preserves the maximum information about \$\Y\$. That is, we squeeze the information that \$\X\$ provides about \$\Y\$ through a `bottleneck' formed by a limited set of codewords \$\tX\$. This constrained optimization problem can be seen as a generalization of rate distortion theory in which the distortion measure \$d(x,\x)\$ emerges from the joint statistics of \$\X\$ and \$\Y\$. This approach yields an exact set of self consistent equations for the coding rules \$X \to \tX\$ and \$\tX \to \Y\$. Solutions to these equations can be found by a convergent re-estimation method that generalizes the Blahut-Arimoto algorithm. Our variational principle provides a surprisingly rich framework for discussing a variety of problems in signal processing and learning, as will be described in detail elsewhere.}, archivePrefix = {arXiv}, author = {Tishby, Naftali and Pereira, Fernando C. and Bialek, William}, booktitle = {Proceedings of the 37-th Annual Allerton Conference on Communication, Control and Computing}, citeulike-article-id = {466027}, citeulike-linkout-0 = {http://arxiv.org/abs/physics/?0004057}, eprint = {physics/?0004057}, keywords = {information-theory}, pages = {368--377}, posted-at = {2006-01-16 13:24:34}, priority = {2}, title = {The information bottleneck method}, url = {http://arxiv.org/abs/physics/?0004057}, year = {1999} }

TY - CONF ID - citeulike:466027 L3 - citeulike-article-id:466027 N2 - We define the relevant information in a signal $x\in X$ as being the information that this signal provides about another signal $y\in \Y$. Examples include the information that face images provide about the names of the people portrayed, or the information that speech sounds provide about the words spoken. Understanding the signal $x$ requires more than just predicting $y$, it also requires specifying which features of $\X$ play a role in the prediction. We formalize this problem as that of finding a short code for $\X$ that preserves the maximum information about $\Y$. That is, we squeeze the information that $\X$ provides about $\Y$ through a `bottleneck' formed by a limited set of codewords $\tX$. This constrained optimization problem can be seen as a generalization of rate distortion theory in which the distortion measure $d(x,\x)$ emerges from the joint statistics of $\X$ and $\Y$. This approach yields an exact set of self consistent equations for the coding rules $X \to \tX$ and $\tX \to \Y$. Solutions to these equations can be found by a convergent re-estimation method that generalizes the Blahut-Arimoto algorithm. Our variational principle provides a surprisingly rich framework for discussing a variety of problems in signal processing and learning, as will be described in detail elsewhere. EP - 377 TI - The information bottleneck method T2 - Proceedings of the 37-th Annual Allerton Conference on Communication, Control and Computing SP - 368 KW - information-theory AU - Tishby, Naftali AU - Pereira, Fernando AU - Bialek, William PY - 1999/// UR - http://arxiv.org/abs/physics/?0004057 ER -