TF-IDF uncovered: a study of theories and probabilities Export

@inproceedings{citeulike:3056638, abstract = {Interpretations of TF-IDF are based on binary independence retrieval, Poisson, information theory, and language modelling. This paper contributes a review of existing interpretations, and then, TF-IDF is systematically related to the probabilities P(q|d) and P(d|q). Two approaches are explored: a space of independent, and a space of disjoint terms. For independent terms, an "extreme" query/non-query term assumption uncovers TF-IDF, and an analogy of P(d|q) and the probabilistic odds O(r|d, q) mirrors relevance feedback. For disjoint terms, a relationship between probability theory and TF-IDF is established through the integral + 1/x dx = log x. This study uncovers components such as divergence from randomness and pivoted document length to be inherent parts of a document-query independence (DQI) measure, and interestingly, an integral of the DQI over the term occurrence probability leads to TF-IDF.}, address = {New York, NY, USA}, author = {Roelleke, Thomas and Wang, Jun}, booktitle = {SIGIR '08: Proceedings of the 31st annual international ACM SIGIR conference on Research and development in information retrieval}, citeulike-article-id = {3056638}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1390334.1390409}, citeulike-linkout-1 = {http://dx.doi.org/10.1145/1390334.1390409}, doi = {10.1145/1390334.1390409}, isbn = {9781605581644}, keywords = {information-retrieval}, location = {Singapore, Singapore}, pages = {435--442}, posted-at = {2008-09-24 03:14:12}, priority = {2}, publisher = {ACM}, title = {TF-IDF uncovered: a study of theories and probabilities}, url = {http://dx.doi.org/10.1145/1390334.1390409}, year = {2008} }

TY - CONF ID - citeulike:3056638 L3 - citeulike-article-id:3056638 N2 - Interpretations of TF-IDF are based on binary independence retrieval, Poisson, information theory, and language modelling. This paper contributes a review of existing interpretations, and then, TF-IDF is systematically related to the probabilities P(q|d) and P(d|q). Two approaches are explored: a space of independent, and a space of disjoint terms. For independent terms, an "extreme" query/non-query term assumption uncovers TF-IDF, and an analogy of P(d|q) and the probabilistic odds O(r|d, q) mirrors relevance feedback. For disjoint terms, a relationship between probability theory and TF-IDF is established through the integral + 1/x dx = log x. This study uncovers components such as divergence from randomness and pivoted document length to be inherent parts of a document-query independence (DQI) measure, and interestingly, an integral of the DQI over the term occurrence probability leads to TF-IDF. CY - Singapore, Singapore SN - 9781605581644 AD - New York, NY, USA EP - 442 TI - TF-IDF uncovered: a study of theories and probabilities PB - ACM T2 - SIGIR '08: Proceedings of the 31st annual international ACM SIGIR conference on Research and development in information retrieval SP - 435 KW - information-retrieval AU - Roelleke, Thomas AU - Wang, Jun PY - 2008/// UR - http://dx.doi.org/10.1145/1390334.1390409 ER -