The PARADISE Evaluation Framework: Issues and Findings

@article{Hajdinjak06paradise, abstract = {There has been a great deal of interest over the past 20 years in developing metrics and frameworks for evaluating and comparing the performance of spoken-language dialogue systems. One of the results of this interest is a potential general methodology, known as the PARADISE framework. This squib highlights some important issues concerning the application of PARADISE that have, up to now, not been sufficiently emphasized or have even been neglected by the dialogue-system community. These include considerations regarding the selection of appropriate regression parameters, normalization effects on the accuracy of the prediction, the influence of speech-recognition errors on the performance function, and the selection of an appropriate user-satisfaction measure. In addition, it gives the results of an evaluation of data from two Wizard-of-Oz experiments. These evaluations include different dependent variables and examination of individual user-satisfaction measures.}, author = {Hajdinjak, Melita and Miheli\v{c}*­, France }, citeulike-article-id = {1175979}, editor = {Dale, Robert }, journal = {Computational Linguistics}, keywords = {dialogue, evaluation, paradise}, month = {June}, number = {2}, pages = {263--272}, posted-at = {2007-03-19 14:46:36}, priority = {4}, publisher = {MIT Press}, title = {The PARADISE Evaluation Framework: Issues and Findings}, url = {http://www.mitpressjournals.org/doi/abs/10.1162/coli.2006.32.2.263}, volume = {32}, year = {2006} }

TY - JOUR ID - Hajdinjak06paradise L3 - citeulike-article-id:1175979 TI - The PARADISE Evaluation Framework: Issues and Findings JF - Computational Linguistics VL - 32 IS - 2 SP - 263 EP - 272 PB - MIT Press N2 - There has been a great deal of interest over the past 20 years in developing metrics and frameworks for evaluating and comparing the performance of spoken-language dialogue systems. One of the results of this interest is a potential general methodology, known as the PARADISE framework. This squib highlights some important issues concerning the application of PARADISE that have, up to now, not been sufficiently emphasized or have even been neglected by the dialogue-system community. These include considerations regarding the selection of appropriate regression parameters, normalization effects on the accuracy of the prediction, the influence of speech-recognition errors on the performance function, and the selection of an appropriate user-satisfaction measure. In addition, it gives the results of an evaluation of data from two Wizard-of-Oz experiments. These evaluations include different dependent variables and examination of individual user-satisfaction measures. KW - dialogue KW - evaluation KW - paradise AU - Hajdinjak, Melita AU - Mihelič*­, France A2 - Dale, Robert PY - 2006/06// UR - http://www.mitpressjournals.org/doi/abs/10.1162/coli.2006.32.2.263 ER -