Damaged merchandise? a review of experiments that compare usability evaluation methods Export

@article{citeulike:2696965, abstract = {An interest in the design of interfaces has been a core topic for researchers and practitioners in the field of human-computer interaction (HCI); an interest in the design of experiments has not. To the extent that reliable and valid guidance for the former depends on the results of the latter, it is necessary that researchers and practitioners understand how small features of an experimental design can cast large shadows over the results and conclusions that can be drawn. In this review we examine the design of 5 experiments that compared usability evaluation methods (UEMs). Each has had an important influence on HCI thought and practice. Unfortunately, our examination shows that small problems in the way these experiments were designed and conducted call into serious question what we thought we knew regarding the efficacy of various UEMs. If the influence of these experiments were trivial, then such small problems could be safely ignored. Unfortunately, the outcomes of these experiments have been used to justify advice to practitioners regarding their choice of UEMs. Making such choices based on misleading or erroneous claims can be detrimental--compromising the quality and integrity of the evaluation, incurring unnecessary costs, or undermining the practitioner's credibility within the design team. The experimental method is a potent vehicle that can help inform the choice of a UEM as well as help to address other HCI issues. However, to obtain the desired outcomes, close attention must be paid to experimental design.}, address = {Hillsdale, NJ, USA}, author = {Gray, Wayne D. and Salzman, Marilyn C.}, citeulike-article-id = {2696965}, citeulike-linkout-0 = {http://portal.acm.org/citation.cfm?id=1462989.1462991}, citeulike-linkout-1 = {http://dx.doi.org/10.1207/s15327051hci1303_2}, doi = {10.1207/s15327051hci1303_2}, issn = {0737-0024}, journal = {Hum.-Comput. Interact.}, keywords = {evaluation, hci, thesis, usability}, number = {3}, pages = {203--261}, posted-at = {2009-06-28 20:17:55}, priority = {0}, publisher = {L. Erlbaum Associates Inc.}, title = {Damaged merchandise? a review of experiments that compare usability evaluation methods}, url = {http://dx.doi.org/10.1207/s15327051hci1303_2}, volume = {13}, year = {1998} }

TY - JOUR ID - citeulike:2696965 L3 - citeulike-article-id:2696965 N2 - An interest in the design of interfaces has been a core topic for researchers and practitioners in the field of human-computer interaction (HCI); an interest in the design of experiments has not. To the extent that reliable and valid guidance for the former depends on the results of the latter, it is necessary that researchers and practitioners understand how small features of an experimental design can cast large shadows over the results and conclusions that can be drawn. In this review we examine the design of 5 experiments that compared usability evaluation methods (UEMs). Each has had an important influence on HCI thought and practice. Unfortunately, our examination shows that small problems in the way these experiments were designed and conducted call into serious question what we thought we knew regarding the efficacy of various UEMs. If the influence of these experiments were trivial, then such small problems could be safely ignored. Unfortunately, the outcomes of these experiments have been used to justify advice to practitioners regarding their choice of UEMs. Making such choices based on misleading or erroneous claims can be detrimental--compromising the quality and integrity of the evaluation, incurring unnecessary costs, or undermining the practitioner's credibility within the design team. The experimental method is a potent vehicle that can help inform the choice of a UEM as well as help to address other HCI issues. However, to obtain the desired outcomes, close attention must be paid to experimental design. IS - 3 JF - Hum.-Comput. Interact. SN - 0737-0024 AD - Hillsdale, NJ, USA EP - 261 TI - Damaged merchandise? a review of experiments that compare usability evaluation methods PB - L. Erlbaum Associates Inc. VL - 13 SP - 203 KW - evaluation KW - hci KW - thesis KW - usability AU - Gray, Wayne AU - Salzman, Marilyn PY - 1998/// UR - http://dx.doi.org/10.1207/s15327051hci1303_2 ER -