The Role of Deliberate Artificial Design Elements in Software Engineering Experiments

@article{citeulike:2635739, abstract = {Increased realism in software engineering experiments is often promoted as an important means to increase generalizability and industrial relevance. In this context, artificiality, e.g., the use of constructed tasks in place of realistic tasks, is seen as a threat. In this article, we examine the opposite view, that deliberately introduced artificial design elements may increase knowledge gain and enhance both generalizability and relevance. In the first part of the article, we identify and evaluate arguments and examples in favor of, and against, deliberately introducing artificiality into software engineering experiments. In the second part of the article, we summarize a content analysis of articles reporting software engineering experiments published over the ten-year period 1993-2002. The analysis reveals a striving for realism and external validity, but little awareness of for what and when, various degrees of artificiality and realism are appropriate. We conclude that an increased awareness and deliberation in these respects is essential. However, arguments in favor of artificial design elements should not be used to justify studies that are badly designed or that have research questions of low relevance.}, author = {Hannay, Jo and J\&\#x0f8 and Rgensen, Magne }, booktitle = {Software Engineering, IEEE Transactions on}, citeulike-article-id = {2635739}, doi = {http://dx.doi.org/10.1109/TSE.2008.13}, journal = {Software Engineering, IEEE Transactions on}, keywords = {empirical}, number = {2}, pages = {242--259}, posted-at = {2008-04-07 01:15:34}, priority = {2}, title = {The Role of Deliberate Artificial Design Elements in Software Engineering Experiments}, url = {http://dx.doi.org/10.1109/TSE.2008.13}, volume = {34}, year = {2008} }

TY - JOUR ID - citeulike:2635739 L3 - citeulike-article-id:2635739 TI - The Role of Deliberate Artificial Design Elements in Software Engineering Experiments T2 - Software Engineering, IEEE Transactions on JF - Software Engineering, IEEE Transactions on VL - 34 IS - 2 SP - 242 EP - 259 N2 - Increased realism in software engineering experiments is often promoted as an important means to increase generalizability and industrial relevance. In this context, artificiality, e.g., the use of constructed tasks in place of realistic tasks, is seen as a threat. In this article, we examine the opposite view, that deliberately introduced artificial design elements may increase knowledge gain and enhance both generalizability and relevance. In the first part of the article, we identify and evaluate arguments and examples in favor of, and against, deliberately introducing artificiality into software engineering experiments. In the second part of the article, we summarize a content analysis of articles reporting software engineering experiments published over the ten-year period 1993-2002. The analysis reveals a striving for realism and external validity, but little awareness of for what and when, various degrees of artificiality and realism are appropriate. We conclude that an increased awareness and deliberation in these respects is essential. However, arguments in favor of artificial design elements should not be used to justify studies that are badly designed or that have research questions of low relevance. KW - empirical AU - Hannay, Jo AU - J&#x0f8 AU - Rgensen, Magne PY - 2008/// UR - http://dx.doi.org/10.1109/TSE.2008.13 ER -