%0 Thesis %1 noauthororeditor %A Götz, Raphael %A Krupitzer, Christian %A Kounev, Samuel %D 2020 %K raphaelgoetz %T Survey on the Evaluation of Self-Aware Computing Systems %X Software is becoming larger and more complex. In 2001 IBM published a manifest in which they stated that the increase in complexity is a major obstacle to progress for the IT industry. A solution to this looming complexity crisis is self-aware computing systems. These systems learn models that, help them to capture knowledge on themselves and their environment, and use these to consider actions in accoradnce with to act in accordance with higher level goals 3. This enables them to act according to their knowledge and reason. One open issue in the field of self-aware computing systems is the growing demand for evaluation methods that make these systems comparable 4. The main problem when evaluating self-aware computing systems is that traditional software metrics, such as re- sponse time or throughput, are not sufficient, as they cannot capture all relevant aspects of self-awareness. This thesis addresses the problems and challenges of evaluating self-aware computing systems and their related concepts, by providing an overview over the currently available evaluation methods for these systems. For this purpose, 37 papers dealing with the evaluation of self-aware computing systems and their related concepts are analyzed and classified using a taxonomy. This answers five research questions. Namely, the questions how and when the evaluation of self-aware computing systems is performed, and which parts of the systems are evaluated most frequently. The question whether the available methods of evaluation do address overshooting, oscillations an the level of self-awareness is also addressed. Answering these research questions results in the identification of research gaps. Namely, the insight that there is a lack of research concerned with the evaluation of the design of self-aware systems and proactive adaptation. Other observations are, that the sets of metrics and quality attributes found in the literature have surprisingly little overlap, suggesting that the do not cover all the relevant facets of self-adaptation and self-awareness, and that the evaluation of the level of self-awareness without access to the internals of the system is still rather difficult. Based on these insights improvements and best practices for future research are suggested.

@mastersthesis{noauthororeditor, abstract = {Software is becoming larger and more complex. In 2001 IBM published a manifest in which they stated that the increase in complexity is a major obstacle to progress for the IT industry. A solution to this looming complexity crisis is self-aware computing systems. These systems learn models that, help them to capture knowledge on themselves and their environment, and use these to consider actions in accoradnce with to act in accordance with higher level goals [3]. This enables them to act according to their knowledge and reason. One open issue in the field of self-aware computing systems is the growing demand for evaluation methods that make these systems comparable [4]. The main problem when evaluating self-aware computing systems is that traditional software metrics, such as re- sponse time or throughput, are not sufficient, as they cannot capture all relevant aspects of self-awareness. This thesis addresses the problems and challenges of evaluating self-aware computing systems and their related concepts, by providing an overview over the currently available evaluation methods for these systems. For this purpose, 37 papers dealing with the evaluation of self-aware computing systems and their related concepts are analyzed and classified using a taxonomy. This answers five research questions. Namely, the questions how and when the evaluation of self-aware computing systems is performed, and which parts of the systems are evaluated most frequently. The question whether the available methods of evaluation do address overshooting, oscillations an the level of self-awareness is also addressed. Answering these research questions results in the identification of research gaps. Namely, the insight that there is a lack of research concerned with the evaluation of the design of self-aware systems and proactive adaptation. Other observations are, that the sets of metrics and quality attributes found in the literature have surprisingly little overlap, suggesting that the do not cover all the relevant facets of self-adaptation and self-awareness, and that the evaluation of the level of self-awareness without access to the internals of the system is still rather difficult. Based on these insights improvements and best practices for future research are suggested.}, added-at = {2023-08-25T23:53:44.000+0200}, author = {Götz, Raphael and Krupitzer, Christian and Kounev, Samuel}, biburl = {https://www.bibsonomy.org/bibtex/2aacd914bb0241948991492fedc92a4ac/sssgroup}, interhash = {ac23c5033fde157e682d70cf9f323926}, intrahash = {aacd914bb0241948991492fedc92a4ac}, keywords = {raphaelgoetz}, timestamp = {2023-08-25T23:53:44.000+0200}, title = {Survey on the Evaluation of Self-Aware Computing Systems}, year = 2020 }