%0 Journal Article %1 Fan2022 %A Fan, Yizhou %A van der Graaf, Joep %A Lim, Lyn %A Raković, Mladen %A Singh, Shaveen %A Kilgour, Jonathan %A Moore, Johanna %A Molenaar, Inge %A Bannert, Maria %A Gasević, Dragan %D 2022 %J Metacognition and Learning %K alignment data-driven learninganalytics measurement self-regulatedlearning theory-driven thinkalouddata tracedata validity %R 10.1007/s11409-022-09291-1 %T Towards investigating the validity of measurement of self-regulated learning based on trace data %U https://doi.org/10.1007/s11409-022-09291-1 %X Contemporary research that looks at self-regulated learning (SRL) as processes of learning events derived from trace data has attracted increasing interest over the past decade. However, limited research has been conducted that looks into the validity of trace-based measurement protocols. In order to fill this gap in the literature, we propose a novel validation approach that combines theory-driven and data-driven perspectives to increase the validity of interpretations of SRL processes extracted from trace-data. The main contribution of this approach consists of three alignments between trace data and think aloud data to improve measurement validity. In addition, we define the match rate between SRL processes extracted from trace data and think aloud as a quantitative indicator together with other three indicators (sensitivity, specificity and trace coverage), to evaluate the ``degree'' of validity. We tested this validation approach in a laboratory study that involved 44 learners who learned individually about the topic of artificial intelligence in education with the use of a technology-enhanced learning environment for 45 minutes. Following this new validation approach, we achieved an improved match rate between SRL processes extracted from trace-data and think aloud data (training set: 54.24\%; testing set: 55.09\%) compared to the match rate before applying the validation approach (training set: 38.97\%; test set: 34.54\%). By considering think aloud data as ``reference point'', this improvement of the match rate quantified the extent to which validity can be improved by using our validation approach. In conclusion, the novel validation approach presented in this study used both empirical evidence from think aloud data and rationale from our theoretical framework of SRL, which now, allows testing and improvement of the validity of trace-based SRL measurements.

@article{Fan2022, abstract = {Contemporary research that looks at self-regulated learning (SRL) as processes of learning events derived from trace data has attracted increasing interest over the past decade. However, limited research has been conducted that looks into the validity of trace-based measurement protocols. In order to fill this gap in the literature, we propose a novel validation approach that combines theory-driven and data-driven perspectives to increase the validity of interpretations of SRL processes extracted from trace-data. The main contribution of this approach consists of three alignments between trace data and think aloud data to improve measurement validity. In addition, we define the match rate between SRL processes extracted from trace data and think aloud as a quantitative indicator together with other three indicators (sensitivity, specificity and trace coverage), to evaluate the ``degree'' of validity. We tested this validation approach in a laboratory study that involved 44 learners who learned individually about the topic of artificial intelligence in education with the use of a technology-enhanced learning environment for 45 minutes. Following this new validation approach, we achieved an improved match rate between SRL processes extracted from trace-data and think aloud data (training set: 54.24{\%}; testing set: 55.09{\%}) compared to the match rate before applying the validation approach (training set: 38.97{\%}; test set: 34.54{\%}). By considering think aloud data as ``reference point'', this improvement of the match rate quantified the extent to which validity can be improved by using our validation approach. In conclusion, the novel validation approach presented in this study used both empirical evidence from think aloud data and rationale from our theoretical framework of SRL, which now, allows testing and improvement of the validity of trace-based SRL measurements.}, added-at = {2022-05-08T11:38:54.000+0200}, author = {Fan, Yizhou and van der Graaf, Joep and Lim, Lyn and Rakovi{\'{c}}, Mladen and Singh, Shaveen and Kilgour, Jonathan and Moore, Johanna and Molenaar, Inge and Bannert, Maria and Ga{\v{s}}evi{\'{c}}, Dragan}, biburl = {https://www.bibsonomy.org/bibtex/2a5b3e2cdda633031fe1cdcd689c1cdf5/ereidt}, day = 04, doi = {10.1007/s11409-022-09291-1}, interhash = {cd7d6bc20b9c30b557fac7bdeafd66b8}, intrahash = {a5b3e2cdda633031fe1cdcd689c1cdf5}, issn = {1556-1631}, journal = {Metacognition and Learning}, keywords = {alignment data-driven learninganalytics measurement self-regulatedlearning theory-driven thinkalouddata tracedata validity}, month = may, timestamp = {2022-05-08T11:38:54.000+0200}, title = {Towards investigating the validity of measurement of self-regulated learning based on trace data}, url = {https://doi.org/10.1007/s11409-022-09291-1}, year = 2022 }