@inproceedings{2022.EDM-short-papers.32,
abstract = {Online educational technologies facilitate pedagogical experimentation, but typical experimental designs assign a fixed proportion of students to each condition, even if early results suggest some are ineffective. Experimental designs using multi-armed bandit (MAB) algorithms vary the probability of condition assignment for a new student based on prior results, placing more students in more effective conditions.
While stochastic MAB algorithms have been used for educational experiments, they collect data that decreases power and increases false positive rates. Instead, we propose using adversarial MAB algorithms, which are less exploitative and thus may exhibit more robustness. Through simulations involving data from 20+ educational experiments (Selent et al. 2016), we show data collected using adversarial MAB algorithms does not have the statistical downsides of that from stochastic MAB algorithms. Further, we explore how differences in condition variability impact MAB versus uniform experimental design, such as if performance gaps between students are narrowed by an intervention. Surprisingly, data from stochastic MAB algorithms systematically reduces power when the better arm is less variable, while increasing it when the better arm is more variable; data from the adversarial MAB algorithms results in the same statistical power as uniform assignment. Overall, these results demonstrate that adversarial MAB algorithms are a viable "off-the-shelf" solution for researchers who want to preserve the statistical power of standard experimental designs while also benefiting student participants.},
address = {Durham, United Kingdom},
author = {Yang Zhi-Han and Shiyue Zhang and Anna Rafferty},
booktitle = {Proceedings of the 15th International Conference on Educational Data Mining},
doi = {10.5281/zenodo.6853039},
editor = {Antonija Mitrovic and Nigel Bosch},
isbn = {978-1-7336736-3-1},
month = {July},
pages = {353--360},
publisher = {International Educational Data Mining Society},
title = {Adversarial bandits for drawing generalizable conclusions in non-adversarial experiments: an empirical study},
year = {2022}
}