@inbook {IOPORT.05547156,
    author       = {Efron, Miles},
    title        = {Using multiple query aspects to build test collections without human relevance judgments.},
    year         = {2009},
    booktitle    = {Advances in information retrieval. 31th European conference on IR research, ECIR 2009, Toulouse, France, April 6--9, 2009. Proceedings},
    isbn         = {978-3-642-00957-0},
    pages        = {276-287},
    publisher    = {Berlin: Springer},
    doi          = {10.1007/978-3-642-00958-7_26},
    abstract     = {Summary: Collecting relevance judgments (qrels) is an especially challenging part of building an information retrieval test collection. This paper presents a novel method for creating test collections by offering a substitute for relevance judgments. Our method is based on an old idea in IR: a single information need can be represented by many query articulations. We call different articulations of a particular need query aspects. By combining the top $k$ documents retrieved by a single system for multiple query aspects, we build judgment-free qrels whose rank ordering of IR systems correlates highly with rankings based on human relevance judgments.},
    identifier   = {05547156},
}