diff --git a/docs/regressions-dl21-doc-d2q-t5.md b/docs/regressions-dl21-doc-d2q-t5.md index 8c34ea1ba3..4c93313593 100644 --- a/docs/regressions-dl21-doc-d2q-t5.md +++ b/docs/regressions-dl21-doc-d2q-t5.md @@ -62,6 +62,13 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt \ -hits 1000 -bm25 -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.dl21.txt \ + -hits 1000 -bm25 -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -76,23 +83,28 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics- tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.dl21.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.dl21.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2387 | 0.2608 | -| **MRR@100** | **BM25 (default)**| **+RM3** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.8866 | 0.8342 | -| **nDCG@10** | **BM25 (default)**| **+RM3** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5792 | 0.5392 | -| **R@100** | **BM25 (default)**| **+RM3** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3443 | 0.3580 | -| **R@1000** | **BM25 (default)**| **+RM3** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7066 | 0.7572 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2387 | 0.2608 | 0.2610 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.8866 | 0.8342 | 0.8459 | +| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5792 | 0.5392 | 0.5509 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3443 | 0.3580 | 0.3616 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7066 | 0.7572 | 0.7583 | Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: diff --git a/docs/regressions-dl21-doc-segmented-d2q-t5.md b/docs/regressions-dl21-doc-segmented-d2q-t5.md index 7944af9e16..7109a162b2 100644 --- a/docs/regressions-dl21-doc-segmented-d2q-t5.md +++ b/docs/regressions-dl21-doc-segmented-d2q-t5.md @@ -62,6 +62,13 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt \ -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt \ + -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -76,23 +83,28 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics- tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.dl21.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2683 | 0.3192 | -| **MRR@100** | **BM25 (default)**| **+RM3** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.9454 | 0.8960 | -| **nDCG@10** | **BM25 (default)**| **+RM3** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6289 | 0.6555 | -| **R@100** | **BM25 (default)**| **+RM3** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3656 | 0.4119 | -| **R@1000** | **BM25 (default)**| **+RM3** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7202 | 0.7941 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2683 | 0.3192 | 0.3218 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.9454 | 0.8960 | 0.9049 | +| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6289 | 0.6555 | 0.6462 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3656 | 0.4119 | 0.4172 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7202 | 0.7941 | 0.7969 | Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: diff --git a/docs/regressions-dl21-doc-segmented-unicoil-0shot-v2.md b/docs/regressions-dl21-doc-segmented-unicoil-0shot-v2.md index 7239f89370..e5705aef31 100644 --- a/docs/regressions-dl21-doc-segmented-unicoil-0shot-v2.md +++ b/docs/regressions-dl21-doc-segmented-unicoil-0shot-v2.md @@ -71,7 +71,7 @@ target/appassembler/bin/IndexCollection \ -input /path/to/msmarco-v2-doc-segmented-unicoil-0shot-v2 \ -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ \ -generator DefaultLuceneDocumentGenerator \ - -threads 18 -impact -pretokenized \ + -threads 18 -impact -pretokenized -storeDocvectors \ >& logs/log.msmarco-v2-doc-segmented-unicoil-0shot-v2 & ``` @@ -97,6 +97,20 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot.topics.dl21.unicoil.0shot.txt \ -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt \ + -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt \ + -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -106,23 +120,33 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics- tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot.topics.dl21.unicoil.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot.topics.dl21.unicoil.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot.topics.dl21.unicoil.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **uniCOIL (with doc2query-T5) zero-shot**| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2718 | -| **MRR@100** | **uniCOIL (with doc2query-T5) zero-shot**| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.9684 | -| **nDCG@10** | **uniCOIL (with doc2query-T5) zero-shot**| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6783 | -| **R@100** | **uniCOIL (with doc2query-T5) zero-shot**| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3700 | -| **R@1000** | **uniCOIL (with doc2query-T5) zero-shot**| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7069 | +| **MAP@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2718 | 0.3297 | 0.3434 | +| **MRR@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.9684 | 0.9357 | 0.9649 | +| **nDCG@10** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6783 | 0.6979 | 0.7061 | +| **R@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3700 | 0.4237 | 0.4374 | +| **R@1000** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7069 | 0.7608 | 0.7809 | This run roughly corresponds to run `p_unicoil0` submitted to the TREC 2021 Deep Learning Track under the "baseline" group. The difference is that here we are using pre-encoded queries, whereas the official submission performed query encoding on the fly. diff --git a/docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot-v2.md b/docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot-v2.md index 4945139b95..4791927822 100644 --- a/docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot-v2.md +++ b/docs/regressions-dl21-doc-segmented-unicoil-noexp-0shot-v2.md @@ -71,7 +71,7 @@ target/appassembler/bin/IndexCollection \ -input /path/to/msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2 \ -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2/ \ -generator DefaultLuceneDocumentGenerator \ - -threads 18 -impact -pretokenized \ + -threads 18 -impact -pretokenized -storeDocvectors \ >& logs/log.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2 & ``` @@ -97,6 +97,20 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot.topics.dl21.unicoil-noexp.0shot.txt \ -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt \ + -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt \ + -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -106,23 +120,33 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics- tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot.topics.dl21.unicoil-noexp.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot.topics.dl21.unicoil-noexp.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot.topics.dl21.unicoil-noexp.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **uniCOIL (noexp) zero-shot**| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2587 | -| **MRR@100** | **uniCOIL (noexp) zero-shot**| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.9282 | -| **nDCG@10** | **uniCOIL (noexp) zero-shot**| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6495 | -| **R@100** | **uniCOIL (noexp) zero-shot**| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3563 | -| **R@1000** | **uniCOIL (noexp) zero-shot**| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6787 | +| **MAP@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2587 | 0.3130 | 0.3223 | +| **MRR@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.9282 | 0.9183 | 0.9448 | +| **nDCG@10** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6495 | 0.6686 | 0.6718 | +| **R@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3563 | 0.4044 | 0.4147 | +| **R@1000** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6787 | 0.7510 | 0.7589 | This run roughly corresponds to run `p_unicoil0` submitted to the TREC 2021 Deep Learning Track under the "baseline" group. The difference is that here we are using pre-encoded queries, whereas the official submission performed query encoding on the fly. diff --git a/docs/regressions-dl21-doc-segmented.md b/docs/regressions-dl21-doc-segmented.md index 6f87dc505e..aff361a425 100644 --- a/docs/regressions-dl21-doc-segmented.md +++ b/docs/regressions-dl21-doc-segmented.md @@ -69,20 +69,6 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented.bm25-default+rocchio.topics.dl21.txt \ -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -rocchio & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc-segmented/ \ - -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.dl21.txt \ - -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc-segmented/ \ - -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.dl21.txt \ - -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -bm25prf & ``` Evaluation can be performed using `trec_eval`: @@ -102,33 +88,23 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics- tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+rocchio.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+rocchio.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+rocchio.topics.dl21.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.dl21.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.dl21.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2436 | 0.2933 | 0.2974 | 0.2808 | 0.2729 | -| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.8937 | 0.9018 | 0.9099 | 0.9221 | 0.9146 | -| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5776 | 0.6185 | 0.6067 | 0.5840 | 0.5936 | -| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3478 | 0.3892 | 0.3957 | 0.3884 | 0.3778 | -| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6930 | 0.7694 | 0.7740 | 0.7934 | 0.7423 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2436 | 0.2933 | 0.2974 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.8937 | 0.9018 | 0.9099 | +| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5776 | 0.6185 | 0.6067 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3478 | 0.3892 | 0.3957 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6930 | 0.7694 | 0.7740 | Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: diff --git a/docs/regressions-dl21-doc.md b/docs/regressions-dl21-doc.md index 57697c55e1..53381a6360 100644 --- a/docs/regressions-dl21-doc.md +++ b/docs/regressions-dl21-doc.md @@ -69,20 +69,6 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc.bm25-default+rocchio.topics.dl21.txt \ -hits 1000 -bm25 -rocchio & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc/ \ - -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc.bm25-default+ax.topics.dl21.txt \ - -hits 1000 -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc/ \ - -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc.bm25-default+prf.topics.dl21.txt \ - -hits 1000 -bm25 -bm25prf & ``` Evaluation can be performed using `trec_eval`: @@ -102,33 +88,23 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics- tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+rocchio.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+rocchio.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+rocchio.topics.dl21.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.dl21.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-doc.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.dl21.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------| -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2126 | 0.2453 | 0.2467 | 0.2034 | 0.2079 | -| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.8367 | 0.7994 | 0.7997 | 0.7434 | 0.7869 | -| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5116 | 0.5339 | 0.5476 | 0.4804 | 0.4850 | -| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3195 | 0.3374 | 0.3456 | 0.3002 | 0.3096 | -| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6739 | 0.7335 | 0.7367 | 0.7089 | 0.7040 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2126 | 0.2453 | 0.2467 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.8367 | 0.7994 | 0.7997 | +| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5116 | 0.5339 | 0.5476 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3195 | 0.3374 | 0.3456 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Doc)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6739 | 0.7335 | 0.7367 | Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: diff --git a/docs/regressions-dl21-passage-augmented-d2q-t5.md b/docs/regressions-dl21-passage-augmented-d2q-t5.md index c4b5b826ad..77efd2941c 100644 --- a/docs/regressions-dl21-passage-augmented-d2q-t5.md +++ b/docs/regressions-dl21-passage-augmented-d2q-t5.md @@ -57,6 +57,13 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt \ -bm25 -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt \ + -bm25 -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -73,20 +80,26 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resou tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.dl21.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.dl21.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.1649 | 0.1932 | -| **MRR@100** | **BM25 (default)**| **+RM3** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6391 | 0.5882 | -| **nDCG@10** | **BM25 (default)**| **+RM3** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4702 | 0.4834 | -| **R@100** | **BM25 (default)**| **+RM3** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3883 | 0.4295 | -| **R@1000** | **BM25 (default)**| **+RM3** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6962 | 0.7668 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.1649 | 0.1932 | 0.1984 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6391 | 0.5882 | 0.6325 | +| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4702 | 0.4834 | 0.4907 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3883 | 0.4295 | 0.4294 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6962 | 0.7668 | 0.7680 | diff --git a/docs/regressions-dl21-passage-augmented.md b/docs/regressions-dl21-passage-augmented.md index 175544edca..bbf6ef305d 100644 --- a/docs/regressions-dl21-passage-augmented.md +++ b/docs/regressions-dl21-passage-augmented.md @@ -64,20 +64,6 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-augmented.bm25-default+rocchio.topics.dl21.txt \ -bm25 -rocchio & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage-augmented/ \ - -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.dl21.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage-augmented/ \ - -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.dl21.txt \ - -bm25 -bm25prf & ``` Evaluation can be performed using `trec_eval`: @@ -100,35 +86,23 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resou tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+rocchio.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+rocchio.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+rocchio.topics.dl21.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.dl21.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.dl21.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.0977 | 0.1050 | 0.1043 | 0.0849 | 0.0959 | -| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5303 | 0.4906 | 0.4809 | 0.4269 | 0.4737 | -| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3977 | 0.3906 | 0.3817 | 0.2927 | 0.3488 | -| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2709 | 0.2795 | 0.2819 | 0.2645 | 0.2729 | -| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5835 | 0.6268 | 0.6372 | 0.6085 | 0.5952 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.0977 | 0.1050 | 0.1043 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5303 | 0.4906 | 0.4809 | +| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3977 | 0.3906 | 0.3817 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2709 | 0.2795 | 0.2819 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5835 | 0.6268 | 0.6372 | Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: diff --git a/docs/regressions-dl21-passage-d2q-t5.md b/docs/regressions-dl21-passage-d2q-t5.md index 1dc1a127ba..872eef2d42 100644 --- a/docs/regressions-dl21-passage-d2q-t5.md +++ b/docs/regressions-dl21-passage-d2q-t5.md @@ -57,6 +57,13 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt \ -bm25 -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.dl21.txt \ + -bm25 -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -73,20 +80,26 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resou tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.dl21.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.dl21.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.dl21.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.1874 | 0.2271 | -| **MRR@100** | **BM25 (default)**| **+RM3** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6848 | 0.6651 | -| **nDCG@10** | **BM25 (default)**| **+RM3** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4816 | 0.5099 | -| **R@100** | **BM25 (default)**| **+RM3** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4076 | 0.4444 | -| **R@1000** | **BM25 (default)**| **+RM3** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7078 | 0.7512 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.1874 | 0.2271 | 0.2282 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6848 | 0.6651 | 0.6780 | +| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4816 | 0.5099 | 0.5159 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4076 | 0.4444 | 0.4444 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7078 | 0.7512 | 0.7548 | diff --git a/docs/regressions-dl21-passage-unicoil-0shot.md b/docs/regressions-dl21-passage-unicoil-0shot.md index 75fa35dba8..40718046d3 100644 --- a/docs/regressions-dl21-passage-unicoil-0shot.md +++ b/docs/regressions-dl21-passage-unicoil-0shot.md @@ -65,7 +65,7 @@ target/appassembler/bin/IndexCollection \ -input /path/to/msmarco-v2-passage-unicoil-0shot \ -index indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ \ -generator DefaultLuceneDocumentGenerator \ - -threads 18 -impact -pretokenized \ + -threads 18 -impact -pretokenized -storeDocvectors \ >& logs/log.msmarco-v2-passage-unicoil-0shot & ``` @@ -91,6 +91,20 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot.topics.dl21.unicoil.0shot.txt \ -impact -pretokenized & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt \ + -impact -pretokenized -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt \ + -impact -pretokenized -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -101,23 +115,35 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resou tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot.topics.dl21.unicoil.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot.topics.dl21.unicoil.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot.topics.dl21.unicoil.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.dl21.unicoil.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.dl21.unicoil.0shot.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **uniCOIL (with doc2query-T5) zero-shot**| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2538 | -| **MRR@100** | **uniCOIL (with doc2query-T5) zero-shot**| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7311 | -| **nDCG@10** | **uniCOIL (with doc2query-T5) zero-shot**| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6159 | -| **R@100** | **uniCOIL (with doc2query-T5) zero-shot**| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4731 | -| **R@1000** | **uniCOIL (with doc2query-T5) zero-shot**| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7551 | +| **MAP@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2538 | 0.2864 | 0.2890 | +| **MRR@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7311 | 0.7493 | 0.7749 | +| **nDCG@10** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6159 | 0.6150 | 0.6383 | +| **R@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4731 | 0.5141 | 0.5147 | +| **R@1000** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.7551 | 0.7906 | 0.8096 | This run roughly corresponds to run `d_unicoil0` submitted to the TREC 2021 Deep Learning Track under the "baseline" group. The difference is that here we are using pre-encoded queries, whereas the official submission performed query encoding on the fly. diff --git a/docs/regressions-dl21-passage-unicoil-noexp-0shot.md b/docs/regressions-dl21-passage-unicoil-noexp-0shot.md index f325ab00ed..38cb908458 100644 --- a/docs/regressions-dl21-passage-unicoil-noexp-0shot.md +++ b/docs/regressions-dl21-passage-unicoil-noexp-0shot.md @@ -65,7 +65,7 @@ target/appassembler/bin/IndexCollection \ -input /path/to/msmarco-v2-passage-unicoil-noexp-0shot \ -index indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ \ -generator DefaultLuceneDocumentGenerator \ - -threads 18 -impact -pretokenized \ + -threads 18 -impact -pretokenized -storeDocvectors \ >& logs/log.msmarco-v2-passage-unicoil-noexp-0shot & ``` @@ -91,6 +91,20 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot.topics.dl21.unicoil-noexp.0shot.txt \ -impact -pretokenized & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.dl21.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -101,23 +115,35 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resou tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot.topics.dl21.unicoil-noexp.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot.topics.dl21.unicoil-noexp.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot.topics.dl21.unicoil-noexp.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.dl21.unicoil-noexp.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.dl21.unicoil-noexp.0shot.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **uniCOIL (noexp) zero-shot**| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2193 | -| **MRR@100** | **uniCOIL (noexp) zero-shot**| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6991 | -| **nDCG@10** | **uniCOIL (noexp) zero-shot**| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5756 | -| **R@100** | **uniCOIL (noexp) zero-shot**| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4246 | -| **R@1000** | **uniCOIL (noexp) zero-shot**| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6897 | +| **MAP@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.2193 | 0.2598 | 0.2666 | +| **MRR@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6991 | 0.7143 | 0.7526 | +| **nDCG@10** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5756 | 0.5937 | 0.6088 | +| **R@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4246 | 0.4809 | 0.4728 | +| **R@1000** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6897 | 0.7310 | 0.7509 | This run roughly corresponds to run `d_unicoil0` submitted to the TREC 2021 Deep Learning Track under the "baseline" group. The difference is that here we are using pre-encoded queries, whereas the official submission performed query encoding on the fly. diff --git a/docs/regressions-dl21-passage.md b/docs/regressions-dl21-passage.md index 2a9650ef1d..332377bc19 100644 --- a/docs/regressions-dl21-passage.md +++ b/docs/regressions-dl21-passage.md @@ -64,20 +64,6 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage.bm25-default+rocchio.topics.dl21.txt \ -bm25 -rocchio & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage/ \ - -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage.bm25-default+ax.topics.dl21.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage/ \ - -topics src/main/resources/topics-and-qrels/topics.dl21.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage.bm25-default+prf.topics.dl21.txt \ - -bm25 -bm25prf & ``` Evaluation can be performed using `trec_eval`: @@ -100,35 +86,23 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resou tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+rocchio.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+rocchio.topics.dl21.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+rocchio.topics.dl21.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.dl21.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m recip_rank -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m ndcg_cut.10 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.dl21.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 -l 2 src/main/resources/topics-and-qrels/qrels.dl21-passage.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.dl21.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------| -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.1357 | 0.1632 | 0.1708 | 0.1907 | 0.1821 | -| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5060 | 0.4925 | 0.5432 | 0.5733 | 0.5532 | -| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4458 | 0.4480 | 0.4560 | 0.4851 | 0.4740 | -| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3261 | 0.3498 | 0.3600 | 0.3803 | 0.3745 | -| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6149 | 0.6619 | 0.6710 | 0.6882 | 0.6643 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.1357 | 0.1632 | 0.1708 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.5060 | 0.4925 | 0.5432 | +| **nDCG@10** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.4458 | 0.4480 | 0.4560 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.3261 | 0.3498 | 0.3600 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [DL21 (Passage)](https://microsoft.github.io/msmarco/TREC-Deep-Learning) | 0.6149 | 0.6619 | 0.6710 | Some of these regressions correspond to official TREC 2021 Deep Learning Track "baseline" submissions: diff --git a/docs/regressions-msmarco-v2-doc-d2q-t5.md b/docs/regressions-msmarco-v2-doc-d2q-t5.md index 06fa9323e2..22065a73be 100644 --- a/docs/regressions-msmarco-v2-doc-d2q-t5.md +++ b/docs/regressions-msmarco-v2-doc-d2q-t5.md @@ -66,6 +66,19 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt \ -bm25 -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev.txt \ + -bm25 -rocchio & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt \ + -bm25 -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -84,22 +97,29 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1988 | 0.1128 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1986 | 0.1151 | -| **MRR@100** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2011 | 0.1144 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2012 | 0.1169 | -| **R@100** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6786 | 0.5935 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6821 | 0.5898 | -| **R@1000** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8614 | 0.8189 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8568 | 0.8245 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1988 | 0.1128 | 0.1152 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1986 | 0.1151 | 0.1171 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2011 | 0.1144 | 0.1168 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2012 | 0.1169 | 0.1187 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6786 | 0.5935 | 0.5961 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6821 | 0.5898 | 0.5961 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8614 | 0.8189 | 0.8199 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8568 | 0.8245 | 0.8288 | diff --git a/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md b/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md index 7484d24b19..3a68560afe 100644 --- a/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md +++ b/docs/regressions-msmarco-v2-doc-segmented-d2q-t5.md @@ -66,6 +66,19 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt \ -bm25 -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev.txt \ + -bm25 -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt \ + -bm25 -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & ``` Evaluation can be performed using `trec_eval`: @@ -84,22 +97,29 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-doc.dev2.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2203 | 0.1955 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2205 | 0.1939 | -| **MRR@100** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2226 | 0.1974 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2234 | 0.1966 | -| **R@100** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7297 | 0.7104 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7316 | 0.7164 | -| **R@1000** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8982 | 0.9000 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8952 | 0.8967 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2203 | 0.1955 | 0.1959 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2205 | 0.1939 | 0.1938 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2226 | 0.1974 | 0.1980 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2234 | 0.1966 | 0.1965 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7297 | 0.7104 | 0.7136 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7316 | 0.7164 | 0.7197 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8982 | 0.9000 | 0.9028 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8952 | 0.8967 | 0.8976 | diff --git a/docs/regressions-msmarco-v2-doc-segmented-unicoil-0shot-v2.md b/docs/regressions-msmarco-v2-doc-segmented-unicoil-0shot-v2.md index c5426ed886..5662da170e 100644 --- a/docs/regressions-msmarco-v2-doc-segmented-unicoil-0shot-v2.md +++ b/docs/regressions-msmarco-v2-doc-segmented-unicoil-0shot-v2.md @@ -68,7 +68,7 @@ target/appassembler/bin/IndexCollection \ -input /path/to/msmarco-v2-doc-segmented-unicoil-0shot-v2 \ -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ \ -generator DefaultLuceneDocumentGenerator \ - -threads 18 -impact -pretokenized \ + -threads 18 -impact -pretokenized -storeDocvectors \ >& logs/log.msmarco-v2-doc-segmented-unicoil-0shot-v2 & ``` @@ -99,6 +99,32 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt \ -impact -pretokenized -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.msmarco-v2-doc.dev.unicoil.0shot.txt \ + -impact -pretokenized -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt \ + -impact -pretokenized -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.msmarco-v2-doc.dev.unicoil.0shot.txt \ + -impact -pretokenized -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt \ + -impact -pretokenized -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & ``` Evaluation can be performed using `trec_eval`: @@ -110,25 +136,39 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.msmarco-v2-doc.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.msmarco-v2-doc.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.msmarco-v2-doc.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rm3.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.msmarco-v2-doc.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.msmarco-v2-doc.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.msmarco-v2-doc.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-0shot-v2.unicoil-0shot+rocchio.topics.msmarco-v2-doc.dev2.unicoil.0shot.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **uniCOIL (with doc2query-T5) zero-shot**| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2388 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2422 | -| **MRR@100** | **uniCOIL (with doc2query-T5) zero-shot**| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2419 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2445 | -| **R@100** | **uniCOIL (with doc2query-T5) zero-shot**| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7791 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7759 | -| **R@1000** | **uniCOIL (with doc2query-T5) zero-shot**| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.9122 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.9172 | +| **MAP@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2388 | 0.2167 | 0.2229 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2422 | 0.2198 | 0.2200 | +| **MRR@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2419 | 0.2190 | 0.2252 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2445 | 0.2225 | 0.2225 | +| **R@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7791 | 0.7675 | 0.7775 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7759 | 0.7639 | 0.7747 | +| **R@1000** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.9122 | 0.9175 | 0.9232 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.9172 | 0.9220 | 0.9253 | ## Reproduction Log[*](reproducibility.md) diff --git a/docs/regressions-msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.md b/docs/regressions-msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.md index 5789097e25..4bd2e18663 100644 --- a/docs/regressions-msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.md +++ b/docs/regressions-msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.md @@ -68,7 +68,7 @@ target/appassembler/bin/IndexCollection \ -input /path/to/msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2 \ -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2/ \ -generator DefaultLuceneDocumentGenerator \ - -threads 18 -impact -pretokenized \ + -threads 18 -impact -pretokenized -storeDocvectors \ >& logs/log.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2 & ``` @@ -99,6 +99,32 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt \ -impact -pretokenized -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & ``` Evaluation can be performed using `trec_eval`: @@ -110,25 +136,39 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rm3.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-doc.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-doc.dev2.unicoil-noexp.0shot.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **uniCOIL (noexp) zero-shot**| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2205 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2291 | -| **MRR@100** | **uniCOIL (noexp) zero-shot**| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2231 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2314 | -| **R@100** | **uniCOIL (noexp) zero-shot**| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7460 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7498 | -| **R@1000** | **uniCOIL (noexp) zero-shot**| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8987 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8995 | +| **MAP@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2205 | 0.1963 | 0.2011 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2291 | 0.2067 | 0.2090 | +| **MRR@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2231 | 0.1982 | 0.2034 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.2314 | 0.2092 | 0.2112 | +| **R@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7460 | 0.7453 | 0.7520 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7498 | 0.7466 | 0.7540 | +| **R@1000** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8987 | 0.9026 | 0.9084 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8995 | 0.9082 | 0.9136 | ## Reproduction Log[*](reproducibility.md) diff --git a/docs/regressions-msmarco-v2-doc-segmented.md b/docs/regressions-msmarco-v2-doc-segmented.md index dda6c4f47e..3641dcd7ea 100644 --- a/docs/regressions-msmarco-v2-doc-segmented.md +++ b/docs/regressions-msmarco-v2-doc-segmented.md @@ -80,32 +80,6 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc-segmented.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt \ -bm25 -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc-segmented/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.msmarco-v2-doc.dev.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc-segmented/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.msmarco-v2-doc.dev2.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc-segmented/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.msmarco-v2-doc.dev.txt \ - -bm25 -bm25prf -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc-segmented/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.msmarco-v2-doc.dev2.txt \ - -bm25 -bm25prf -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 & ``` Evaluation can be performed using `trec_eval`: @@ -131,36 +105,22 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.msmarco-v2-doc.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.msmarco-v2-doc.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+ax.topics.msmarco-v2-doc.dev2.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.msmarco-v2-doc.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.msmarco-v2-doc.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc-segmented.bm25-default+prf.topics.msmarco-v2-doc.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1875 | 0.1648 | 0.1660 | 0.1344 | 0.1528 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1903 | 0.1679 | 0.1689 | 0.1345 | 0.1565 | -| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1896 | 0.1664 | 0.1679 | 0.1360 | 0.1547 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1930 | 0.1701 | 0.1710 | 0.1366 | 0.1587 | -| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6555 | 0.6558 | 0.6535 | 0.5593 | 0.5945 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6629 | 0.6561 | 0.6618 | 0.5800 | 0.6134 | -| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8542 | 0.8605 | 0.8659 | 0.8161 | 0.8270 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8549 | 0.8633 | 0.8642 | 0.8210 | 0.8280 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1875 | 0.1648 | 0.1660 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1903 | 0.1679 | 0.1689 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1896 | 0.1664 | 0.1679 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1930 | 0.1701 | 0.1710 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6555 | 0.6558 | 0.6535 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6629 | 0.6561 | 0.6618 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8542 | 0.8605 | 0.8659 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8549 | 0.8633 | 0.8642 | diff --git a/docs/regressions-msmarco-v2-doc.md b/docs/regressions-msmarco-v2-doc.md index 706fafdd87..2ad94a940f 100644 --- a/docs/regressions-msmarco-v2-doc.md +++ b/docs/regressions-msmarco-v2-doc.md @@ -80,32 +80,6 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-doc.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt \ -bm25 -rocchio & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc.bm25-default+ax.topics.msmarco-v2-doc.dev.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc.bm25-default+ax.topics.msmarco-v2-doc.dev2.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc.bm25-default+prf.topics.msmarco-v2-doc.dev.txt \ - -bm25 -bm25prf & -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-doc/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-doc.dev2.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-doc.bm25-default+prf.topics.msmarco-v2-doc.dev2.txt \ - -bm25 -bm25prf & ``` Evaluation can be performed using `trec_eval`: @@ -131,36 +105,22 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+rocchio.topics.msmarco-v2-doc.dev2.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.msmarco-v2-doc.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.msmarco-v2-doc.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+ax.topics.msmarco-v2-doc.dev2.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.msmarco-v2-doc.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.msmarco-v2-doc.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.msmarco-v2-doc.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-doc.dev2.txt runs/run.msmarco-v2-doc.bm25-default+prf.topics.msmarco-v2-doc.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------| -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1552 | 0.0966 | 0.0965 | 0.0665 | 0.0834 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1639 | 0.1011 | 0.1037 | 0.0722 | 0.0876 | -| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1572 | 0.0974 | 0.0974 | 0.0675 | 0.0845 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1659 | 0.1028 | 0.1052 | 0.0733 | 0.0892 | -| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5956 | 0.5121 | 0.5135 | 0.4075 | 0.4681 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5970 | 0.5245 | 0.5259 | 0.4192 | 0.4831 | -| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8054 | 0.7694 | 0.7697 | 0.6852 | 0.7385 | -| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8029 | 0.7736 | 0.7762 | 0.6960 | 0.7482 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1552 | 0.0966 | 0.0965 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1639 | 0.1011 | 0.1037 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1572 | 0.0974 | 0.0974 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1659 | 0.1028 | 0.1052 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5956 | 0.5121 | 0.5135 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5970 | 0.5245 | 0.5259 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Doc: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8054 | 0.7694 | 0.7697 | +| [MS MARCO V2 Doc: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.8029 | 0.7736 | 0.7762 | diff --git a/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md b/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md index e73b131403..21fae6ca31 100644 --- a/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md +++ b/docs/regressions-msmarco-v2-passage-augmented-d2q-t5.md @@ -65,6 +65,19 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt \ -bm25 -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev.txt \ + -bm25 -rocchio & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-augmented-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt \ + -bm25 -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -83,22 +96,29 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1160 | 0.0870 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1158 | 0.0879 | -| **MRR@100** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1172 | 0.0880 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1170 | 0.0887 | -| **R@100** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5039 | 0.4742 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5158 | 0.4905 | -| **R@1000** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7647 | 0.7625 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7659 | 0.7649 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1160 | 0.0870 | 0.0882 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1158 | 0.0879 | 0.0900 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1172 | 0.0880 | 0.0893 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1170 | 0.0887 | 0.0908 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5039 | 0.4742 | 0.4754 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5158 | 0.4905 | 0.4880 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7647 | 0.7625 | 0.7648 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7659 | 0.7649 | 0.7666 | diff --git a/docs/regressions-msmarco-v2-passage-augmented.md b/docs/regressions-msmarco-v2-passage-augmented.md index 8fed8641e3..aa051c425a 100644 --- a/docs/regressions-msmarco-v2-passage-augmented.md +++ b/docs/regressions-msmarco-v2-passage-augmented.md @@ -80,32 +80,6 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-augmented.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt \ -bm25 -rocchio & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage-augmented/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.msmarco-v2-passage.dev.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage-augmented/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.msmarco-v2-passage.dev2.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage-augmented/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.msmarco-v2-passage.dev.txt \ - -bm25 -bm25prf & -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage-augmented/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.msmarco-v2-passage.dev2.txt \ - -bm25 -bm25prf & ``` Evaluation can be performed using `trec_eval`: @@ -131,36 +105,22 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.msmarco-v2-passage.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.msmarco-v2-passage.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+ax.topics.msmarco-v2-passage.dev2.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.msmarco-v2-passage.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.msmarco-v2-passage.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-augmented.bm25-default+prf.topics.msmarco-v2-passage.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0863 | 0.0669 | 0.0665 | 0.0431 | 0.0542 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0904 | 0.0692 | 0.0675 | 0.0442 | 0.0563 | -| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0872 | 0.0674 | 0.0671 | 0.0434 | 0.0547 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0917 | 0.0701 | 0.0684 | 0.0446 | 0.0572 | -| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4030 | 0.3734 | 0.3755 | 0.2998 | 0.3338 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4159 | 0.3852 | 0.3878 | 0.3220 | 0.3505 | -| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6925 | 0.6859 | 0.6912 | 0.5887 | 0.6157 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6933 | 0.6838 | 0.6864 | 0.6043 | 0.6351 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0863 | 0.0669 | 0.0665 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0904 | 0.0692 | 0.0675 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0872 | 0.0674 | 0.0671 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0917 | 0.0701 | 0.0684 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4030 | 0.3734 | 0.3755 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4159 | 0.3852 | 0.3878 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6925 | 0.6859 | 0.6912 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.6933 | 0.6838 | 0.6864 | diff --git a/docs/regressions-msmarco-v2-passage-d2q-t5.md b/docs/regressions-msmarco-v2-passage-d2q-t5.md index 194fd9e5fa..bfbabb02d0 100644 --- a/docs/regressions-msmarco-v2-passage-d2q-t5.md +++ b/docs/regressions-msmarco-v2-passage-d2q-t5.md @@ -65,6 +65,19 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt \ -bm25 -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev.txt \ + -bm25 -rocchio & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-d2q-t5/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt \ + -bm25 -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -83,22 +96,29 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rm3.topics.msmarco-v2-passage.dev2.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-d2q-t5.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1057 | 0.0939 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1112 | 0.0979 | -| **MRR@100** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1072 | 0.0948 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1123 | 0.0985 | -| **R@100** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4670 | 0.4723 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4803 | 0.4800 | -| **R@1000** | **BM25 (default)**| **+RM3** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7083 | 0.7174 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7151 | 0.7240 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1057 | 0.0939 | 0.0943 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1112 | 0.0979 | 0.0990 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1072 | 0.0948 | 0.0952 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1123 | 0.0985 | 0.0997 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4670 | 0.4723 | 0.4725 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4803 | 0.4800 | 0.4815 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7083 | 0.7174 | 0.7211 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7151 | 0.7240 | 0.7274 | diff --git a/docs/regressions-msmarco-v2-passage-unicoil-0shot.md b/docs/regressions-msmarco-v2-passage-unicoil-0shot.md index 9be8c8f470..071d233e9b 100644 --- a/docs/regressions-msmarco-v2-passage-unicoil-0shot.md +++ b/docs/regressions-msmarco-v2-passage-unicoil-0shot.md @@ -62,7 +62,7 @@ target/appassembler/bin/IndexCollection \ -input /path/to/msmarco-v2-passage-unicoil-0shot \ -index indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ \ -generator DefaultLuceneDocumentGenerator \ - -threads 18 -impact -pretokenized \ + -threads 18 -impact -pretokenized -storeDocvectors \ >& logs/log.msmarco-v2-passage-unicoil-0shot & ``` @@ -93,6 +93,32 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt \ -impact -pretokenized & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.msmarco-v2-passage.dev.unicoil.0shot.txt \ + -impact -pretokenized -rm3 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt \ + -impact -pretokenized -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.msmarco-v2-passage.dev.unicoil.0shot.txt \ + -impact -pretokenized -rocchio & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.unicoil.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt \ + -impact -pretokenized -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -104,25 +130,39 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.msmarco-v2-passage.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.msmarco-v2-passage.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.msmarco-v2-passage.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rm3.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.msmarco-v2-passage.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.msmarco-v2-passage.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.msmarco-v2-passage.dev.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-0shot.unicoil-0shot+rocchio.topics.msmarco-v2-passage.dev2.unicoil.0shot.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **uniCOIL (with doc2query-T5) zero-shot**| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1485 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1561 | -| **MRR@100** | **uniCOIL (with doc2query-T5) zero-shot**| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1499 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1577 | -| **R@100** | **uniCOIL (with doc2query-T5) zero-shot**| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5518 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5661 | -| **R@1000** | **uniCOIL (with doc2query-T5) zero-shot**| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7616 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7671 | +| **MAP@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1485 | 0.1320 | 0.1319 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1561 | 0.1339 | 0.1371 | +| **MRR@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1499 | 0.1332 | 0.1334 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1577 | 0.1348 | 0.1380 | +| **R@100** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5518 | 0.5534 | 0.5557 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5661 | 0.5550 | 0.5696 | +| **R@1000** | **uniCOIL (with doc2query-T5) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7616 | 0.7731 | 0.7800 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7671 | 0.7698 | 0.7849 | ## Reproduction Log[*](reproducibility.md) diff --git a/docs/regressions-msmarco-v2-passage-unicoil-noexp-0shot.md b/docs/regressions-msmarco-v2-passage-unicoil-noexp-0shot.md index 0092593ce6..a60a8322f2 100644 --- a/docs/regressions-msmarco-v2-passage-unicoil-noexp-0shot.md +++ b/docs/regressions-msmarco-v2-passage-unicoil-noexp-0shot.md @@ -62,7 +62,7 @@ target/appassembler/bin/IndexCollection \ -input /path/to/msmarco-v2-passage-unicoil-noexp-0shot \ -index indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ \ -generator DefaultLuceneDocumentGenerator \ - -threads 18 -impact -pretokenized \ + -threads 18 -impact -pretokenized -storeDocvectors \ >& logs/log.msmarco-v2-passage-unicoil-noexp-0shot & ``` @@ -93,6 +93,32 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt \ -impact -pretokenized & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rm3 & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rm3 & + +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rocchio & +target/appassembler/bin/SearchCollection \ + -index indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ \ + -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.tsv.gz \ + -topicreader TsvInt \ + -output runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt \ + -impact -pretokenized -rocchio & ``` Evaluation can be performed using `trec_eval`: @@ -104,25 +130,39 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rm3.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt + +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-passage.dev.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt +tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage-unicoil-noexp-0shot.unicoil-noexp-0shot+rocchio.topics.msmarco-v2-passage.dev2.unicoil-noexp.0shot.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **uniCOIL (noexp) zero-shot**| -|:-------------------------------------------------------------------------------------------------------------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1333 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1374 | -| **MRR@100** | **uniCOIL (noexp) zero-shot**| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1342 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1385 | -| **R@100** | **uniCOIL (noexp) zero-shot**| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4976 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5217 | -| **R@1000** | **uniCOIL (noexp) zero-shot**| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7010 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7114 | +| **MAP@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1333 | 0.1118 | 0.1163 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1374 | 0.1189 | 0.1179 | +| **MRR@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1342 | 0.1127 | 0.1172 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.1385 | 0.1196 | 0.1186 | +| **R@100** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.4976 | 0.4998 | 0.5082 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5217 | 0.5134 | 0.5218 | +| **R@1000** | **uniCOIL (noexp) zero-shot**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7010 | 0.7248 | 0.7299 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.7114 | 0.7344 | 0.7387 | ## Reproduction Log[*](reproducibility.md) diff --git a/docs/regressions-msmarco-v2-passage.md b/docs/regressions-msmarco-v2-passage.md index b2f121135a..7ff218ad14 100644 --- a/docs/regressions-msmarco-v2-passage.md +++ b/docs/regressions-msmarco-v2-passage.md @@ -80,32 +80,6 @@ target/appassembler/bin/SearchCollection \ -topicreader TsvInt \ -output runs/run.msmarco-v2-passage.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt \ -bm25 -rocchio & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage.bm25-default+ax.topics.msmarco-v2-passage.dev.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage.bm25-default+ax.topics.msmarco-v2-passage.dev2.txt \ - -bm25 -axiom -axiom.deterministic -rerankCutoff 20 & - -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage.bm25-default+prf.topics.msmarco-v2-passage.dev.txt \ - -bm25 -bm25prf & -target/appassembler/bin/SearchCollection \ - -index indexes/lucene-index.msmarco-v2-passage/ \ - -topics src/main/resources/topics-and-qrels/topics.msmarco-v2-passage.dev2.txt \ - -topicreader TsvInt \ - -output runs/run.msmarco-v2-passage.bm25-default+prf.topics.msmarco-v2-passage.dev2.txt \ - -bm25 -bm25prf & ``` Evaluation can be performed using `trec_eval`: @@ -131,36 +105,22 @@ tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank sr tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+rocchio.topics.msmarco-v2-passage.dev2.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.msmarco-v2-passage.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.msmarco-v2-passage.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+ax.topics.msmarco-v2-passage.dev2.txt - -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.msmarco-v2-passage.dev.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.100 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.msmarco-v2-passage.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -m recall.1000 src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.msmarco-v2-passage.dev2.txt -tools/eval/trec_eval.9.0.4/trec_eval -c -M 100 -m map -c -M 100 -m recip_rank src/main/resources/topics-and-qrels/qrels.msmarco-v2-passage.dev2.txt runs/run.msmarco-v2-passage.bm25-default+prf.topics.msmarco-v2-passage.dev2.txt ``` ## Effectiveness With the above commands, you should be able to reproduce the following results: -| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------|-----------|-----------| -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0709 | 0.0611 | 0.0625 | 0.0592 | 0.0595 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0794 | 0.0647 | 0.0662 | 0.0642 | 0.0632 | -| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0719 | 0.0619 | 0.0634 | 0.0601 | 0.0607 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0802 | 0.0654 | 0.0670 | 0.0647 | 0.0640 | -| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.3397 | 0.3377 | 0.3415 | 0.3482 | 0.3495 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.3459 | 0.3435 | 0.3516 | 0.3554 | 0.3595 | -| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| **+Ax** | **+PRF** | -| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5733 | 0.5933 | 0.5966 | 0.6064 | 0.5968 | -| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5839 | 0.6049 | 0.6104 | 0.6254 | 0.6169 | +| **MAP@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +|:-------------------------------------------------------------------------------------------------------------|-----------|-----------|-----------| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0709 | 0.0611 | 0.0625 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0794 | 0.0647 | 0.0662 | +| **MRR@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0719 | 0.0619 | 0.0634 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.0802 | 0.0654 | 0.0670 | +| **R@100** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.3397 | 0.3377 | 0.3415 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.3459 | 0.3435 | 0.3516 | +| **R@1000** | **BM25 (default)**| **+RM3** | **+Rocchio**| +| [MS MARCO V2 Passage: Dev](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5733 | 0.5933 | 0.5966 | +| [MS MARCO V2 Passage: Dev2](https://microsoft.github.io/msmarco/TREC-Deep-Learning.html) | 0.5839 | 0.6049 | 0.6104 | diff --git a/src/main/resources/regression/dl21-doc-d2q-t5.yaml b/src/main/resources/regression/dl21-doc-d2q-t5.yaml index 3027f36655..5a3726fc40 100644 --- a/src/main/resources/regression/dl21-doc-d2q-t5.yaml +++ b/src/main/resources/regression/dl21-doc-d2q-t5.yaml @@ -87,3 +87,17 @@ models: - 0.3580 R@1000: - 0.7572 + - name: bm25-default+rocchio + display: +Rocchio + params: -hits 1000 -bm25 -rocchio + results: + MAP@100: + - 0.2610 + MRR@100: + - 0.8459 + nDCG@10: + - 0.5509 + R@100: + - 0.3616 + R@1000: + - 0.7583 diff --git a/src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml b/src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml index b5b55370d6..102d3cdcac 100644 --- a/src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml +++ b/src/main/resources/regression/dl21-doc-segmented-d2q-t5.yaml @@ -87,3 +87,17 @@ models: - 0.4119 R@1000: - 0.7941 + - name: bm25-default+rocchio + display: +Rocchio + params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -rocchio + results: + MAP@100: + - 0.3218 + MRR@100: + - 0.9049 + nDCG@10: + - 0.6462 + R@100: + - 0.4172 + R@1000: + - 0.7969 diff --git a/src/main/resources/regression/dl21-doc-segmented-unicoil-0shot-v2.yaml b/src/main/resources/regression/dl21-doc-segmented-unicoil-0shot-v2.yaml index 1e9b4465a4..aeed061fdd 100644 --- a/src/main/resources/regression/dl21-doc-segmented-unicoil-0shot-v2.yaml +++ b/src/main/resources/regression/dl21-doc-segmented-unicoil-0shot-v2.yaml @@ -10,7 +10,7 @@ index_path: indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ collection_class: JsonVectorCollection generator_class: DefaultLuceneDocumentGenerator index_threads: 18 -index_options: -impact -pretokenized +index_options: -impact -pretokenized -storeDocvectors index_stats: documents: 124131414 documents (non-empty): 124131414 @@ -77,3 +77,31 @@ models: - 0.3700 R@1000: - 0.7069 + - name: unicoil-0shot+rm3 + display: +RM3 + params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized -rm3 + results: + MAP@100: + - 0.3297 + MRR@100: + - 0.9357 + nDCG@10: + - 0.6979 + R@100: + - 0.4237 + R@1000: + - 0.7608 + - name: unicoil-0shot+rocchio + display: +Rocchio + params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized -rocchio + results: + MAP@100: + - 0.3434 + MRR@100: + - 0.9649 + nDCG@10: + - 0.7061 + R@100: + - 0.4374 + R@1000: + - 0.7809 diff --git a/src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot-v2.yaml b/src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot-v2.yaml index 092c2d1782..667cb46288 100644 --- a/src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot-v2.yaml +++ b/src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot-v2.yaml @@ -10,7 +10,7 @@ index_path: indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2 collection_class: JsonVectorCollection generator_class: DefaultLuceneDocumentGenerator index_threads: 18 -index_options: -impact -pretokenized +index_options: -impact -pretokenized -storeDocvectors index_stats: documents: 124131404 documents (non-empty): 124131404 @@ -76,4 +76,32 @@ models: R@100: - 0.3563 R@1000: - - 0.6787 \ No newline at end of file + - 0.6787 + - name: unicoil-noexp-0shot+rm3 + display: +RM3 + params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized -rm3 + results: + MAP@100: + - 0.3130 + MRR@100: + - 0.9183 + nDCG@10: + - 0.6686 + R@100: + - 0.4044 + R@1000: + - 0.7510 + - name: unicoil-noexp-0shot+rocchio + display: +Rocchio + params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -impact -pretokenized -rocchio + results: + MAP@100: + - 0.3223 + MRR@100: + - 0.9448 + nDCG@10: + - 0.6718 + R@100: + - 0.4147 + R@1000: + - 0.7589 diff --git a/src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot.yaml b/src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot.yaml index 0bb953ff4d..f3e3b0efad 100644 --- a/src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot.yaml +++ b/src/main/resources/regression/dl21-doc-segmented-unicoil-noexp-0shot.yaml @@ -76,4 +76,4 @@ models: R@100: - 0.3497 R@1000: - - 0.6767 \ No newline at end of file + - 0.6767 diff --git a/src/main/resources/regression/dl21-doc-segmented.yaml b/src/main/resources/regression/dl21-doc-segmented.yaml index 62557d9752..f36208a11b 100644 --- a/src/main/resources/regression/dl21-doc-segmented.yaml +++ b/src/main/resources/regression/dl21-doc-segmented.yaml @@ -101,31 +101,3 @@ models: - 0.3957 R@1000: - 0.7740 - - name: bm25-default+ax - display: +Ax - params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -axiom -axiom.deterministic -rerankCutoff 20 - results: - MAP@100: - - 0.2808 - MRR@100: - - 0.9221 - nDCG@10: - - 0.5840 - R@100: - - 0.3884 - R@1000: - - 0.7934 - - name: bm25-default+prf - display: +PRF - params: -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 -bm25 -bm25prf - results: - MAP@100: - - 0.2729 - MRR@100: - - 0.9146 - nDCG@10: - - 0.5936 - R@100: - - 0.3778 - R@1000: - - 0.7423 diff --git a/src/main/resources/regression/dl21-doc.yaml b/src/main/resources/regression/dl21-doc.yaml index 14861f6873..611cf57560 100644 --- a/src/main/resources/regression/dl21-doc.yaml +++ b/src/main/resources/regression/dl21-doc.yaml @@ -101,31 +101,3 @@ models: - 0.3456 R@1000: - 0.7367 - - name: bm25-default+ax - display: +Ax - params: -hits 1000 -bm25 -axiom -axiom.deterministic -rerankCutoff 20 - results: - MAP@100: - - 0.2034 - MRR@100: - - 0.7434 - nDCG@10: - - 0.4804 - R@100: - - 0.3002 - R@1000: - - 0.7089 - - name: bm25-default+prf - display: +PRF - params: -hits 1000 -bm25 -bm25prf - results: - MAP@100: - - 0.2079 - MRR@100: - - 0.7869 - nDCG@10: - - 0.4850 - R@100: - - 0.3096 - R@1000: - - 0.7040 diff --git a/src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml b/src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml index ae19e5826c..62301714b3 100644 --- a/src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml +++ b/src/main/resources/regression/dl21-passage-augmented-d2q-t5.yaml @@ -87,3 +87,17 @@ models: - 0.4295 R@1000: - 0.7668 + - name: bm25-default+rocchio + display: +Rocchio + params: -bm25 -rocchio + results: + MAP@100: + - 0.1984 + MRR@100: + - 0.6325 + nDCG@10: + - 0.4907 + R@100: + - 0.4294 + R@1000: + - 0.7680 diff --git a/src/main/resources/regression/dl21-passage-augmented.yaml b/src/main/resources/regression/dl21-passage-augmented.yaml index 8a0963c871..b72d2b4efe 100644 --- a/src/main/resources/regression/dl21-passage-augmented.yaml +++ b/src/main/resources/regression/dl21-passage-augmented.yaml @@ -101,31 +101,3 @@ models: - 0.2819 R@1000: - 0.6372 - - name: bm25-default+ax - display: +Ax - params: -bm25 -axiom -axiom.deterministic -rerankCutoff 20 - results: - MAP@100: - - 0.0849 - MRR@100: - - 0.4269 - nDCG@10: - - 0.2927 - R@100: - - 0.2645 - R@1000: - - 0.6085 - - name: bm25-default+prf - display: +PRF - params: -bm25 -bm25prf - results: - MAP@100: - - 0.0959 - MRR@100: - - 0.4737 - nDCG@10: - - 0.3488 - R@100: - - 0.2729 - R@1000: - - 0.5952 diff --git a/src/main/resources/regression/dl21-passage-d2q-t5.yaml b/src/main/resources/regression/dl21-passage-d2q-t5.yaml index f83b0c07e9..22477217ed 100644 --- a/src/main/resources/regression/dl21-passage-d2q-t5.yaml +++ b/src/main/resources/regression/dl21-passage-d2q-t5.yaml @@ -87,3 +87,17 @@ models: - 0.4444 R@1000: - 0.7512 + - name: bm25-default+rocchio + display: +Rocchio + params: -bm25 -rocchio + results: + MAP@100: + - 0.2282 + MRR@100: + - 0.6780 + nDCG@10: + - 0.5159 + R@100: + - 0.4444 + R@1000: + - 0.7548 diff --git a/src/main/resources/regression/dl21-passage-unicoil-0shot.yaml b/src/main/resources/regression/dl21-passage-unicoil-0shot.yaml index e46b4f1c1a..0cdcf8d4bb 100644 --- a/src/main/resources/regression/dl21-passage-unicoil-0shot.yaml +++ b/src/main/resources/regression/dl21-passage-unicoil-0shot.yaml @@ -10,7 +10,7 @@ index_path: indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ collection_class: JsonVectorCollection generator_class: DefaultLuceneDocumentGenerator index_threads: 18 -index_options: -impact -pretokenized +index_options: -impact -pretokenized -storeDocvectors index_stats: documents: 138364198 documents (non-empty): 138364198 @@ -77,3 +77,31 @@ models: - 0.4731 R@1000: - 0.7551 + - name: unicoil-0shot+rm3 + display: +RM3 + params: -impact -pretokenized -rm3 + results: + MAP@100: + - 0.2864 + MRR@100: + - 0.7493 + nDCG@10: + - 0.6150 + R@100: + - 0.5141 + R@1000: + - 0.7906 + - name: unicoil-0shot+rocchio + display: +Rocchio + params: -impact -pretokenized -rocchio + results: + MAP@100: + - 0.2890 + MRR@100: + - 0.7749 + nDCG@10: + - 0.6383 + R@100: + - 0.5147 + R@1000: + - 0.8096 diff --git a/src/main/resources/regression/dl21-passage-unicoil-noexp-0shot.yaml b/src/main/resources/regression/dl21-passage-unicoil-noexp-0shot.yaml index bf0c8c7939..57825bf11b 100644 --- a/src/main/resources/regression/dl21-passage-unicoil-noexp-0shot.yaml +++ b/src/main/resources/regression/dl21-passage-unicoil-noexp-0shot.yaml @@ -10,7 +10,7 @@ index_path: indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ collection_class: JsonVectorCollection generator_class: DefaultLuceneDocumentGenerator index_threads: 18 -index_options: -impact -pretokenized +index_options: -impact -pretokenized -storeDocvectors index_stats: documents: 138364198 documents (non-empty): 138364198 @@ -77,3 +77,31 @@ models: - 0.4246 R@1000: - 0.6897 + - name: unicoil-noexp-0shot+rm3 + display: +RM3 + params: -impact -pretokenized -rm3 + results: + MAP@100: + - 0.2598 + MRR@100: + - 0.7143 + nDCG@10: + - 0.5937 + R@100: + - 0.4809 + R@1000: + - 0.7310 + - name: unicoil-noexp-0shot+rocchio + display: +Rocchio + params: -impact -pretokenized -rocchio + results: + MAP@100: + - 0.2666 + MRR@100: + - 0.7526 + nDCG@10: + - 0.6088 + R@100: + - 0.4728 + R@1000: + - 0.7509 diff --git a/src/main/resources/regression/dl21-passage.yaml b/src/main/resources/regression/dl21-passage.yaml index 101df98f61..53f718820a 100644 --- a/src/main/resources/regression/dl21-passage.yaml +++ b/src/main/resources/regression/dl21-passage.yaml @@ -101,31 +101,3 @@ models: - 0.3600 R@1000: - 0.6710 - - name: bm25-default+ax - display: +Ax - params: -bm25 -axiom -axiom.deterministic -rerankCutoff 20 - results: - MAP@100: - - 0.1907 - MRR@100: - - 0.5733 - nDCG@10: - - 0.4851 - R@100: - - 0.3803 - R@1000: - - 0.6882 - - name: bm25-default+prf - display: +PRF - params: -bm25 -bm25prf - results: - MAP@100: - - 0.1821 - MRR@100: - - 0.5532 - nDCG@10: - - 0.4740 - R@100: - - 0.3745 - R@1000: - - 0.6643 diff --git a/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml index fcf73772ea..f1cc42b2df 100644 --- a/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-d2q-t5.yaml @@ -88,3 +88,19 @@ models: R@1000: - 0.8189 - 0.8245 + - name: bm25-default+rocchio + display: +Rocchio + params: -bm25 -rocchio + results: + MAP@100: + - 0.1152 + - 0.1171 + MRR@100: + - 0.1168 + - 0.1187 + R@100: + - 0.5961 + - 0.5961 + R@1000: + - 0.8199 + - 0.8288 diff --git a/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml index 2c79d77d86..15d949b6d5 100644 --- a/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-segmented-d2q-t5.yaml @@ -88,3 +88,19 @@ models: R@1000: - 0.9000 - 0.8967 + - name: bm25-default+rocchio + display: +Rocchio + params: -bm25 -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + MAP@100: + - 0.1959 + - 0.1938 + MRR@100: + - 0.1980 + - 0.1965 + R@100: + - 0.7136 + - 0.7197 + R@1000: + - 0.9028 + - 0.8976 diff --git a/src/main/resources/regression/msmarco-v2-doc-segmented-unicoil-0shot-v2.yaml b/src/main/resources/regression/msmarco-v2-doc-segmented-unicoil-0shot-v2.yaml index ab00af36b9..bce728b04b 100644 --- a/src/main/resources/regression/msmarco-v2-doc-segmented-unicoil-0shot-v2.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-segmented-unicoil-0shot-v2.yaml @@ -10,7 +10,7 @@ index_path: indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-0shot-v2/ collection_class: JsonVectorCollection generator_class: DefaultLuceneDocumentGenerator index_threads: 18 -index_options: -impact -pretokenized +index_options: -impact -pretokenized -storeDocvectors index_stats: documents: 124131414 documents (non-empty): 124131414 @@ -76,3 +76,35 @@ models: R@1000: - 0.9122 - 0.9172 + - name: unicoil-0shot+rm3 + display: +RM3 + params: -impact -pretokenized -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + MAP@100: + - 0.2167 + - 0.2198 + MRR@100: + - 0.2190 + - 0.2225 + R@100: + - 0.7675 + - 0.7639 + R@1000: + - 0.9175 + - 0.9220 + - name: unicoil-0shot+rocchio + display: +Rocchio + params: -impact -pretokenized -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + MAP@100: + - 0.2229 + - 0.2200 + MRR@100: + - 0.2252 + - 0.2225 + R@100: + - 0.7775 + - 0.7747 + R@1000: + - 0.9232 + - 0.9253 diff --git a/src/main/resources/regression/msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.yaml b/src/main/resources/regression/msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.yaml index 86e623a7ed..fd5113c215 100644 --- a/src/main/resources/regression/msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2.yaml @@ -10,7 +10,7 @@ index_path: indexes/lucene-index.msmarco-v2-doc-segmented-unicoil-noexp-0shot-v2 collection_class: JsonVectorCollection generator_class: DefaultLuceneDocumentGenerator index_threads: 18 -index_options: -impact -pretokenized +index_options: -impact -pretokenized -storeDocvectors index_stats: documents: 124131404 documents (non-empty): 124131404 @@ -76,3 +76,35 @@ models: R@1000: - 0.8987 - 0.8995 + - name: unicoil-noexp-0shot+rm3 + display: +RM3 + params: -impact -pretokenized -rm3 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + MAP@100: + - 0.1963 + - 0.2067 + MRR@100: + - 0.1982 + - 0.2092 + R@100: + - 0.7453 + - 0.7466 + R@1000: + - 0.9026 + - 0.9082 + - name: unicoil-noexp-0shot+rocchio + display: +Rocchio + params: -impact -pretokenized -rocchio -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 + results: + MAP@100: + - 0.2011 + - 0.2090 + MRR@100: + - 0.2034 + - 0.2112 + R@100: + - 0.7520 + - 0.7540 + R@1000: + - 0.9084 + - 0.9136 diff --git a/src/main/resources/regression/msmarco-v2-doc-segmented.yaml b/src/main/resources/regression/msmarco-v2-doc-segmented.yaml index ff8fd0921a..4cd9d4d48d 100644 --- a/src/main/resources/regression/msmarco-v2-doc-segmented.yaml +++ b/src/main/resources/regression/msmarco-v2-doc-segmented.yaml @@ -104,35 +104,3 @@ models: R@1000: - 0.8659 - 0.8642 - - name: bm25-default+ax - display: +Ax - params: -bm25 -axiom -axiom.deterministic -rerankCutoff 20 -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 - results: - MAP@100: - - 0.1344 - - 0.1345 - MRR@100: - - 0.1360 - - 0.1366 - R@100: - - 0.5593 - - 0.5800 - R@1000: - - 0.8161 - - 0.8210 - - name: bm25-default+prf - display: +PRF - params: -bm25 -bm25prf -hits 10000 -selectMaxPassage -selectMaxPassage.delimiter "#" -selectMaxPassage.hits 1000 - results: - MAP@100: - - 0.1528 - - 0.1565 - MRR@100: - - 0.1547 - - 0.1587 - R@100: - - 0.5945 - - 0.6134 - R@1000: - - 0.8270 - - 0.8280 diff --git a/src/main/resources/regression/msmarco-v2-doc.yaml b/src/main/resources/regression/msmarco-v2-doc.yaml index 3c7e90312c..69defd9d04 100644 --- a/src/main/resources/regression/msmarco-v2-doc.yaml +++ b/src/main/resources/regression/msmarco-v2-doc.yaml @@ -104,35 +104,3 @@ models: R@1000: - 0.7697 - 0.7762 - - name: bm25-default+ax - display: +Ax - params: -bm25 -axiom -axiom.deterministic -rerankCutoff 20 - results: - MAP@100: - - 0.0665 - - 0.0722 - MRR@100: - - 0.0675 - - 0.0733 - R@100: - - 0.4075 - - 0.4192 - R@1000: - - 0.6852 - - 0.6960 - - name: bm25-default+prf - display: +PRF - params: -bm25 -bm25prf - results: - MAP@100: - - 0.0834 - - 0.0876 - MRR@100: - - 0.0845 - - 0.0892 - R@100: - - 0.4681 - - 0.4831 - R@1000: - - 0.7385 - - 0.7482 diff --git a/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml index ae4e106f3b..5ea08a4fbe 100644 --- a/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-passage-augmented-d2q-t5.yaml @@ -87,4 +87,20 @@ models: - 0.4905 R@1000: - 0.7625 - - 0.7649 \ No newline at end of file + - 0.7649 + - name: bm25-default+rocchio + display: +Rocchio + params: -bm25 -rocchio + results: + MAP@100: + - 0.0882 + - 0.0900 + MRR@100: + - 0.0893 + - 0.0908 + R@100: + - 0.4754 + - 0.4880 + R@1000: + - 0.7648 + - 0.7666 diff --git a/src/main/resources/regression/msmarco-v2-passage-augmented.yaml b/src/main/resources/regression/msmarco-v2-passage-augmented.yaml index 708e8f4e0d..17992f1585 100644 --- a/src/main/resources/regression/msmarco-v2-passage-augmented.yaml +++ b/src/main/resources/regression/msmarco-v2-passage-augmented.yaml @@ -104,35 +104,3 @@ models: R@1000: - 0.6912 - 0.6864 - - name: bm25-default+ax - display: +Ax - params: -bm25 -axiom -axiom.deterministic -rerankCutoff 20 - results: - MAP@100: - - 0.0431 - - 0.0442 - MRR@100: - - 0.0434 - - 0.0446 - R@100: - - 0.2998 - - 0.3220 - R@1000: - - 0.5887 - - 0.6043 - - name: bm25-default+prf - display: +PRF - params: -bm25 -bm25prf - results: - MAP@100: - - 0.0542 - - 0.0563 - MRR@100: - - 0.0547 - - 0.0572 - R@100: - - 0.3338 - - 0.3505 - R@1000: - - 0.6157 - - 0.6351 diff --git a/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml b/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml index 3fa7baffd8..a4a4be6ca2 100644 --- a/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml +++ b/src/main/resources/regression/msmarco-v2-passage-d2q-t5.yaml @@ -87,4 +87,20 @@ models: - 0.4800 R@1000: - 0.7174 - - 0.7240 \ No newline at end of file + - 0.7240 + - name: bm25-default+rocchio + display: +Rocchio + params: -bm25 -rocchio + results: + MAP@100: + - 0.0943 + - 0.0990 + MRR@100: + - 0.0952 + - 0.0997 + R@100: + - 0.4725 + - 0.4815 + R@1000: + - 0.7211 + - 0.7274 diff --git a/src/main/resources/regression/msmarco-v2-passage-unicoil-0shot.yaml b/src/main/resources/regression/msmarco-v2-passage-unicoil-0shot.yaml index fc15f863cc..79954ca135 100644 --- a/src/main/resources/regression/msmarco-v2-passage-unicoil-0shot.yaml +++ b/src/main/resources/regression/msmarco-v2-passage-unicoil-0shot.yaml @@ -10,7 +10,7 @@ index_path: indexes/lucene-index.msmarco-v2-passage-unicoil-0shot/ collection_class: JsonVectorCollection generator_class: DefaultLuceneDocumentGenerator index_threads: 18 -index_options: -impact -pretokenized +index_options: -impact -pretokenized -storeDocvectors index_stats: documents: 138364198 documents (non-empty): 138364198 @@ -76,3 +76,35 @@ models: R@1000: - 0.7616 - 0.7671 + - name: unicoil-0shot+rm3 + display: +RM3 + params: -impact -pretokenized -rm3 + results: + MAP@100: + - 0.1320 + - 0.1339 + MRR@100: + - 0.1332 + - 0.1348 + R@100: + - 0.5534 + - 0.5550 + R@1000: + - 0.7731 + - 0.7698 + - name: unicoil-0shot+rocchio + display: +Rocchio + params: -impact -pretokenized -rocchio + results: + MAP@100: + - 0.1319 + - 0.1371 + MRR@100: + - 0.1334 + - 0.1380 + R@100: + - 0.5557 + - 0.5696 + R@1000: + - 0.7800 + - 0.7849 diff --git a/src/main/resources/regression/msmarco-v2-passage-unicoil-noexp-0shot.yaml b/src/main/resources/regression/msmarco-v2-passage-unicoil-noexp-0shot.yaml index 04da480dc0..e2e9d7e6b1 100644 --- a/src/main/resources/regression/msmarco-v2-passage-unicoil-noexp-0shot.yaml +++ b/src/main/resources/regression/msmarco-v2-passage-unicoil-noexp-0shot.yaml @@ -10,7 +10,7 @@ index_path: indexes/lucene-index.msmarco-v2-passage-unicoil-noexp-0shot/ collection_class: JsonVectorCollection generator_class: DefaultLuceneDocumentGenerator index_threads: 18 -index_options: -impact -pretokenized +index_options: -impact -pretokenized -storeDocvectors index_stats: documents: 138364198 documents (non-empty): 138364198 @@ -76,3 +76,35 @@ models: R@1000: - 0.7010 - 0.7114 + - name: unicoil-noexp-0shot+rm3 + display: +RM3 + params: -impact -pretokenized -rm3 + results: + MAP@100: + - 0.1118 + - 0.1189 + MRR@100: + - 0.1127 + - 0.1196 + R@100: + - 0.4998 + - 0.5134 + R@1000: + - 0.7248 + - 0.7344 + - name: unicoil-noexp-0shot+rocchio + display: +Rocchio + params: -impact -pretokenized -rocchio + results: + MAP@100: + - 0.1163 + - 0.1179 + MRR@100: + - 0.1172 + - 0.1186 + R@100: + - 0.5082 + - 0.5218 + R@1000: + - 0.7299 + - 0.7387 diff --git a/src/main/resources/regression/msmarco-v2-passage.yaml b/src/main/resources/regression/msmarco-v2-passage.yaml index ecba9ea147..0675798037 100644 --- a/src/main/resources/regression/msmarco-v2-passage.yaml +++ b/src/main/resources/regression/msmarco-v2-passage.yaml @@ -104,35 +104,3 @@ models: R@1000: - 0.5966 - 0.6104 - - name: bm25-default+ax - display: +Ax - params: -bm25 -axiom -axiom.deterministic -rerankCutoff 20 - results: - MAP@100: - - 0.0592 - - 0.0642 - MRR@100: - - 0.0601 - - 0.0647 - R@100: - - 0.3482 - - 0.3554 - R@1000: - - 0.6064 - - 0.6254 - - name: bm25-default+prf - display: +PRF - params: -bm25 -bm25prf - results: - MAP@100: - - 0.0595 - - 0.0632 - MRR@100: - - 0.0607 - - 0.0640 - R@100: - - 0.3495 - - 0.3595 - R@1000: - - 0.5968 - - 0.6169