-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #419 from NYPL/SCC-4329-rank-eval
SCC-4329: rank eval script
- Loading branch information
Showing
4 changed files
with
587 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,353 @@ | ||
| --- | ||
| search: "A Reader's Guide to Hart Craine's White Buildings" | ||
| scope: "keyword" | ||
| metric: "precision" | ||
| metric_at: 3 | ||
| relevant: | ||
| - "b11785956" | ||
| - "pb998308253506421" | ||
| notes: | ||
| - "Misspelling of 'Crane' for 'Craine' " | ||
| - "@3 ensures we find them all. Fragile though" | ||
| reason: "initial target" | ||
| --- | ||
| search: "A Reader's Guide to Hart Craine's White Buildings" | ||
| scope: "keyword" | ||
| metric: "precision" | ||
| metric_at: 1 | ||
| relevant: | ||
| - "b11785956" | ||
| notes: | ||
| - "Misspelling of 'Crane' for 'Craine' " | ||
| - "Like previous but maybe better, ensures the NYPL record the top hit" | ||
| reason: "initial target" | ||
| --- | ||
| search: "candide" | ||
| scope: "keyword" | ||
| metric: "recall" | ||
| metric_at: 10 | ||
| relevant: | ||
| - "b10690141" | ||
| - "b10780034" | ||
| - "b14000114" | ||
| - "cb620952" | ||
| - "cb5834647" | ||
| - "cb10171120" | ||
| - "cb11607983" | ||
| - "b11785903" | ||
| - "b16593688" | ||
| - "b16600011" | ||
| notes: | ||
| - "Current results, but there might be better Candides" | ||
| - "Ensures similar terms ('candid', 'candidate') don't dominate" | ||
| reason: "initial target" | ||
| --- | ||
| search: "liberator" | ||
| scope: "keyword" | ||
| metric: "recall" | ||
| metric_at: 25 | ||
| relevant: | ||
| - "b12316212" | ||
| - "b13204619" | ||
| notes: | ||
| - "There is probably a fuller list" | ||
| reason: "initial target" | ||
| --- | ||
| search: "time magazine" | ||
| scope: "journal title" | ||
| metric: "recall" | ||
| metric_at: 50 | ||
| relevant: | ||
| - "b10682828" | ||
| - "b17394425" | ||
| - "b21462251" | ||
| - "b16542771" | ||
| notes: | ||
| - "Results aren't great because 'magazine' throws it off" | ||
| - "Something we want to track and improve" | ||
| reason: "initial target" | ||
| --- | ||
| search: "time" | ||
| scope: "journal title" | ||
| metric: "recall" | ||
| metric_at: 50 | ||
| relevant: | ||
| - "b10682828" | ||
| - "b17394425" | ||
| - "b21462251" | ||
| - "b16542771" | ||
| notes: | ||
| - "Should be better thab 'time magazine'" | ||
| reason: "initial target" | ||
| --- | ||
| search: "sassy" | ||
| scope: "keyword" | ||
| metric: "recall" | ||
| metric_at: 50 | ||
| relevant: | ||
| - "b11612329" | ||
| notes: | ||
| - "Should be better than 'sassy magazine'" | ||
| reason: "initial target" | ||
| --- | ||
| search: "letter from tombs prison" | ||
| scope: "title" | ||
| metric: "recall" | ||
| metric_at: 10 | ||
| relevant: | ||
| - "b11782546" | ||
| - "pb9938471653506421" | ||
| - "cb2080258" | ||
| notes: | ||
| - "Title is 'Letter from the Tombs' and the addition of 'prison' throws it off" | ||
| reason: "initial target" | ||
| --- | ||
| search: "letter from the tombs" | ||
| scope: "title" | ||
| metric: "precision" | ||
| metric_at: 1 | ||
| relevant: | ||
| - "b11782546" | ||
| notes: | ||
| - "Should be better than 'letter from tombs prison'" | ||
| reason: "initial target" | ||
| --- | ||
| search: "gold, leonard" | ||
| scope: "contributor" | ||
| metric: "precision" | ||
| metric_at: 2 | ||
| relevant: | ||
| - "b10002582" | ||
| - "b14650132" | ||
| reason: "initial target" | ||
| --- | ||
| search: "alexander, victor" | ||
| scope: "contributor" | ||
| metric: "recall" | ||
| metric_at: 50 | ||
| relevant: | ||
| - "b16476737" | ||
| - "b13998639" | ||
| - "hb990031177450203941" | ||
| - "cb2235121" | ||
| reason: "initial target" | ||
| --- | ||
| search: "journal of jewish studies" | ||
| scope: "journal title" | ||
| metric: "precision" | ||
| metric_at: 1 | ||
| relevant: | ||
| - "b11151367" | ||
| reason: "initial target" | ||
| --- | ||
| search: "East and West" | ||
| scope: "journal title" | ||
| metric: "recall" | ||
| metric_at: 50 | ||
| relevant: | ||
| - "b11378929" | ||
| - "b10988853" | ||
| - "b21433970" | ||
| - "b15276563" | ||
| - "b15087757" | ||
| - "b15088556" | ||
| reason: "initial target" | ||
| --- | ||
| search: "a man's a man" | ||
| scope: "title" | ||
| metric: "precision" | ||
| metric_at: 3 | ||
| relevant: | ||
| - "b12411226" | ||
| - "b16761751" | ||
| - "b20009609" | ||
| reason: "initial target" | ||
| --- | ||
| search: "beginnings of jewishness" | ||
| scope: "title" | ||
| metric: "precision" | ||
| metric_at: 2 | ||
| relevant: | ||
| - "b15264124" | ||
| - "cb2167369" | ||
| reason: "initial target" | ||
| notes: | ||
| - "We would like to see the NYPL copy first" | ||
| --- | ||
| search: "a mom's life" | ||
| scope: "title" | ||
| metric: "recall" | ||
| metric_at: 4 | ||
| relevant: | ||
| - "b16556599" | ||
| - "b19927929" | ||
| - "b16542290" | ||
| - "b16649091" | ||
| reason: "initial target" | ||
| --- | ||
| search: "new york times" | ||
| scope: "keyword" | ||
| metric: "recall" | ||
| metric_at: 50 | ||
| relevant: | ||
| - "pb626684" | ||
| - "b12639440" | ||
| - "b21835201" | ||
| - "b12403270" | ||
| - "b21941401" | ||
| - "cb524908" | ||
| - "b21148115" | ||
| - "b21168678" | ||
| - "b21148447" | ||
| - "b11801637" | ||
| - "b21148114" | ||
| reason: "initial target" | ||
| --- | ||
| search: "wired" | ||
| scope: "keyword" | ||
| metric: "recall" | ||
| metric_at: 10 | ||
| relevant: | ||
| - "b22965079" | ||
| - "b22214385" | ||
| - "b21937839" | ||
| - "b17218096" | ||
| - "b22160964" | ||
| - "pb999474273506421" | ||
| - "b11686313" | ||
| - "b11514515" | ||
| - "b21884998" | ||
| reason: "initial target" | ||
| --- | ||
| search: "\u0022In memory of memory\u0022" | ||
| scope: "title" | ||
| metric: "precision" | ||
| metric_at: 1 | ||
| relevant: | ||
| - "b22405724" | ||
| reason: "initial target" | ||
| notes: | ||
| - "Search with quotes" | ||
| --- | ||
| search: "\u0022in cold blood\u0022" | ||
| scope: "title" | ||
| metric: "precision" | ||
| metric_at: 2 | ||
| relevant: | ||
| - "b11064010" | ||
| - "b21582147" | ||
| reason: "initial target" | ||
| notes: | ||
| - "Search with quotes" | ||
| --- | ||
| search: "the shroud of christ" | ||
| scope: "title" | ||
| metric: "precision" | ||
| metric_at: 4 | ||
| relevant: | ||
| - "b11095468" | ||
| - "cb4692334" | ||
| - "pb1267391" | ||
| - "b23193131" | ||
| reason: "initial target" | ||
| notes: | ||
| - "Search with quotes" | ||
| --- | ||
| search: "kosovsky otsar ha-leshon" | ||
| scope: "keyword" | ||
| metric: "recall" | ||
| metric_at: 10 | ||
| relevant: | ||
| - "b14258861" | ||
| - "b14258864" | ||
| - "b11904261" | ||
| - "b14197265" | ||
| reason: "initial target" | ||
| notes: | ||
| - "Prioritize onsite" | ||
| --- | ||
| search: "encyclopedia of jewish diaspora" | ||
| scope: "keyword" | ||
| metric: "precision" | ||
| metric_at: 1 | ||
| relevant: | ||
| - "b22180471" | ||
| reason: "initial target" | ||
| notes: | ||
| - "Prioritize onsite" | ||
| --- | ||
| search: "33433135877482" | ||
| scope: "keyword" | ||
| metric: "precision" | ||
| metric_at: 1 | ||
| relevant: | ||
| - "b22935532" | ||
| reason: "initial target" | ||
| notes: | ||
| - "Exact barcode" | ||
| --- | ||
| search: "Vinipeg" | ||
| scope: "keyword" | ||
| metric: "precision" | ||
| metric_at: 4 | ||
| relevant: | ||
| - "b13959162" | ||
| - "b16422751" | ||
| - "b11448009" | ||
| - "cb2859634" | ||
| reason: "initial target" | ||
| --- | ||
| search: "JFD 16-5143" | ||
| scope: "keyword" | ||
| metric: "precision" | ||
| metric_at: 1 | ||
| relevant: | ||
| - "b21099627" | ||
| reason: "initial target" | ||
| --- | ||
| search: "\u0022center of life\u0022" | ||
| scope: "keyword" | ||
| metric: "precision" | ||
| metric_at: 2 | ||
| relevant: | ||
| - "cb4793271" | ||
| - "cb270248" | ||
| reason: "initial target" | ||
| --- | ||
| search: "Peribáñez y el comendador de Ocaña" | ||
| scope: "keyword" | ||
| metric: "precision" | ||
| metric_at: 6 | ||
| relevant: | ||
| - "b15578492" | ||
| - "b10853092" | ||
| - "b11361733" | ||
| - "b13439547" | ||
| - "b11041128" | ||
| - "b13266844" | ||
| reason: "initial target" | ||
| --- | ||
| search: "the power broker" | ||
| scope: "keyword" | ||
| metric: "recall" | ||
| metric_at: 5 | ||
| relevant: | ||
| - "b10311633" | ||
| - "b21586756" | ||
| - "cb202556" | ||
| notes: | ||
| - "Note to self: maybe the is a good example for using Recall. There are other books called 'The Power Broker' and we want to be sure that all the Caro books (3) are at least in the top 5" | ||
| reason: "initial target" | ||
| --- | ||
| search: "\u0022the power broker\u0022" | ||
| scope: "keyword" | ||
| metric: "recall" | ||
| metric_at: 5 | ||
| relevant: | ||
| - "b10311633" | ||
| - "b21586756" | ||
| - "cb202556" | ||
| reason: "initial target" | ||
|
|
||
|
|
||
|
|
Oops, something went wrong.