|
315 | 315 | "from hinteval.evaluation.relevance import Rouge\n",
|
316 | 316 | "from hinteval.evaluation.readability import MachineLearningBased\n",
|
317 | 317 | "from hinteval.evaluation.convergence import LlmBased\n",
|
318 |
| - "from hinteval.evaluation.familiarity import Wikipedia\n", |
| 318 | + "from hinteval.evaluation.familiarity import WordFrequency\n", |
319 | 319 | "from hinteval.evaluation.answer_leakage import ContextualEmbeddings"
|
320 | 320 | ],
|
321 | 321 | "metadata": {
|
|
332 | 332 | "1. **Relevance (_Rouge_)**: Measures the relevance of the hints to the question using ROUGE-L algorithms.\n",
|
333 | 333 | "2. **Readability (_MachineLearningBased_)**: Uses a Random Forest algorithm to measure the readability of the hints and questions.\n",
|
334 | 334 | "3. **Convergence (_LLmBased_)**: Assesses how well the hints help eliminate incorrect answers using the Meta LLaMA-3.1-70b-Instruct-Turbo model.\n",
|
335 |
| - "4. **Familiarity (_Wikipedia_)**: Evaluates the familiarity of the information in the hints, questions, and answers based on Wikipedia view counts.\n", |
| 335 | + "4. **Familiarity (_WordFrequency_)**: Evaluates the familiarity of the information in the hints, questions, and answers based on word occurancy.\n", |
336 | 336 | "5. **AnswerLeakage (_ContextualEmbeddings_)**: Measures how much the hints reveal the answers by calculating the similarity between the hints and answers using contextual embeddings.\n",
|
337 | 337 | "\n",
|
338 | 338 | "To explore other metrics, check the [metrics guide](https://hinteval.readthedocs.io/en/latest/concepts/metrics/index.html)."
|
|
384 | 384 | "Rouge('rougeL', enable_tqdm=True).evaluate(instances)\n",
|
385 | 385 | "MachineLearningBased('random_forest', enable_tqdm=True).evaluate(questions + hints)\n",
|
386 | 386 | "LlmBased('llama-3-70b', together_ai_api_key=api_key, enable_tqdm=True).evaluate(instances)\n",
|
387 |
| - "Wikipedia(enable_tqdm=True).evaluate(questions + hints + answers)\n", |
| 387 | + "WordFrequency(enable_tqdm=True).evaluate(questions + hints + answers)\n", |
388 | 388 | "ContextualEmbeddings(enable_tqdm=True).evaluate(instances)\n",
|
389 | 389 | "pass"
|
390 | 390 | ],
|
|
0 commit comments