Bases: BaseRetrievalEvaluator
Retriever evaluator.
This module will evaluate a retriever using a set of metrics.
Parameters:
Name |
Type |
Description |
Default |
metrics |
List[BaseRetrievalMetric]
|
Sequence of metrics to evaluate
|
required
|
retriever |
BaseRetriever
|
|
required
|
node_postprocessors |
Optional[List[BaseNodePostprocessor]]
|
Post-processor to apply after retrieval.
|
None
|
Source code in llama-index-core/llama_index/core/evaluation/retrieval/evaluator.py
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134 | class MultiModalRetrieverEvaluator(BaseRetrievalEvaluator):
"""Retriever evaluator.
This module will evaluate a retriever using a set of metrics.
Args:
metrics (List[BaseRetrievalMetric]): Sequence of metrics to evaluate
retriever: Retriever to evaluate.
node_postprocessors (Optional[List[BaseNodePostprocessor]]): Post-processor to apply after retrieval.
"""
retriever: BaseRetriever = Field(..., description="Retriever to evaluate")
node_postprocessors: Optional[List[BaseNodePostprocessor]] = Field(
default=None, description="Optional post-processor"
)
def __init__(
self,
metrics: Sequence[BaseRetrievalMetric],
retriever: BaseRetriever,
node_postprocessors: Optional[List[BaseNodePostprocessor]] = None,
**kwargs: Any,
) -> None:
"""Init params."""
super().__init__(
metrics=metrics,
retriever=retriever,
node_postprocessors=node_postprocessors,
**kwargs,
)
async def _aget_retrieved_ids_texts(
self, query: str, mode: RetrievalEvalMode = RetrievalEvalMode.TEXT
) -> Tuple[List[str], List[str]]:
"""Get retrieved ids."""
retrieved_nodes = await self.retriever.aretrieve(query)
image_nodes: List[ImageNode] = []
text_nodes: List[TextNode] = []
if self.node_postprocessors:
for node_postprocessor in self.node_postprocessors:
retrieved_nodes = node_postprocessor.postprocess_nodes(
retrieved_nodes, query_str=query
)
for scored_node in retrieved_nodes:
node = scored_node.node
if isinstance(node, ImageNode):
image_nodes.append(node)
if node.text:
text_nodes.append(node)
if mode == "text":
return (
[node.node_id for node in text_nodes],
[node.text for node in text_nodes],
)
elif mode == "image":
return (
[node.node_id for node in image_nodes],
[node.text for node in image_nodes],
)
else:
raise ValueError("Unsupported mode.")
|