Chapter 108
evaluate vector
NotebookPython 3 (ipykernel)29 cells
In [1]python · cell 1
python
import json
with open('documents-with-ids.json', 'rt') as f_in:
documents = json.load(f_in)In [2]python · cell 2
python
from sentence_transformers import SentenceTransformerIn [3]python · cell 3
python
model_name = 'multi-qa-MiniLM-L6-cos-v1'
model = SentenceTransformer(model_name)In [6]python · cell 4
python
v = model.encode('I just discovered the course. Can I still join?')In [10]python · cell 5
python
len(v)Output
384
In [11]python · cell 6
python
from elasticsearch import Elasticsearch
es_client = Elasticsearch('http://localhost:9200')
index_settings = {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 0
},
"mappings": {
"properties": {
"text": {"type": "text"},
"section": {"type": "text"},
"question": {"type": "text"},
"course": {"type": "keyword"},
"id": {"type": "keyword"},
"question_vector": {
"type": "dense_vector",
"dims": 384,
"index": True,
"similarity": "cosine"
},
"text_vector": {
"type": "dense_vector",
"dims": 384,
"index": True,
"similarity": "cosine"
},
"question_text_vector": {
"type": "dense_vector",
"dims": 384,
"index": True,
"similarity": "cosine"
},
}
}
}
index_name = "course-questions"
es_client.indices.delete(index=index_name, ignore_unavailable=True)
es_client.indices.create(index=index_name, body=index_settings)Output
ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'course-questions'})In [12]python · cell 7
python
from tqdm.auto import tqdmIn [13]python · cell 8
python
for doc in tqdm(documents):
question = doc['question']
text = doc['text']
qt = question + ' ' + text
doc['question_vector'] = model.encode(question)
doc['text_vector'] = model.encode(text)
doc['question_text_vector'] = model.encode(qt)Output
0%| | 0/948 [00:00<?, ?it/s]
In [14]python · cell 9
python
for doc in tqdm(documents):
es_client.index(index=index_name, document=doc)Output
0%| | 0/948 [00:00<?, ?it/s]
In [15]python · cell 10
python
query = 'I just discovered the course. Can I still join it?'In [16]python · cell 11
python
v_q = model.encode(query)In [29]python · cell 12
python
def elastic_search_knn(field, vector, course):
knn = {
"field": field,
"query_vector": vector,
"k": 5,
"num_candidates": 10000,
"filter": {
"term": {
"course": course
}
}
}
search_query = {
"knn": knn,
"_source": ["text", "section", "question", "course", "id"]
}
es_results = es_client.search(
index=index_name,
body=search_query
)
result_docs = []
for hit in es_results['hits']['hits']:
result_docs.append(hit['_source'])
return result_docsIn [36]python · cell 13
python
def question_vector_knn(q):
question = q['question']
course = q['course']
v_q = model.encode(question)
return elastic_search_knn('question_vector', v_q, course)In [32]python · cell 14
python
import pandas as pdIn [33]python · cell 15
python
df_ground_truth = pd.read_csv('ground-truth-data.csv')In [34]python · cell 16
python
ground_truth = df_ground_truth.to_dict(orient='records')In [35]python · cell 17
python
ground_truth[0]Output
{'question': 'When does the course begin?',
'course': 'data-engineering-zoomcamp',
'document': 'c02e79ef'}In [37]python · cell 18
python
def hit_rate(relevance_total):
cnt = 0
for line in relevance_total:
if True in line:
cnt = cnt + 1
return cnt / len(relevance_total)In [ ]python · cell 19
python
In [38]python · cell 20
python
def mrr(relevance_total):
total_score = 0.0
for line in relevance_total:
for rank in range(len(line)):
if line[rank] == True:
total_score = total_score + 1 / (rank + 1)
return total_score / len(relevance_total)In [39]python · cell 21
python
def evaluate(ground_truth, search_function):
relevance_total = []
for q in tqdm(ground_truth):
doc_id = q['document']
results = search_function(q)
relevance = [d['id'] == doc_id for d in results]
relevance_total.append(relevance)
return {
'hit_rate': hit_rate(relevance_total),
'mrr': mrr(relevance_total),
}In [40]python · cell 22
python
evaluate(ground_truth, question_vector_knn)Output
0%| | 0/4627 [00:00<?, ?it/s]
{'hit_rate': 0.773071104387292, 'mrr': 0.6666810748505158}ES text only: 0.7395720769397017, 0.6032418413658963
In [44]python · cell 24
python
def text_vector_knn(q):
question = q['question']
course = q['course']
v_q = model.encode(question)
return elastic_search_knn('text_vector', v_q, course)In [46]python · cell 25
python
evaluate(ground_truth, text_vector_knn)Output
0%| | 0/4627 [00:00<?, ?it/s]
{'hit_rate': 0.8286146531229739, 'mrr': 0.7062315395144454}In [48]python · cell 26
python
def question_text_vector_knn(q):
question = q['question']
course = q['course']
v_q = model.encode(question)
return elastic_search_knn('question_text_vector', v_q, course)
evaluate(ground_truth, question_text_vector_knn)Output
0%| | 0/4627 [00:00<?, ?it/s]
{'hit_rate': 0.9172249837907932, 'mrr': 0.824306606152295}In [61]python · cell 27
python
def elastic_search_knn_combined(vector, course):
search_query = {
"size": 5,
"query": {
"bool": {
"must": [
{
"script_score": {
"query": {
"term": {
"course": course
}
},
"script": {
"source": """
cosineSimilarity(params.query_vector, 'question_vector') +
cosineSimilarity(params.query_vector, 'text_vector') +
cosineSimilarity(params.query_vector, 'question_text_vector') +
1
""",
"params": {
"query_vector": vector
}
}
}
}
],
"filter": {
"term": {
"course": course
}
}
}
},
"_source": ["text", "section", "question", "course", "id"]
}
es_results = es_client.search(
index=index_name,
body=search_query
)
result_docs = []
for hit in es_results['hits']['hits']:
result_docs.append(hit['_source'])
return result_docsIn [63]python · cell 28
python
def vector_combined_knn(q):
question = q['question']
course = q['course']
v_q = model.encode(question)
return elastic_search_knn_combined(v_q, course)
evaluate(ground_truth, vector_combined_knn)Output
0%| | 0/4627 [00:00<?, ?it/s]
{'hit_rate': 0.9023125135076724, 'mrr': 0.804480945176861}In [ ]python · cell 29
python