Chapter 74

Indexing stage

NotebookPython 3 (ipykernel)28 cells

In [1]python · cell 1

python

import json
import pandas as pd
from tqdm.auto import tqdm
from sentence_transformers import SentenceTransformer
from elasticsearch import Elasticsearch

In [2]python · cell 2

python

with open('documents-with-ids.json', 'rt') as f_in:
    documents = json.load(f_in)

In [3]python · cell 3

python

model_name = 'multi-qa-MiniLM-L6-cos-v1'
model = SentenceTransformer(model_name)

Output

/usr/local/python/3.10.13/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884
  warnings.warn(

Indexing stage

In [4]python · cell 5

python

for doc in tqdm(documents):
    question = doc['question']
    text = doc['text']
    qt = question + ' ' + text

    doc['question_vector'] = model.encode(question)
    doc['text_vector'] = model.encode(text)
    doc['question_text_vector'] = model.encode(qt)

Output

  0%|          | 0/948 [00:00<?, ?it/s]

In [5]python · cell 6

python

es_client = Elasticsearch('http://localhost:9200') 

index_settings = {
    "settings": {
        "number_of_shards": 1,
        "number_of_replicas": 0
    },
    "mappings": {
        "properties": {
            "text": {"type": "text"},
            "section": {"type": "text"},
            "question": {"type": "text"},
            "course": {"type": "keyword"},
            "id": {"type": "keyword"},
            "question_vector": {
                "type": "dense_vector",
                "dims": 384,
                "index": True,
                "similarity": "cosine"
            },
            "text_vector": {
                "type": "dense_vector",
                "dims": 384,
                "index": True,
                "similarity": "cosine"
            },
            "question_text_vector": {
                "type": "dense_vector",
                "dims": 384,
                "index": True,
                "similarity": "cosine"
            },
        }
    }
}

index_name = "course-questions"

es_client.indices.delete(index=index_name, ignore_unavailable=True)
es_client.indices.create(index=index_name, body=index_settings)

Output

ObjectApiResponse({'acknowledged': True, 'shards_acknowledged': True, 'index': 'course-questions'})

In [6]python · cell 7

python

for doc in tqdm(documents):
    es_client.index(index=index_name, document=doc)

Output

  0%|          | 0/948 [00:00<?, ?it/s]

Retrieval stage

In [7]python · cell 9

python

from langchain.embeddings import SentenceTransformerEmbeddings
from typing import Dict
from langchain_elasticsearch import ElasticsearchRetriever

In [8]python · cell 10

python

es_url = 'http://localhost:9200'

In [9]python · cell 11

python

query = 'I just discovered the course. Can I still join it?'
course = "data-engineering-zoomcamp"

In [15]python · cell 12

python

embeddings = SentenceTransformerEmbeddings(model_name="sentence-transformers/multi-qa-MiniLM-L6-cos-v1")

In [21]python · cell 13

python

def hybrid_query(search_query: str) -> Dict:
    vector = embeddings.embed_query(search_query)  # same embeddings as for indexing
    return {
        "query": {
            "bool": {
                "must": {
                    "multi_match": {
                        "query": search_query,
                        "fields": ["question", "text", "section"],
                        "type": "best_fields",
                        "boost": 0.5,
                    }
                },
                "filter": {
                    "term": {
                        "course": course
                    }
                }
            }
        },
        "knn": {
            "field": "question_text_vector",
            "query_vector": vector,
            "k": 5,
            "num_candidates": 10000,
            "boost": 0.5,
            "filter": {
                "term": {
                    "course": course
                }
            }
        },
        "size": 5,
        # "rank": {"rrf": {}},
    }


hybrid_retriever = ElasticsearchRetriever.from_es_params(
    index_name=index_name,
    body_func=hybrid_query,
    content_field='text',
    url=es_url,
)

In [22]python · cell 14

python

hybrid_results = hybrid_retriever.invoke(query)

In [23]python · cell 15

python

for result in hybrid_results:
    print(result.metadata['_source']['question'], result.metadata['_source']['course'], result.metadata['_score'])

Output

Course - Can I still join the course after the start date? data-engineering-zoomcamp 12.559245
Course - Can I follow the course after it finishes? data-engineering-zoomcamp 9.39959
Course - What can I do before the course starts? data-engineering-zoomcamp 7.306914
Course - Can I get support if I take the course in the self-paced mode? data-engineering-zoomcamp 7.1085525
Course - When will the course start? data-engineering-zoomcamp 6.7513986

Hybrid search

In [24]python · cell 17

python

df_ground_truth = pd.read_csv('ground-truth-data.csv')

In [25]python · cell 18

python

ground_truth = df_ground_truth.to_dict(orient='records')

In [26]python · cell 19

python

def hit_rate(relevance_total):
    cnt = 0

    for line in relevance_total:
        if True in line:
            cnt = cnt + 1

    return cnt / len(relevance_total)

In [27]python · cell 20

python

def mrr(relevance_total):
    total_score = 0.0

    for line in relevance_total:
        for rank in range(len(line)):
            if line[rank] == True:
                total_score = total_score + 1 / (rank + 1)

    return total_score / len(relevance_total)

In [34]python · cell 21

python

def elastic_search_hybrid(field, query, course):
    def hybrid_query(search_query: str) -> Dict:
        vector = embeddings.embed_query(search_query)  # same embeddings as for indexing
        return {
            "query": {
                "bool": {
                    "must": {
                        "multi_match": {
                            "query": search_query,
                            "fields": ["question", "text", "section"],
                            "type": "best_fields",
                            "boost": 0.5,
                        }
                    },
                    "filter": {
                        "term": {
                            "course": course
                        }
                    }
                }
            },
            "knn": {
                "field": field,
                "query_vector": vector,
                "k": 5,
                "num_candidates": 10000,
                "boost": 0.5,
                "filter": {
                    "term": {
                        "course": course
                    }
                }
            },
            "size": 5,
            "_source": ["text", "section", "question", "course", "id"],
            # "rank": {"rrf": {}},
        }
    
    
    hybrid_retriever = ElasticsearchRetriever.from_es_params(
        index_name=index_name,
        body_func=hybrid_query,
        content_field='text',
        url=es_url,
    )

    hybrid_results = hybrid_retriever.invoke(query)
    
    result_docs = []
    
    for hit in hybrid_results:
        result_docs.append(hit.metadata['_source'])

    return result_docs

In [35]python · cell 22

python

ground_truth[0]

Output

{'question': 'When does the course begin?',
 'course': 'data-engineering-zoomcamp',
 'document': 'c02e79ef'}

In [36]python · cell 23

python

question = ground_truth[0]['question']
course = ground_truth[0]['course']
elastic_search_hybrid('question_text_vector', question, course)

Output

[{'section': 'General course-related questions',
  'question': 'Course - When will the course start?',
  'course': 'data-engineering-zoomcamp',
  'id': 'c02e79ef'},
 {'section': 'General course-related questions',
  'question': 'Course - Can I still join the course after the start date?',
  'course': 'data-engineering-zoomcamp',
  'id': '7842b56a'},
 {'section': 'General course-related questions',
  'question': 'Course - Can I follow the course after it finishes?',
  'course': 'data-engineering-zoomcamp',
  'id': 'a482086d'},
 {'section': 'Module 1: Docker and Terraform',
  'question': 'PGCLI - error column c.relhasoids does not exist',
  'course': 'data-engineering-zoomcamp',
  'id': 'c91ad8f2'},
 {'section': 'General course-related questions',
  'question': 'Course - What are the prerequisites for this course?',
  'course': 'data-engineering-zoomcamp',
  'id': '1f6520ca'}]

In [37]python · cell 24

python

def question_text_hybrid(q):
    question = q['question']
    course = q['course']

    return elastic_search_hybrid('question_text_vector', question, course)

In [38]python · cell 25

python

def evaluate(ground_truth, search_function):
    relevance_total = []

    for q in tqdm(ground_truth):
        doc_id = q['document']
        results = search_function(q)
        relevance = [d['id'] == doc_id for d in results]
        relevance_total.append(relevance)

    return {
        'hit_rate': hit_rate(relevance_total),
        'mrr': mrr(relevance_total),
    }

In [39]python · cell 26

python

evaluate(ground_truth, question_text_hybrid)

Output

  0%|          | 0/4627 [00:00<?, ?it/s]

{'hit_rate': 0.9250054030689432, 'mrr': 0.8506231539514445}

Hybrid search with ES: {'hit_rate': 0.9250054030689432, 'mrr': 0.8506231539514445}

/usr/local/python/3.10.13/lib/python3.10/site-packages/transformers/tokenization_utils_base.py:1601: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted in transformers v4.45, and will be then set to `False` by default. For more details check this issue: https://github.com/huggingface/transformers/issues/31884 warnings.warn(

for doc in tqdm(documents): question = doc['question'] text = doc['text'] qt = question + ' ' + text doc['question_vector'] = model.encode(question) doc['text_vector'] = model.encode(text) doc['question_text_vector'] = model.encode(qt)

es_client = Elasticsearch('http://localhost:9200') index_settings = { "settings": { "number_of_shards": 1, "number_of_replicas": 0 }, "mappings": { "properties": { "text": {"type": "text"}, "section": {"type": "text"}, "question": {"type": "text"}, "course": {"type": "keyword"}, "id": {"type": "keyword"}, "question_vector": { "type": "dense_vector", "dims": 384, "index": True, "similarity": "cosine" }, "text_vector": { "type": "dense_vector", "dims": 384, "index": True, "similarity": "cosine" }, "question_text_vector": { "type": "dense_vector", "dims": 384, "index": True, "similarity": "cosine" }, } } } index_name = "course-questions" es_client.indices.delete(index=index_name, ignore_unavailable=True) es_client.indices.create(index=index_name, body=index_settings)

def hybrid_query(search_query: str) -> Dict: vector = embeddings.embed_query(search_query) # same embeddings as for indexing return { "query": { "bool": { "must": { "multi_match": { "query": search_query, "fields": ["question", "text", "section"], "type": "best_fields", "boost": 0.5, } }, "filter": { "term": { "course": course } } } }, "knn": { "field": "question_text_vector", "query_vector": vector, "k": 5, "num_candidates": 10000, "boost": 0.5, "filter": { "term": { "course": course } } }, "size": 5, # "rank": {"rrf": {}}, } hybrid_retriever = ElasticsearchRetriever.from_es_params( index_name=index_name, body_func=hybrid_query, content_field='text', url=es_url, )

Course - Can I still join the course after the start date? data-engineering-zoomcamp 12.559245 Course - Can I follow the course after it finishes? data-engineering-zoomcamp 9.39959 Course - What can I do before the course starts? data-engineering-zoomcamp 7.306914 Course - Can I get support if I take the course in the self-paced mode? data-engineering-zoomcamp 7.1085525 Course - When will the course start? data-engineering-zoomcamp 6.7513986

def mrr(relevance_total): total_score = 0.0 for line in relevance_total: for rank in range(len(line)): if line[rank] == True: total_score = total_score + 1 / (rank + 1) return total_score / len(relevance_total)

def elastic_search_hybrid(field, query, course): def hybrid_query(search_query: str) -> Dict: vector = embeddings.embed_query(search_query) # same embeddings as for indexing return { "query": { "bool": { "must": { "multi_match": { "query": search_query, "fields": ["question", "text", "section"], "type": "best_fields", "boost": 0.5, } }, "filter": { "term": { "course": course } } } }, "knn": { "field": field, "query_vector": vector, "k": 5, "num_candidates": 10000, "boost": 0.5, "filter": { "term": { "course": course } } }, "size": 5, "_source": ["text", "section", "question", "course", "id"], # "rank": {"rrf": {}}, } hybrid_retriever = ElasticsearchRetriever.from_es_params( index_name=index_name, body_func=hybrid_query, content_field='text', url=es_url, ) hybrid_results = hybrid_retriever.invoke(query) result_docs = [] for hit in hybrid_results: result_docs.append(hit.metadata['_source']) return result_docs

[{'section': 'General course-related questions', 'question': 'Course - When will the course start?', 'course': 'data-engineering-zoomcamp', 'id': 'c02e79ef'}, {'section': 'General course-related questions', 'question': 'Course - Can I still join the course after the start date?', 'course': 'data-engineering-zoomcamp', 'id': '7842b56a'}, {'section': 'General course-related questions', 'question': 'Course - Can I follow the course after it finishes?', 'course': 'data-engineering-zoomcamp', 'id': 'a482086d'}, {'section': 'Module 1: Docker and Terraform', 'question': 'PGCLI - error column c.relhasoids does not exist', 'course': 'data-engineering-zoomcamp', 'id': 'c91ad8f2'}, {'section': 'General course-related questions', 'question': 'Course - What are the prerequisites for this course?', 'course': 'data-engineering-zoomcamp', 'id': '1f6520ca'}]

def evaluate(ground_truth, search_function): relevance_total = [] for q in tqdm(ground_truth): doc_id = q['document'] results = search_function(q) relevance = [d['id'] == doc_id for d in results] relevance_total.append(relevance) return { 'hit_rate': hit_rate(relevance_total), 'mrr': mrr(relevance_total), }