Chapter 60

02 search eval

Notebookllm-zoomcamp-2026-evals44 cells

In [79]python · cell 1

python

import pandas as pd

df_ground_truth = pd.read_csv("data/ground_truth-new.csv") #-data.csv")

In [51]python · cell 2

python

df_ground_truth.head()

Output

                                            question        course    document
0  Can I take this course at my own pace and stil...  llm-zoomcamp  69d122f12e
1  Is a certificate available if I complete the c...  llm-zoomcamp  69d122f12e
2  Do self-paced learners get any certificate for...  llm-zoomcamp  69d122f12e
3  Why are certificates not issued for the self-p...  llm-zoomcamp  69d122f12e
4  Is peer review of capstone projects required i...  llm-zoomcamp  69d122f12e

.dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; }

	question	course	document
0	Can I take this course at my own pace and stil...	llm-zoomcamp	69d122f12e
1	Is a certificate available if I complete the c...	llm-zoomcamp	69d122f12e
2	Do self-paced learners get any certificate for...	llm-zoomcamp	69d122f12e
3	Why are certificates not issued for the self-p...	llm-zoomcamp	69d122f12e
4	Is peer review of capstone projects required i...	llm-zoomcamp	69d122f12e

In [80]python · cell 3

python

ground_truth = df_ground_truth.to_dict(orient="records")

In [81]python · cell 4

python

ground_truth[10]

Output

{'question': 'How do I join the Office Hours or live workshop if I don’t have the Zoom link?',
 'document': '489dd1c9d9'}

In [82]python · cell 5

python

from ingest import load_faq_data, build_index

documents = load_faq_data()

documents_llm = []

for doc in documents:
    if doc["course"] == "llm-zoomcamp":
        documents_llm.append(doc)

documents = documents_llm
index = build_index(documents)

In [83]python · cell 6

python

boost = {'question': 3.0}

index.search(
    "What is the course about?",
    num_results=5,
    boost_dict=boost
)

Output

[{'id': 'db78580409',
  'course': 'llm-zoomcamp',
  'section': 'Module 3: Vector Search',
  'question': 'What is the cosine similarity?',
  'answer': 'Cosine similarity is a measure used to calculate the similarity between two non-zero vectors, often used in text analysis to determine how similar two documents are based on their content. This metric computes the cosine of the angle between two vectors, which are typically word counts or TF-IDF values of the documents. The cosine similarity value ranges from -1 to 1, where 1 indicates that the vectors are identical, 0 indicates that the vectors are orthogonal (no similarity), and -1 represents completely opposite vectors.'},
 {'id': '489dd1c9d9',
  'course': 'llm-zoomcamp',
  'section': 'General Course-Related Questions',
  'question': 'What is the video/zoom link to the stream for the “Office Hours” or live/workshop sessions?',
  'answer': 'The zoom link is only published to instructors/presenters/TAs.\n\nStudents participate via YouTube Live and submit questions to Slido (link is pinned in the chat when live). The video URL should be posted in the [announcements channel on Telegram & Slack](https://t.me/dezoomcamp) before it begins. You can also watch live on the DataTalksClub [YouTube Channel](https://www.youtube.com/c/DataTalksClub).\n\nDon’t post questions in chat as they may be missed if the room is very active.'},
 {'id': 'bd31146b0e',
  'course': 'llm-zoomcamp',
  'section': 'General Course-Related Questions',
  'question': 'When will the course be offered next?',
  'answer': 'Summer 2025.'},
 {'id': '74eb249bbf',
  'course': 'llm-zoomcamp',
  'section': 'General Course-Related Questions',
  'question': 'I just discovered the course. Can I still join?',
  'answer': 'Yes, but if you want to receive a certificate, you need to submit your project while we’re still accepting submissions.'},
 {'id': '977bf7786c',
  'course': 'llm-zoomcamp',
  'section': 'General Course-Related Questions',
  'question': 'Course: I have registered for the LLM Zoomcamp. When can I expect to receive the confirmation email?',
  'answer': "You don't need it. You're accepted. You can also just start learning and submitting homework (while the form is open) without registering. It is not checked against any registered list. Registration is just to gauge interest before the start date."}]

In [84]python · cell 7

python

def text_search(query):
    boost_dict = {"question": 3.0, "section": 0.5}

    return index.search(
        query,
        num_results=5,
        boost_dict=boost_dict
    )

In [85]python · cell 8

python

q = ground_truth[0]
q

Output

{'question': 'Is it okay to join the course late if I just found it now?',
 'document': '74eb249bbf'}

In [86]python · cell 9

python

doc_id = q['document']
doc_id

Output

'74eb249bbf'

In [87]python · cell 10

python

results = text_search(q['question'])
results

Output

[{'id': '74eb249bbf',
  'course': 'llm-zoomcamp',
  'section': 'General Course-Related Questions',
  'question': 'I just discovered the course. Can I still join?',
  'answer': 'Yes, but if you want to receive a certificate, you need to submit your project while we’re still accepting submissions.'},
 {'id': '0fab61eca2',
  'course': 'llm-zoomcamp',
  'section': 'Capstone Project',
  'question': 'Is it a group project?',
  'answer': 'No, the capstone is a solo project.'},
 {'id': '977bf7786c',
  'course': 'llm-zoomcamp',
  'section': 'General Course-Related Questions',
  'question': 'Course: I have registered for the LLM Zoomcamp. When can I expect to receive the confirmation email?',
  'answer': "You don't need it. You're accepted. You can also just start learning and submitting homework (while the form is open) without registering. It is not checked against any registered list. Registration is just to gauge interest before the start date."},
 {'id': '86d99bbf21',
  'course': 'llm-zoomcamp',
  'section': 'Module 1: Introduction to LLMs and RAG',
  'question': 'Authentication: Why is my OPENAI_API_KEY not found in the Jupyter notebook?',
  'answer': "There are two options to resolve this issue:\n\n**Option 1: Using direnv**\n\n1. Create a `.envrc` file and add your API key.\n2. Run `direnv allow` in the terminal.\n\nIf you encounter the error:\n\n```python\nOpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\n```\n\n- Install `dotenv` by running:\n\n    ```bash\n    pip install python-dotenv\n    ```\n\n- Add the following to a cell in the notebook:\n\n    ```python\n    from dotenv import load_dotenv\n    \n    load_dotenv('.envrc')\n    ```\n\n**Option 2: Using Codespaces Secrets**\n\n1. Log in to your GitHub account and navigate to **Settings > Codespaces**.\n2. In the Secrets section, create a secret like `OPENAI_API_KEY` and select the repositories for which the secret should be available.\n3. Once set up, the key will be available in your Codespaces session."},
 {'id': '489dd1c9d9',
  'course': 'llm-zoomcamp',
  'section': 'General Course-Related Questions',
  'question': 'What is the video/zoom link to the stream for the “Office Hours” or live/workshop sessions?',
  'answer': 'The zoom link is only published to instructors/presenters/TAs.\n\nStudents participate via YouTube Live and submit questions to Slido (link is pinned in the chat when live). The video URL should be posted in the [announcements channel on Telegram & Slack](https://t.me/dezoomcamp) before it begins. You can also watch live on the DataTalksClub [YouTube Channel](https://www.youtube.com/c/DataTalksClub).\n\nDon’t post questions in chat as they may be missed if the room is very active.'}]

In [88]python · cell 11

python

for d in results:
    print(f'{d["id"]} == {doc_id}: {d["id"] == doc_id}')

Output

74eb249bbf == 74eb249bbf: True
0fab61eca2 == 74eb249bbf: False
977bf7786c == 74eb249bbf: False
86d99bbf21 == 74eb249bbf: False
489dd1c9d9 == 74eb249bbf: False

In [89]python · cell 12

python

relevance = []

for d in results:
    relevance.append(int(d["id"] == doc_id))

relevance

Output

[1, 0, 0, 0, 0]

In [90]python · cell 13

python

def compute_relevance_text(q):
    doc_id = q["document"]
    results = text_search(query=q["question"])

    relevance = []
    for d in results:
        relevance.append(int(d["id"] == doc_id))

    return relevance

In [91]python · cell 14

python

q = ground_truth[0]
print(q["question"])
compute_relevance_text(q)
# [1, 0, 0, 0, 0]

Output

Is it okay to join the course late if I just found it now?

[1, 0, 0, 0, 0]

In [92]python · cell 15

python

q = ground_truth[11]
print(q["question"])
compute_relevance_text(q)
# [0, 0, 1, 0, 0]

Output

Where can I watch the live sessions for the course, and how do I ask questions during them?

[1, 0, 0, 0, 0]

In [93]python · cell 16

python

[0, 0, 1, 0, 0]

Output

[0, 0, 1, 0, 0]

In [94]python · cell 17

python

q = ground_truth[50]
print(q["question"])
compute_relevance_text(q)
# [0, 0, 0, 0, 0]

Output

Why does WSL2 say the model needs more memory even though my PC has enough RAM?

[1, 0, 0, 0, 0]

In [95]python · cell 18

python

[0, 0, 0, 0, 0]

Output

[0, 0, 0, 0, 0]

In [96]python · cell 19

python

from tqdm.auto import tqdm

def compute_relevance_total_text(ground_truth):
    relevance_total = []

    for q in tqdm(ground_truth):
        relevance = compute_relevance_text(q)
        relevance_total.append(relevance)

    return relevance_total

In [97]python · cell 20

python

relevance = compute_relevance_total_text(ground_truth)

Output

  0%|          | 0/395 [00:00<?, ?it/s]

In [98]python · cell 21

python

relevance[:15]

Output

[[1, 0, 0, 0, 0],
 [0, 1, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [0, 0, 0, 0, 0],
 [0, 1, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [0, 0, 1, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0]]

In [99]python · cell 22

python

def compute_relevance(q, search_function):
    doc_id = q["document"]
    results = search_function(query=q["question"])

    relevance = []
    for d in results:
        relevance.append(int(d["id"] == doc_id))

    return relevance

In [100]python · cell 23

python

def compute_relevance_total(ground_truth, search_function):
    relevance_total = []

    for q in tqdm(ground_truth):
        relevance = compute_relevance(q, search_function)
        relevance_total.append(relevance)

    return relevance_total

In [101]python · cell 24

python

relevance_total = compute_relevance_total(ground_truth, text_search)

Output

  0%|          | 0/395 [00:00<?, ?it/s]

In [102]python · cell 25

python

sample = relevance_total[:15]

In [103]python · cell 26

python

sample

Output

[[1, 0, 0, 0, 0],
 [0, 1, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [0, 0, 0, 0, 0],
 [0, 1, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [0, 0, 1, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0],
 [1, 0, 0, 0, 0]]

In [104]python · cell 27

python

14 / 15

Output

0.9333333333333333

In [105]python · cell 28

python

cnt = 0

for line in sample:
    if 1 in line:
        cnt = cnt + 1

cnt / len(sample)

Output

0.9333333333333333

In [106]python · cell 29

python

def hit_rate(relevance):
    cnt = 0

    for line in relevance:
        if 1 in line:
            cnt = cnt + 1

    return cnt / len(relevance)

In [109]python · cell 30

python

hit_rate(relevance)

Output

0.8987341772151899

In [110]python · cell 31

python

total_score = 0.0

for line in sample:
    for rank in range(len(line)):
        if line[rank] == 1:
            score = 1 / (rank + 1)
            total_score = total_score + score
            break

total_score / len(sample)

Output

0.8222222222222222

In [117]python · cell 32

python

def mrr(relevance):
    total_score = 0.0

    for line in relevance:
        for rank in range(len(line)):
            if line[rank] == 1:
                score = 1 / (rank + 1)
                total_score = total_score + score
                break

    return total_score / len(relevance)

In [118]python · cell 33

python

mrr(sample)

Output

0.8222222222222222

In [119]python · cell 34

python

mrr(relevance)

Output

0.7693248945147676

In [121]python · cell 35

python

def evaluate(ground_truth, search_function):
    relevance_total = compute_relevance_total(ground_truth, search_function)

    return {
        "hit_rate": hit_rate(relevance_total),
        "mrr": mrr(relevance_total),
    }

In [122]python · cell 36

python

evaluate(ground_truth, text_search)

Output

  0%|          | 0/395 [00:00<?, ?it/s]

{'hit_rate': 0.8987341772151899, 'mrr': 0.7693248945147676}

In [124]python · cell 37

python

def text_search_v2(query):
    boost_dict = {"question": 2.0, "section": 0.5}

    return index.search(
        query,
        num_results=5,
        boost_dict=boost_dict
    )

In [125]python · cell 38

python

evaluate(ground_truth, text_search_v2)

Output

  0%|          | 0/395 [00:00<?, ?it/s]

{'hit_rate': 0.9088607594936708, 'mrr': 0.791561181434599}

In [126]python · cell 39

python

def search_boost(query, question_boost):
    boost_dict = {"question": question_boost, "section": 0.5}

    return index.search(
        query,
        num_results=5,
        boost_dict=boost_dict,
    )

In [127]python · cell 40

python

for boost in [0.5, 1.0, 3.0, 5.0, 10.0]:
    result = evaluate(
        ground_truth,
        lambda query, boost=boost: search_boost(query, boost)
    )
    print(f"boost={boost}: {result}")

Output

  0%|          | 0/395 [00:00<?, ?it/s]

boost=0.5: {'hit_rate': 0.9113924050632911, 'mrr': 0.800548523206751}

  0%|          | 0/395 [00:00<?, ?it/s]

boost=1.0: {'hit_rate': 0.9240506329113924, 'mrr': 0.8139240506329113}

  0%|          | 0/395 [00:00<?, ?it/s]

boost=3.0: {'hit_rate': 0.8987341772151899, 'mrr': 0.7693248945147676}

  0%|          | 0/395 [00:00<?, ?it/s]

boost=5.0: {'hit_rate': 0.8708860759493671, 'mrr': 0.7401265822784809}

  0%|          | 0/395 [00:00<?, ?it/s]

boost=10.0: {'hit_rate': 0.8582278481012658, 'mrr': 0.7122362869198313}

In [128]python · cell 41

python

def search_boosts(query, question_boost, answer_boost, section_boost):
    boost_dict = {
        "question": question_boost,
        "section": section_boost,
        "answer": answer_boost,
    }

    return index.search(
        query,
        num_results=5,
        boost_dict=boost_dict,
    )

In [129]python · cell 42

python

results = []

for question_boost in [1.0, 2.0, 5.0]:
    for answer_boost in [1.0, 2.0, 4.0, 10.0]:
        for section_boost in [0.1, 0.2, 0.5]:
            print(f"Evaluating question_boost={question_boost}, answer_boost={answer_boost}, section_boost={section_boost}...")
            result = evaluate(
                ground_truth,
                lambda query, question_boost=question_boost, answer_boost=answer_boost, section_boost=section_boost: search_boosts(
                    query,
                    question_boost,
                    answer_boost,
                    section_boost
                )
            )

            results.append({
                "question": question_boost,
                "answer": answer_boost,
                "section": section_boost,
                "hit_rate": result["hit_rate"],
                "mrr": result["mrr"],
            })

Output

Evaluating question_boost=1.0, answer_boost=1.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=1.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=1.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=2.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=2.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=2.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=4.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=4.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=4.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=10.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=10.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=1.0, answer_boost=10.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=1.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=1.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=1.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=2.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=2.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=2.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=4.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=4.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=4.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=10.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=10.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=2.0, answer_boost=10.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=1.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=1.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=1.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=2.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=2.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=2.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=4.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=4.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=4.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=10.0, section_boost=0.1...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=10.0, section_boost=0.2...

  0%|          | 0/395 [00:00<?, ?it/s]

Evaluating question_boost=5.0, answer_boost=10.0, section_boost=0.5...

  0%|          | 0/395 [00:00<?, ?it/s]

In [132]python · cell 43

python

df_results = pd.DataFrame(results)
df_results.sort_values("mrr", ascending=False).head(10)

Output

    question  answer  section  hit_rate       mrr
3        1.0     2.0      0.1  0.974684  0.884515
19       2.0     4.0      0.2  0.974684  0.884515
35       5.0    10.0      0.5  0.974684  0.884515
34       5.0    10.0      0.2  0.974684  0.883797
33       5.0    10.0      0.1  0.974684  0.883671
18       2.0     4.0      0.1  0.974684  0.883671
20       2.0     4.0      0.5  0.977215  0.883629
4        1.0     2.0      0.2  0.977215  0.883544
5        1.0     2.0      0.5  0.964557  0.862447
6        1.0     4.0      0.1  0.969620  0.861561

.dataframe tbody tr th:only-of-type { vertical-align: middle; } .dataframe tbody tr th { vertical-align: top; } .dataframe thead th { text-align: right; }

	question	answer	section	hit_rate	mrr
3	1.0	2.0	0.1	0.974684	0.884515
19	2.0	4.0	0.2	0.974684	0.884515
35	5.0	10.0	0.5	0.974684	0.884515
34	5.0	10.0	0.2	0.974684	0.883797
33	5.0	10.0	0.1	0.974684	0.883671
18	2.0	4.0	0.1	0.974684	0.883671
20	2.0	4.0	0.5	0.977215	0.883629
4	1.0	2.0	0.2	0.977215	0.883544
5	1.0	2.0	0.5	0.964557	0.862447
6	1.0	4.0	0.1	0.969620	0.861561

In [ ]python · cell 44

python

question course document 0 Can I take this course at my own pace and stil... llm-zoomcamp 69d122f12e 1 Is a certificate available if I complete the c... llm-zoomcamp 69d122f12e 2 Do self-paced learners get any certificate for... llm-zoomcamp 69d122f12e 3 Why are certificates not issued for the self-p... llm-zoomcamp 69d122f12e 4 Is peer review of capstone projects required i... llm-zoomcamp 69d122f12e

question

course

document

Can I take this course at my own pace and stil...

llm-zoomcamp

69d122f12e

Is a certificate available if I complete the c...

llm-zoomcamp

69d122f12e

Do self-paced learners get any certificate for...

llm-zoomcamp

69d122f12e

Why are certificates not issued for the self-p...

llm-zoomcamp

69d122f12e

Is peer review of capstone projects required i...

llm-zoomcamp

69d122f12e

from ingest import load_faq_data, build_index documents = load_faq_data() documents_llm = [] for doc in documents: if doc["course"] == "llm-zoomcamp": documents_llm.append(doc) documents = documents_llm index = build_index(documents)

[{'id': 'db78580409', 'course': 'llm-zoomcamp', 'section': 'Module 3: Vector Search', 'question': 'What is the cosine similarity?', 'answer': 'Cosine similarity is a measure used to calculate the similarity between two non-zero vectors, often used in text analysis to determine how similar two documents are based on their content. This metric computes the cosine of the angle between two vectors, which are typically word counts or TF-IDF values of the documents. The cosine similarity value ranges from -1 to 1, where 1 indicates that the vectors are identical, 0 indicates that the vectors are orthogonal (no similarity), and -1 represents completely opposite vectors.'}, {'id': '489dd1c9d9', 'course': 'llm-zoomcamp', 'section': 'General Course-Related Questions', 'question': 'What is the video/zoom link to the stream for the “Office Hours” or live/workshop sessions?', 'answer': 'The zoom link is only published to instructors/presenters/TAs.\n\nStudents participate via YouTube Live and submit questions to Slido (link is pinned in the chat when live). The video URL should be posted in the [announcements channel on Telegram & Slack](https://t.me/dezoomcamp) before it begins. You can also watch live on the DataTalksClub [YouTube Channel](https://www.youtube.com/c/DataTalksClub).\n\nDon’t post questions in chat as they may be missed if the room is very active.'}, {'id': 'bd31146b0e', 'course': 'llm-zoomcamp', 'section': 'General Course-Related Questions', 'question': 'When will the course be offered next?', 'answer': 'Summer 2025.'}, {'id': '74eb249bbf', 'course': 'llm-zoomcamp', 'section': 'General Course-Related Questions', 'question': 'I just discovered the course. Can I still join?', 'answer': 'Yes, but if you want to receive a certificate, you need to submit your project while we’re still accepting submissions.'}, {'id': '977bf7786c', 'course': 'llm-zoomcamp', 'section': 'General Course-Related Questions', 'question': 'Course: I have registered for the LLM Zoomcamp. When can I expect to receive the confirmation email?', 'answer': "You don't need it. You're accepted. You can also just start learning and submitting homework (while the form is open) without registering. It is not checked against any registered list. Registration is just to gauge interest before the start date."}]

[{'id': '74eb249bbf', 'course': 'llm-zoomcamp', 'section': 'General Course-Related Questions', 'question': 'I just discovered the course. Can I still join?', 'answer': 'Yes, but if you want to receive a certificate, you need to submit your project while we’re still accepting submissions.'}, {'id': '0fab61eca2', 'course': 'llm-zoomcamp', 'section': 'Capstone Project', 'question': 'Is it a group project?', 'answer': 'No, the capstone is a solo project.'}, {'id': '977bf7786c', 'course': 'llm-zoomcamp', 'section': 'General Course-Related Questions', 'question': 'Course: I have registered for the LLM Zoomcamp. When can I expect to receive the confirmation email?', 'answer': "You don't need it. You're accepted. You can also just start learning and submitting homework (while the form is open) without registering. It is not checked against any registered list. Registration is just to gauge interest before the start date."}, {'id': '86d99bbf21', 'course': 'llm-zoomcamp', 'section': 'Module 1: Introduction to LLMs and RAG', 'question': 'Authentication: Why is my OPENAI_API_KEY not found in the Jupyter notebook?', 'answer': "There are two options to resolve this issue:\n\n**Option 1: Using direnv**\n\n1. Create a `.envrc` file and add your API key.\n2. Run `direnv allow` in the terminal.\n\nIf you encounter the error:\n\n```python\nOpenAIError: The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable\n```\n\n- Install `dotenv` by running:\n\n ```bash\n pip install python-dotenv\n ```\n\n- Add the following to a cell in the notebook:\n\n ```python\n from dotenv import load_dotenv\n \n load_dotenv('.envrc')\n ```\n\n**Option 2: Using Codespaces Secrets**\n\n1. Log in to your GitHub account and navigate to **Settings > Codespaces**.\n2. In the Secrets section, create a secret like `OPENAI_API_KEY` and select the repositories for which the secret should be available.\n3. Once set up, the key will be available in your Codespaces session."}, {'id': '489dd1c9d9', 'course': 'llm-zoomcamp', 'section': 'General Course-Related Questions', 'question': 'What is the video/zoom link to the stream for the “Office Hours” or live/workshop sessions?', 'answer': 'The zoom link is only published to instructors/presenters/TAs.\n\nStudents participate via YouTube Live and submit questions to Slido (link is pinned in the chat when live). The video URL should be posted in the [announcements channel on Telegram & Slack](https://t.me/dezoomcamp) before it begins. You can also watch live on the DataTalksClub [YouTube Channel](https://www.youtube.com/c/DataTalksClub).\n\nDon’t post questions in chat as they may be missed if the room is very active.'}]

def compute_relevance_text(q): doc_id = q["document"] results = text_search(query=q["question"]) relevance = [] for d in results: relevance.append(int(d["id"] == doc_id)) return relevance

from tqdm.auto import tqdm def compute_relevance_total_text(ground_truth): relevance_total = [] for q in tqdm(ground_truth): relevance = compute_relevance_text(q) relevance_total.append(relevance) return relevance_total

[[1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 0, 0, 0], [0, 1, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0], [1, 0, 0, 0, 0]]

def compute_relevance(q, search_function): doc_id = q["document"] results = search_function(query=q["question"]) relevance = [] for d in results: relevance.append(int(d["id"] == doc_id)) return relevance

def compute_relevance_total(ground_truth, search_function): relevance_total = [] for q in tqdm(ground_truth): relevance = compute_relevance(q, search_function) relevance_total.append(relevance) return relevance_total

total_score = 0.0 for line in sample: for rank in range(len(line)): if line[rank] == 1: score = 1 / (rank + 1) total_score = total_score + score break total_score / len(sample)

def mrr(relevance): total_score = 0.0 for line in relevance: for rank in range(len(line)): if line[rank] == 1: score = 1 / (rank + 1) total_score = total_score + score break return total_score / len(relevance)

def evaluate(ground_truth, search_function): relevance_total = compute_relevance_total(ground_truth, search_function) return { "hit_rate": hit_rate(relevance_total), "mrr": mrr(relevance_total), }

def search_boost(query, question_boost): boost_dict = {"question": question_boost, "section": 0.5} return index.search( query, num_results=5, boost_dict=boost_dict, )

def search_boosts(query, question_boost, answer_boost, section_boost): boost_dict = { "question": question_boost, "section": section_boost, "answer": answer_boost, } return index.search( query, num_results=5, boost_dict=boost_dict, )

results = [] for question_boost in [1.0, 2.0, 5.0]: for answer_boost in [1.0, 2.0, 4.0, 10.0]: for section_boost in [0.1, 0.2, 0.5]: print(f"Evaluating question_boost={question_boost}, answer_boost={answer_boost}, section_boost={section_boost}...") result = evaluate( ground_truth, lambda query, question_boost=question_boost, answer_boost=answer_boost, section_boost=section_boost: search_boosts( query, question_boost, answer_boost, section_boost ) ) results.append({ "question": question_boost, "answer": answer_boost, "section": section_boost, "hit_rate": result["hit_rate"], "mrr": result["mrr"], })

question answer section hit_rate mrr 3 1.0 2.0 0.1 0.974684 0.884515 19 2.0 4.0 0.2 0.974684 0.884515 35 5.0 10.0 0.5 0.974684 0.884515 34 5.0 10.0 0.2 0.974684 0.883797 33 5.0 10.0 0.1 0.974684 0.883671 18 2.0 4.0 0.1 0.974684 0.883671 20 2.0 4.0 0.5 0.977215 0.883629 4 1.0 2.0 0.2 0.977215 0.883544 5 1.0 2.0 0.5 0.964557 0.862447 6 1.0 4.0 0.1 0.969620 0.861561

question

answer

section

hit_rate

mrr

1.0

2.0

0.1

0.974684

0.884515

2.0

4.0

0.2

0.974684

0.884515

5.0

10.0

0.5

0.974684

0.884515

5.0

10.0

0.2

0.974684

0.883797

5.0

10.0

0.1

0.974684

0.883671

2.0

4.0

0.1

0.974684

0.883671

2.0

4.0

0.5

0.977215

0.883629

1.0

2.0

0.2

0.977215

0.883544

1.0

2.0

0.5

0.964557

0.862447

1.0

4.0

0.1

0.969620

0.861561