Chapter 59
01 data gen
Notebookllm-zoomcamp-2026-evals36 cells
In [2]python · cell 1
python
from ingest import load_faq_data
documents = load_faq_data()In [3]python · cell 2
python
documents[10]Output
{'id': '2b5ff70c77',
'course': 'machine-learning-zoomcamp',
'section': 'General Course-Related Questions',
'question': 'Do I need to enroll in the course before submitting homework?',
'answer': 'No enrollment is required to submit homework. Just log into the homework form when it opens. The Airtable registration you may see is only for announcements; actual submissions are made on the course platform forms and via your GitHub as specified in the homework guidelines.'}In [3]python · cell 3
python
documents_llm = []
for doc in documents:
if doc["course"] == "llm-zoomcamp":
documents_llm.append(doc)
len(documents_llm)Output
79
In [4]python · cell 4
python
documents = documents_llmIn [11]python · cell 5
python
doc = documents[0]
print(doc["id"])
print(doc["question"])
print(doc["answer"])Output
74eb249bbf I just discovered the course. Can I still join? Yes, but if you want to receive a certificate, you need to submit your project while we’re still accepting submissions.
In [5]python · cell 6
python
from pydantic import BaseModel
class Questions(BaseModel):
questions: list[str]In [6]python · cell 7
python
data_gen_instructions = """
You emulate a student who's taking our course.
Formulate 5 questions this student might ask based on a FAQ record. The record
should contain the answer to the questions, and the questions should be complete and not too short.
If possible, use as fewer words as possible from the record.
The output should resemble how people ask questions
on the internet. Not too formal, not too short, not too long.
""".strip()In [7]python · cell 8
python
from dotenv import load_dotenv
from openai import OpenAI
load_dotenv()
openai_client = OpenAI()In [8]python · cell 9
python
import json
user_prompt = json.dumps(doc)In [17]python · cell 10
python
user_promptOutput
'{"id": "74eb249bbf", "course": "llm-zoomcamp", "section": "General Course-Related Questions", "question": "I just discovered the course. Can I still join?", "answer": "Yes, but if you want to receive a certificate, you need to submit your project while we\\u2019re still accepting submissions."}'In [23]python · cell 11
python
messages = [
{"role": "developer", "content": data_gen_instructions},
{"role": "user", "content": user_prompt}
]In [24]python · cell 12
python
response = openai_client.responses.parse(
model="gpt-5.4-mini",
input=messages,
text_format=Questions
)In [25]python · cell 13
python
response.output_parsed.questionsOutput
['I just found this course — is it too late to join now?', 'Can I still enroll if I discovered the course after it started?', 'If I join late, will I still be able to get a certificate?', 'What do I need to do to qualify for the certificate if I’m starting now?', 'Is the only deadline the project submission for getting the certificate?']
In [27]python · cell 14
python
docOutput
{'id': '74eb249bbf',
'course': 'llm-zoomcamp',
'section': 'General Course-Related Questions',
'question': 'I just discovered the course. Can I still join?',
'answer': 'Yes, but if you want to receive a certificate, you need to submit your project while we’re still accepting submissions.'}In [9]python · cell 15
python
from evaluation_utils import llm_structuredIn [10]python · cell 16
python
result, usage = llm_structured(
openai_client,
data_gen_instructions,
user_prompt,
Questions
)
print(result.questions)Output
['How is the capstone homework graded, and what do I need to do to get all the points?', 'What are the point values for answering the questions, adding learning resources, and submitting a FAQ question?', 'How many points can I earn from one homework in this project?', 'Does the homework score depend on correct answers only, or are public learning items and FAQ contributions counted too?', 'What tasks should I complete to maximize my homework score and improve my leaderboard position?']
In [31]python · cell 17
python
usageOutput
ResponseUsage(input_tokens=207, input_tokens_details=InputTokensDetails(cached_tokens=0), output_tokens=105, output_tokens_details=OutputTokensDetails(reasoning_tokens=0), total_tokens=312)
In [11]python · cell 18
python
from evaluation_utils import calc_priceIn [12]python · cell 19
python
calc_price(usage)Output
{'input_cost': 0.00019725, 'output_cost': 0.0004815, 'total_cost': 0.00067875}In [13]python · cell 20
python
records = []
for q in result.questions:
records.append({
"question": q,
"document": doc["id"]
})
recordsOutput
[{'question': 'How is the capstone homework graded, and what do I need to do to get all the points?',
'document': '0d200c8c58'},
{'question': 'What are the point values for answering the questions, adding learning resources, and submitting a FAQ question?',
'document': '0d200c8c58'},
{'question': 'How many points can I earn from one homework in this project?',
'document': '0d200c8c58'},
{'question': 'Does the homework score depend on correct answers only, or are public learning items and FAQ contributions counted too?',
'document': '0d200c8c58'},
{'question': 'What tasks should I complete to maximize my homework score and improve my leaderboard position?',
'document': '0d200c8c58'}]In [14]python · cell 21
python
import pandas as pdIn [15]python · cell 22
python
pd.DataFrame(records)Output
question document 0 How is the capstone homework graded, and what ... 0d200c8c58 1 What are the point values for answering the qu... 0d200c8c58 2 How many points can I earn from one homework i... 0d200c8c58 3 Does the homework score depend on correct answ... 0d200c8c58 4 What tasks should I complete to maximize my ho... 0d200c8c58
.dataframe tbody tr th:only-of-type {
vertical-align: middle;
}
.dataframe tbody tr th {
vertical-align: top;
}
.dataframe thead th {
text-align: right;
}
| question | document | |
|---|---|---|
| 0 | How is the capstone homework graded, and what ... | 0d200c8c58 |
| 1 | What are the point values for answering the qu... | 0d200c8c58 |
| 2 | How many points can I earn from one homework i... | 0d200c8c58 |
| 3 | Does the homework score depend on correct answ... | 0d200c8c58 |
| 4 | What tasks should I complete to maximize my ho... | 0d200c8c58 |
In [16]python · cell 23
python
from evaluation_utils import llm_structured_retryIn [17]python · cell 24
python
def generate_ground_truth(doc):
user_prompt = json.dumps(doc)
out, usage = llm_structured_retry(
openai_client,
data_gen_instructions,
user_prompt,
Questions
)
results = []
for q in out.questions:
results.append({
"question": q,
"document": doc["id"]
})
return results, usageIn [18]python · cell 25
python
generate_ground_truth(doc)Output
([{'question': 'How are the capstone homework points calculated?',
'document': '0d200c8c58'},
{'question': 'What do I need to do to get full score on a homework?',
'document': '0d200c8c58'},
{'question': 'How many points can I earn from one homework assignment in this project?',
'document': '0d200c8c58'},
{'question': 'Is there a point breakdown for answers, learning items, and FAQ questions?',
'document': '0d200c8c58'},
{'question': 'What actions count toward the homework leaderboard score?',
'document': '0d200c8c58'}],
ResponseUsage(input_tokens=263, input_tokens_details=InputTokensDetails(cached_tokens=0), output_tokens=77, output_tokens_details=OutputTokensDetails(reasoning_tokens=0), total_tokens=340))In [19]python · cell 26
python
from tqdm.auto import tqdm
ground_truth = []
usages = []
for doc in tqdm(documents[:5]):
records, usage = generate_ground_truth(doc)
ground_truth.extend(records)
usages.append(usage)Output
0%| | 0/5 [00:00<?, ?it/s]
In [20]python · cell 27
python
from concurrent.futures import ThreadPoolExecutor
from evaluation_utils import map_progressIn [21]python · cell 28
python
with ThreadPoolExecutor(max_workers=6) as pool:
results = map_progress(pool, documents, generate_ground_truth)Output
0%| | 0/79 [00:00<?, ?it/s]
In [24]python · cell 29
python
ground_truth = []
usages = []
for records, usage in results:
ground_truth.extend(records)
usages.append(usage)
len(ground_truth)Output
395
In [27]python · cell 30
python
ground_truth[10]Output
{'question': 'How do I join the Office Hours or live workshop if I don’t have the Zoom link?',
'document': '489dd1c9d9'}In [28]python · cell 31
python
from evaluation_utils import calc_price
total_cost = 0.0
for usage in usages:
cost = calc_price(usage)
total_cost = total_cost + cost["total_cost"]
total_costOutput
0.05718675000000001
In [29]python · cell 32
python
from evaluation_utils import calc_total_price
calc_total_price(usages)Output
0.05718675000000001
In [30]python · cell 33
python
df_ground_truth = pd.DataFrame(ground_truth)In [32]python · cell 34
python
df_ground_truth.to_csv("data/ground_truth.csv", index=False)In [33]python · cell 35
python
len(df_ground_truth)Output
395
In [ ]python · cell 36
python