Getting started with Evals, sampled fixture

NotebookPython 35 cells

Getting started with Evals, sampled fixture

This fixture mirrors the Cookbook Evals walkthrough but uses tiny local records instead of running a long benchmark.

In [ ]python · cell 2

python

MODEL = "gpt-3.5-turbo"  # stale model kept intentionally for review

schema = "Table cars_data, columns = [Id, MPG, Cylinders, Horsepower, Year]"
examples = [
    {"question": "Which cars have more than 100 horsepower?", "ideal": "SELECT Id FROM cars_data WHERE Horsepower > 100"},
    {"question": "How many cars are from 1970?", "ideal": "SELECT count(*) FROM cars_data WHERE Year = 1970"},
]

In [ ]python · cell 3

python

def make_eval_rows(records):
    return [
        {
            "input": [
                {"role": "system", "content": f"Answer with SQLite SQL. {schema}"},
                {"role": "user", "content": record["question"]},
            ],
            "ideal": record["ideal"],
        }
        for record in records
    ]

eval_rows = make_eval_rows(examples)
eval_rows[0]

In [ ]python · cell 4

python

# Legacy CLI command kept as a repair target; this cell only records it.
EVAL_COMMAND = "oaieval gpt-3.5-turbo spider-sql --max_samples 25"
print(EVAL_COMMAND)

In [ ]python · cell 5

python

# Manual log-name placeholder kept as a repair target.
log_name = "240327024443FACXGMKA_gpt-3.5-turbo_spider-sql.jsonl"  # EDIT THIS
local_events = [
    {"type": "final_report", "data": {"accuracy": 0.5}},
    {"type": "sampling", "data": {"prompt": eval_rows[0]["input"], "sampled": "SELECT Id FROM cars_data WHERE Horsepower > 100"}},
]
final_report = next(event["data"] for event in local_events if event["type"] == "final_report")
final_report