Chapter 101
huggingface flan t5
Notebooksaturn (Python 3)13 cells
If you're not running in Saturn Cloud, you need to install these libraries:
Make sure you use the latest versions
code
pip install -U transformers accelerate bitsandbytesIn [2]python · cell 2
python
import os
os.environ['HF_HOME'] = '/run/cache/'In [1]python · cell 3
python
!rm -f minsearch.py
!wget https://raw.githubusercontent.com/alexeygrigorev/minsearch/main/minsearch.pyOutput
--2024-06-13 19:36:42-- https://raw.githubusercontent.com/alexeygrigorev/minsearch/main/minsearch.py Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.111.133, 185.199.108.133, ... Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 3832 (3.7K) [text/plain] Saving to: ‘minsearch.py’ minsearch.py 100%[===================>] 3.74K --.-KB/s in 0s 2024-06-13 19:36:42 (50.3 MB/s) - ‘minsearch.py’ saved [3832/3832]
In [3]python · cell 4
python
import requests
import minsearch
docs_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main/01-intro/documents.json?raw=1'
docs_response = requests.get(docs_url)
documents_raw = docs_response.json()
documents = []
for course in documents_raw:
course_name = course['course']
for doc in course['documents']:
doc['course'] = course_name
documents.append(doc)
index = minsearch.Index(
text_fields=["question", "text", "section"],
keyword_fields=["course"]
)
index.fit(documents)Output
<minsearch.Index at 0x7f07b4322940>
In [4]python · cell 5
python
def search(query):
boost = {'question': 3.0, 'section': 0.5}
results = index.search(
query=query,
filter_dict={'course': 'data-engineering-zoomcamp'},
boost_dict=boost,
num_results=5
)
return resultsIn [6]python · cell 6
python
def rag(query):
search_results = search(query)
prompt = build_prompt(query, search_results)
answer = llm(prompt)
return answerIn [7]python · cell 7
python
!df -hOutput
Filesystem Size Used Avail Use% Mounted on overlay 100G 36G 65G 36% / tmpfs 64M 0 64M 0% /dev tmpfs 7.7G 0 7.7G 0% /sys/fs/cgroup /dev/nvme0n1p1 100G 36G 65G 36% /run tmpfs 14G 0 14G 0% /dev/shm /dev/nvme2n1 2.0G 1.1G 858M 56% /home/jovyan tmpfs 14G 120K 14G 1% /home/jovyan/.saturn tmpfs 14G 12K 14G 1% /run/secrets/kubernetes.io/serviceaccount tmpfs 7.7G 12K 7.7G 1% /proc/driver/nvidia tmpfs 7.7G 3.6M 7.7G 1% /run/nvidia-persistenced/socket tmpfs 7.7G 0 7.7G 0% /proc/acpi tmpfs 7.7G 0 7.7G 0% /sys/firmware
In [8]python · cell 8
python
from transformers import T5Tokenizer, T5ForConditionalGenerationIn [9]python · cell 9
python
tokenizer = T5Tokenizer.from_pretrained("google/flan-t5-xl")
model = T5ForConditionalGeneration.from_pretrained("google/flan-t5-xl", device_map="auto")Output
tokenizer_config.json: 0%| | 0.00/2.54k [00:00<?, ?B/s]
spiece.model: 0%| | 0.00/792k [00:00<?, ?B/s]
special_tokens_map.json: 0%| | 0.00/2.20k [00:00<?, ?B/s]
tokenizer.json: 0%| | 0.00/2.42M [00:00<?, ?B/s]
You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565 Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.
config.json: 0%| | 0.00/1.44k [00:00<?, ?B/s]
model.safetensors.index.json: 0%| | 0.00/53.0k [00:00<?, ?B/s]
Downloading shards: 0%| | 0/2 [00:00<?, ?it/s]
model-00001-of-00002.safetensors: 0%| | 0.00/9.45G [00:00<?, ?B/s]
model-00002-of-00002.safetensors: 0%| | 0.00/1.95G [00:00<?, ?B/s]
Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]
generation_config.json: 0%| | 0.00/147 [00:00<?, ?B/s]
In [26]python · cell 10
python
def build_prompt(query, search_results):
prompt_template = """
You're a course teaching assistant. Answer the QUESTION based on the CONTEXT from the FAQ database.
Use only the facts from the CONTEXT when answering the QUESTION.
QUESTION: {question}
CONTEXT:
{context}
""".strip()
context = ""
for doc in search_results:
context = context + f"section: {doc['section']}\nquestion: {doc['question']}\nanswer: {doc['text']}\n\n"
prompt = prompt_template.format(question=query, context=context).strip()
return prompt
def llm(prompt):
input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to("cuda")
outputs = model.generate(input_ids, )
result = tokenizer.decode(outputs[0])
return resultIn [29]python · cell 11
python
def llm(prompt, generate_params=None):
if generate_params is None:
generate_params = {}
input_ids = tokenizer(prompt, return_tensors="pt").input_ids.to("cuda")
outputs = model.generate(
input_ids,
max_length=generate_params.get("max_length", 100),
num_beams=generate_params.get("num_beams", 5),
do_sample=generate_params.get("do_sample", False),
temperature=generate_params.get("temperature", 1.0),
top_k=generate_params.get("top_k", 50),
top_p=generate_params.get("top_p", 0.95),
)
result = tokenizer.decode(outputs[0], skip_special_tokens=True)
return resultIn [30]python · cell 12
python
rag("I just discovered the course. Can I still join it?")Output
"Yes, even if you don't register, you're still eligible to submit the homeworks. Be aware, however, that there will be deadlines for turning in the final projects. So don't leave everything for the last minute."
In [ ]python · cell 13
python