Chapter 102
huggingface mistral 7b
Notebooksaturn (Python 3)14 cells
If you're not running in Saturn Cloud, you need to install these libraries:
Make sure you use the latest versions
code
pip install -U transformers accelerate bitsandbytesIn [1]python · cell 2
python
import os
os.environ['HF_HOME'] = '/run/cache/'In [1]python · cell 3
python
!rm -f minsearch.py
!wget https://raw.githubusercontent.com/alexeygrigorev/minsearch/main/minsearch.pyOutput
--2024-06-13 19:36:42-- https://raw.githubusercontent.com/alexeygrigorev/minsearch/main/minsearch.py Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.111.133, 185.199.108.133, ... Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected. HTTP request sent, awaiting response... 200 OK Length: 3832 (3.7K) [text/plain] Saving to: ‘minsearch.py’ minsearch.py 100%[===================>] 3.74K --.-KB/s in 0s 2024-06-13 19:36:42 (50.3 MB/s) - ‘minsearch.py’ saved [3832/3832]
In [2]python · cell 4
python
import requests
import minsearch
docs_url = 'https://github.com/DataTalksClub/llm-zoomcamp/blob/main/01-intro/documents.json?raw=1'
docs_response = requests.get(docs_url)
documents_raw = docs_response.json()
documents = []
for course in documents_raw:
course_name = course['course']
for doc in course['documents']:
doc['course'] = course_name
documents.append(doc)
index = minsearch.Index(
text_fields=["question", "text", "section"],
keyword_fields=["course"]
)
index.fit(documents)Output
<minsearch.Index at 0x7fc130288970>
In [44]python · cell 5
python
def search(query):
boost = {'question': 3.0, 'section': 0.5}
results = index.search(
query=query,
filter_dict={'course': 'data-engineering-zoomcamp'},
boost_dict=boost,
num_results=3
)
return resultsIn [4]python · cell 6
python
def rag(query):
search_results = search(query)
prompt = build_prompt(query, search_results)
answer = llm(prompt)
return answerIn [8]python · cell 7
python
os.environ['HF_TOKEN'] = 'hf_blabla'In [10]python · cell 8
python
from huggingface_hub import loginIn [11]python · cell 9
python
login(token=os.environ['HF_TOKEN'])Output
The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well. Token is valid (permission: read). Your token has been saved to /run/cache/token Login successful
In [5]python · cell 10
python
from transformers import AutoModelForCausalLM
from transformers import AutoTokenizerIn [12]python · cell 11
python
model = AutoModelForCausalLM.from_pretrained(
"mistralai/Mistral-7B-v0.1", device_map="auto", load_in_4bit=True
)
tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1", padding_side="left")Output
config.json: 0%| | 0.00/571 [00:00<?, ?B/s]
The `load_in_4bit` and `load_in_8bit` arguments are deprecated and will be removed in the future versions. Please, pass a `BitsAndBytesConfig` object in `quantization_config` argument instead.
model.safetensors.index.json: 0%| | 0.00/25.1k [00:00<?, ?B/s]
Downloading shards: 0%| | 0/2 [00:00<?, ?it/s]
model-00001-of-00002.safetensors: 0%| | 0.00/9.94G [00:00<?, ?B/s]
model-00002-of-00002.safetensors: 0%| | 0.00/4.54G [00:00<?, ?B/s]
Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]
generation_config.json: 0%| | 0.00/116 [00:00<?, ?B/s]
tokenizer_config.json: 0%| | 0.00/967 [00:00<?, ?B/s]
tokenizer.model: 0%| | 0.00/493k [00:00<?, ?B/s]
tokenizer.json: 0%| | 0.00/1.80M [00:00<?, ?B/s]
special_tokens_map.json: 0%| | 0.00/72.0 [00:00<?, ?B/s]
In [ ]python · cell 12
python
from transformers import pipeline
generator = pipeline("text-generation", model=model, tokenizer=tokenizer)In [46]python · cell 13
python
def build_prompt(query, search_results):
prompt_template = """
QUESTION: {question}
CONTEXT:
{context}
ANSWER:
""".strip()
context = ""
for doc in search_results:
context = context + f"{doc['question']}\n{doc['text']}\n\n"
prompt = prompt_template.format(question=query, context=context).strip()
return prompt
def llm(prompt):
response = generator(prompt, max_length=500, temperature=0.7, top_p=0.95, num_return_sequences=1)
response_final = response[0]['generated_text']
return response_final[len(prompt):].strip()In [47]python · cell 14
python
rag("I just discovered the course. Can I still join it?")Output
Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.
'Yes, you can still join the course.'