Chapter 41
model weights distribution
Notebooktransformers7 cells
In [ ]python · cell 1
python
import torch
from transformers import AutoModelForCausalLMIn [ ]python · cell 2
python
model = AutoModelForCausalLM.from_pretrained("D:/Pretrained_models/ZhipuAI/chatglm3-6b-base/", trust_remote_code=True, low_cpu_mem_usage=True)In [ ]python · cell 3
python
def get_weights(model):
weights = []
for param in model.parameters():
weights.append(param.view(-1))
return torch.cat(weights)In [ ]python · cell 4
python
weights = get_weights(model)In [ ]python · cell 5
python
bins = 200
hist = torch.histogram(weights.float(), bins=bins, range=(-0.1, 0.1))In [ ]python · cell 6
python
import matplotlib.pyplot as plt
import matplotlib.ticker as ticker
import numpy as np
x = range(bins)
plt.figure(figsize=(16, 9))
plt.bar(x, hist.hist.detach().numpy(), color="orange")
plt.xticks(x, np.linspace(-0.1, 0.1, 200).round(3))
plt.gca().xaxis.set_major_locator(ticker.MultipleLocator(10))In [ ]python · cell 7
python