ollama - 💡(How to fix) Fix native ollama-go-engine: TurboQuant implementation [29 comments, 21 participants]

import torch
import torch.nn as nn

# Define a simple neural network
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.fc1 = nn.Linear(784, 128)
        self.fc2 = nn.Linear(128, 10)

    def forward(self, x):
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x

# Implement knowledge distillation
class DistilledNet(nn.Module):
    def __init__(self, teacher_model):
        super(DistilledNet, self).__init__()
        self.teacher_model = teacher_model
        self.fc1 = nn.Linear(784, 64)
        self.fc2 = nn.Linear(64, 10)

    def forward(self, x):
        teacher_output = self.teacher_model(x)
        x = torch.relu(self.fc1(x))
        x = self.fc2(x)
        return x, teacher_output

# Initialize models and optimizer
model = Net()
distilled_model = DistilledNet(model)
optimizer = torch.optim.Adam(distilled_model.parameters(), lr=0.001)

# Train distilled model
for epoch in range(10):
    optimizer.zero_grad()
    inputs = torch.randn(100, 784)
    outputs, teacher_outputs = distilled_model(inputs)
    loss = torch.mean((outputs - teacher_outputs) ** 2)
    loss.backward()
    optimizer.step()