/ PYTORCH

Pytorch - Simple Regression

Simple Linear Regression

import torch
import numpy as np

Dataset

x = torch.FloatTensor([[1],[2],[3],[4],[5],[6],[7],[8],[9],[10]])
y = x * 2

Linear Model

$y = Wx + b$

W = torch.zeros(1, requires_grad=True)
b = torch.zeros(1, requires_grad=True)
print(W, b)

tensor([0.], requires_grad=True) tensor([0.], requires_grad=True)

y_hat = x * W + b
print(y_hat.ravel())

tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0.], grad_fn=<ViewBackward>)

Loss Function

$loss(W, b) = \frac{1}{n} \sum_{i=1}^{n}[y^{(i)} - H(x^{(i)})]^{2}$

loss_fn = torch.mean((y_hat-y) ** 2)
print(loss_fn)

tensor(154., grad_fn=<MeanBackward0>)

SGD

optimizer = torch.optim.SGD([W, b], lr=0.01)

print(W.data, W.grad, b.data, b.grad)

tensor([0.]) None tensor([0.]) None

optimizer.zero_grad() # gradient를 0으로 초기화
loss_fn.backward() # loss_fn을 미분하여 gradient 계산
optimizer.step() # W, b update

print(W.data, W.grad, b.data, b.grad)

tensor([1.5400]) tensor([-154.0000]) tensor([0.2200]) tensor([-22.])

Summary

x = torch.FloatTensor([[1],[2],[3],[4],[5],[6],[7],[8],[9],[10]])
y = x * 2

W = torch.zeros(1, requires_grad=True)
b = torch.zeros(1, requires_grad=True)

optimizer = torch.optim.SGD([W, b], lr=0.01)

epochs = 4000
for epoch in range(epochs+1):
    y_hat = x * W + b
    loss_fn = torch.mean((y_hat-y) ** 2)
    
    optimizer.zero_grad()
    loss_fn.backward()
    optimizer.step()
    
    if epoch % 500 == 0:
        print('Epoch {:4d}/{} W: {:.3f}, b: {:.3f}, Loss: {:.6f}'.format(epoch, epochs, W.item(), b.item(), loss_fn.item()))


Epoch    0/4000 W: 1.540, b: 0.220, Loss: 154.000000
Epoch  500/4000 W: 1.995, b: 0.034, Loss: 0.000252
Epoch 1000/4000 W: 1.999, b: 0.004, Loss: 0.000004
Epoch 1500/4000 W: 2.000, b: 0.001, Loss: 0.000000
Epoch 2000/4000 W: 2.000, b: 0.000, Loss: 0.000000
Epoch 2500/4000 W: 2.000, b: 0.000, Loss: 0.000000
Epoch 3000/4000 W: 2.000, b: 0.000, Loss: 0.000000
Epoch 3500/4000 W: 2.000, b: 0.000, Loss: 0.000000
Epoch 4000/4000 W: 2.000, b: 0.000, Loss: 0.000000

y_hat.ravel()

tensor([ 2.0000,  4.0000,  6.0000,  8.0000, 10.0000, 12.0000, 14.0000, 16.0000,
        18.0000, 20.0000], grad_fn=<ViewBackward>)

y.ravel()

tensor([ 2.,  4.,  6.,  8., 10., 12., 14., 16., 18., 20.])

Multivariable Linear regression

$ y = w_{1}x_{1} + w_{2}x_{2} + w_{3}x_{3} + b$

x1 = torch.FloatTensor([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]])
x2 = x1 / 2
x3 = x1 * 3
y = x1 + x2 + x3

w1 = torch.zeros(1, requires_grad = True)
w2 = torch.zeros(1, requires_grad = True)
w3 = torch.zeros(1, requires_grad = True)
b = torch.zeros(1, requires_grad = True)

optimizer = torch.optim.SGD([w1, w2, w3, b], lr=1e-5)

epochs = 1000

for epoch in range(epochs+1):
    y_hat = x1*w1 + x2*w2 + x3*w3 + b
    
    loss = torch.mean((y_hat - y) **2)
    
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
    if epoch % 100 == 0:
        print('Epoch {:4d}/{} [ w1: {:.6f}, w2: {:.6f}, w3: {:.6f}, b: {:.6f}, Loss: {:.6f}]'.format(
            epoch, epochs, w1.item(), w2.item(), w3.item(), b.item(), loss.item() ))

Epoch    0/1000 [ w1: 0.003465, w2: 0.001732, w3: 0.010395, b: 0.000495, Loss: 779.625000]
Epoch  100/1000 [ w1: 0.241652, w2: 0.120826, w3: 0.724957, b: 0.034513, Loss: 159.315308]
Epoch  200/1000 [ w1: 0.349325, w2: 0.174662, w3: 1.047974, b: 0.049877, Loss: 32.556362]
Epoch  300/1000 [ w1: 0.397998, w2: 0.198999, w3: 1.193994, b: 0.056807, Loss: 6.653516]
Epoch  400/1000 [ w1: 0.420001, w2: 0.210001, w3: 1.260003, b: 0.059925, Loss: 1.360293]
Epoch  500/1000 [ w1: 0.429948, w2: 0.214974, w3: 1.289842, b: 0.061320, Loss: 0.278642]
Epoch  600/1000 [ w1: 0.434444, w2: 0.217222, w3: 1.303332, b: 0.061936, Loss: 0.057607]
Epoch  700/1000 [ w1: 0.436477, w2: 0.218238, w3: 1.309430, b: 0.062200, Loss: 0.012437]
Epoch  800/1000 [ w1: 0.437396, w2: 0.218698, w3: 1.312188, b: 0.062304, Loss: 0.003205]
Epoch  900/1000 [ w1: 0.437812, w2: 0.218906, w3: 1.313435, b: 0.062337, Loss: 0.001318]
Epoch 1000/1000 [ w1: 0.438000, w2: 0.219000, w3: 1.314000, b: 0.062337, Loss: 0.000932]

y_hat.ravel()

tensor([ 4.5518,  9.0413, 13.5308, 18.0203, 22.5098, 26.9993, 31.4887, 35.9782,
        40.4677, 44.9572], grad_fn=<ViewBackward>)

y.ravel()

tensor([ 4.5000,  9.0000, 13.5000, 18.0000, 22.5000, 27.0000, 31.5000, 36.0000,
        40.5000, 45.0000])

Linear Regression with Module

import torch.nn as nn
import torch.nn.functional as F

x = torch.FloatTensor([[1],[2],[3],[4],[5],[6],[7],[8],[9],[10]])
y = x * 2

model = nn.Linear(1, 1)

print(list(model.parameters()))

[Parameter containing:
tensor([[0.6426]], requires_grad=True), Parameter containing:
tensor([0.6916], requires_grad=True)]

optimizer = torch.optim.SGD(model.parameters(), lr=0.01)

epochs = 1000
for epoch in range(epochs+1):
    y_hat = model(x)
    
    loss = F.mse_loss(y_hat, y)
    
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
    w, b = list(model.parameters())
    
    if epoch % 100 == 0:
        print('Epoch {:4d}/{} W: {:.3f}, b: {:.3f}, Loss: {:.6f}'.format(epoch, epochs, w.item(), b.item(), loss.item()))

Epoch    0/1000 W: 1.612, b: 0.827, Loss: 61.086681
Epoch  100/1000 W: 1.918, b: 0.568, Loss: 0.069697
Epoch  200/1000 W: 1.946, b: 0.373, Loss: 0.030038
Epoch  300/1000 W: 1.965, b: 0.245, Loss: 0.012946
Epoch  400/1000 W: 1.977, b: 0.161, Loss: 0.005580
Epoch  500/1000 W: 1.985, b: 0.105, Loss: 0.002405
Epoch  600/1000 W: 1.990, b: 0.069, Loss: 0.001036
Epoch  700/1000 W: 1.993, b: 0.045, Loss: 0.000447
Epoch  800/1000 W: 1.996, b: 0.030, Loss: 0.000193
Epoch  900/1000 W: 1.997, b: 0.020, Loss: 0.000083
Epoch 1000/1000 W: 1.998, b: 0.013, Loss: 0.000036

y_hat.ravel()

tensor([ 2.0111,  4.0092,  6.0074,  8.0055, 10.0036, 12.0018, 13.9999, 15.9981,
        17.9962, 19.9944], grad_fn=<ViewBackward>)

y.ravel()

tensor([ 2.,  4.,  6.,  8., 10., 12., 14., 16., 18., 20.])

new_data = torch.FloatTensor([[20]])

target = model(new_data)
target

tensor([[39.9759]], grad_fn=<AddmmBackward>)

print(list(model.parameters()))

[Parameter containing:
tensor([[1.9982]], requires_grad=True), Parameter containing:
tensor([0.0129], requires_grad=True)]

Multivariates Linear Regression with Module

x1 = torch.FloatTensor([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]])
x2 = x1 / 2
x3 = x1 * 3

x = torch.cat([x1, x2, x3], axis=1)
y = x1 + x2 + x3

model = nn.Linear(3, 1)

list(model.parameters())

[Parameter containing:
 tensor([[-0.0295, -0.4642,  0.2788]], requires_grad=True),
 Parameter containing:
 tensor([-0.2519], requires_grad=True)]

optimizer = torch.optim.SGD(model.parameters(), lr=1e-5)

epochs = 5000
for epoch in range(epochs+1):
    y_hat = model(x)
    
    loss = F.mse_loss(y_hat, y)
    
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
    w, b = list(model.parameters())
    w1, w2, w3 = w[0]
    
    if epoch % 500 == 0:
        print('Epoch {:4d}/{} [ w1: {:.6f}, w2: {:.6f}, w3: {:.6f}, b: {:.6f}, Loss: {:.6f}]'.format(epoch, epochs, w1.item(), w2.item(), w3.item(), b.item(), loss.item() ))

Epoch    0/5000 [ w1: -0.026445, w2: -0.462709, w3: 0.287969, b: -0.251455, Loss: 604.103333]
Epoch  500/5000 [ w1: 0.348962, w2: -0.275006, w3: 1.414190, b: -0.197376, Loss: 0.223519]
Epoch 1000/5000 [ w1: 0.356043, w2: -0.271466, w3: 1.435432, b: -0.195943, Loss: 0.008302]
Epoch 1500/5000 [ w1: 0.356171, w2: -0.271402, w3: 1.435814, b: -0.195506, Loss: 0.008191]
Epoch 2000/5000 [ w1: 0.356165, w2: -0.271402, w3: 1.435810, b: -0.195088, Loss: 0.008156]
Epoch 2500/5000 [ w1: 0.356151, w2: -0.271402, w3: 1.435795, b: -0.194671, Loss: 0.008121]
Epoch 3000/5000 [ w1: 0.356136, w2: -0.271402, w3: 1.435781, b: -0.194254, Loss: 0.008086]
Epoch 3500/5000 [ w1: 0.356121, w2: -0.271402, w3: 1.435766, b: -0.193837, Loss: 0.008051]
Epoch 4000/5000 [ w1: 0.356106, w2: -0.271402, w3: 1.435751, b: -0.193423, Loss: 0.008017]
Epoch 4500/5000 [ w1: 0.356091, w2: -0.271402, w3: 1.435736, b: -0.193013, Loss: 0.007983]
Epoch 5000/5000 [ w1: 0.356076, w2: -0.271402, w3: 1.435722, b: -0.192604, Loss: 0.007949]

y_hat.ravel()

tensor([ 4.3349,  8.8625, 13.3900, 17.9176, 22.4451, 26.9726, 31.5002, 36.0277,
        40.5553, 45.0828], grad_fn=<ViewBackward>)

y.ravel()

tensor([ 4.5000,  9.0000, 13.5000, 18.0000, 22.5000, 27.0000, 31.5000, 36.0000,
        40.5000, 45.0000])

print(list(model.parameters()))

[Parameter containing:
tensor([[ 0.3561, -0.2714,  1.4357]], requires_grad=True), Parameter containing:
tensor([-0.1926], requires_grad=True)]

Simple Linear Regression with class

x = torch.FloatTensor([[1],[2],[3],[4],[5],[6],[7],[8],[9],[10]])
y = x * 2

class simple_linear_regression(nn.Module):  # nn.Module을 상속받는다.
    def __init__(self):                     # 모델의 구조와 동작을 정의하는 생성자를 정의한다.
        super().__init__()
        self.linear = nn.Linear(1,1)        # input_dim = 1, output_dim = 1
        
    def forward(self, x):                   # 모델이 학습데이터를 입력 받아서 forward연산을 진행시키는 함수
        return self.linear(x)

model = simple_linear_regression()

model

simple_linear_regression(
  (linear): Linear(in_features=1, out_features=1, bias=True)
)

optimizer = torch.optim.SGD(model.parameters(), lr=0.01)

epochs = 1000
for epoch in range(epochs+1):
    y_hat = model(x)
    loss = F.mse_loss(y_hat, y)
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
    w, b = list(model.parameters())
    
    if epoch % 100 == 0:
        print('Epoch {:4d}/{} W: {:.3f}, b: {:.3f}, Loss: {:.6f}'.format(epoch, epochs, w.item(), b.item(), loss.item()))

Epoch    0/1000 W: 1.775, b: -0.265, Loss: 57.871025
Epoch  100/1000 W: 2.021, b: -0.150, Loss: 0.004836
Epoch  200/1000 W: 2.014, b: -0.098, Loss: 0.002084
Epoch  300/1000 W: 2.009, b: -0.064, Loss: 0.000898
Epoch  400/1000 W: 2.006, b: -0.042, Loss: 0.000387
Epoch  500/1000 W: 2.004, b: -0.028, Loss: 0.000167
Epoch  600/1000 W: 2.003, b: -0.018, Loss: 0.000072
Epoch  700/1000 W: 2.002, b: -0.012, Loss: 0.000031
Epoch  800/1000 W: 2.001, b: -0.008, Loss: 0.000013
Epoch  900/1000 W: 2.001, b: -0.005, Loss: 0.000006
Epoch 1000/1000 W: 2.000, b: -0.003, Loss: 0.000002

Multivariate Linear Regression with class

x1 = torch.FloatTensor([[1], [2], [3], [4], [5], [6], [7], [8], [9], [10]])
x2 = x1 / 2
x3 = x1 * 3

x = torch.cat([x1, x2, x3], axis=1)
y = x1 + x2 + x3

class multivariate_linear_regression(nn.Module):
    def __init__(self):
        super().__init__()
        self.linear = nn.Linear(3, 1)
    
    def forward(self, x):
        return self.linear(x)

model = multivariate_linear_regression()

model

multivariate_linear_regression(
  (linear): Linear(in_features=3, out_features=1, bias=True)
)

optimizer = torch.optim.SGD(model.parameters(), lr=1e-5)

epochs = 5000
for epoch in range(epochs+1):
    y_hat = model(x)
    
    loss = F.mse_loss(y_hat, y)
    
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()
    
    w, b = list(model.parameters())
    w1, w2, w3 = w[0]
    
    if epoch % 500 == 0:
        print('Epoch {:4d}/{} [ w1: {:.6f}, w2: {:.6f}, w3: {:.6f}, b: {:.6f}, Loss: {:.6f}]'.format(epoch, epochs, w1.item(), w2.item(), w3.item(), b.item(), loss.item() ))

Epoch    0/5000 [ w1: -0.035776, w2: -0.500726, w3: 0.147928, b: -0.241821, Loss: 749.372070]
Epoch  500/5000 [ w1: 0.382341, w2: -0.291667, w3: 1.402279, b: -0.181670, Loss: 0.274007]
Epoch 1000/5000 [ w1: 0.390228, w2: -0.287723, w3: 1.425941, b: -0.180157, Loss: 0.007048]
Epoch 1500/5000 [ w1: 0.390373, w2: -0.287652, w3: 1.426371, b: -0.179751, Loss: 0.006924]
Epoch 2000/5000 [ w1: 0.390368, w2: -0.287652, w3: 1.426369, b: -0.179366, Loss: 0.006894]
Epoch 2500/5000 [ w1: 0.390353, w2: -0.287652, w3: 1.426356, b: -0.178986, Loss: 0.006865]
Epoch 3000/5000 [ w1: 0.390338, w2: -0.287652, w3: 1.426343, b: -0.178606, Loss: 0.006836]
Epoch 3500/5000 [ w1: 0.390323, w2: -0.287652, w3: 1.426330, b: -0.178226, Loss: 0.006807]
Epoch 4000/5000 [ w1: 0.390309, w2: -0.287652, w3: 1.426316, b: -0.177846, Loss: 0.006778]
Epoch 4500/5000 [ w1: 0.390294, w2: -0.287652, w3: 1.426303, b: -0.177466, Loss: 0.006749]
Epoch 5000/5000 [ w1: 0.390279, w2: -0.287652, w3: 1.426290, b: -0.177086, Loss: 0.006720]