[딥러닝] 딥러닝 소프트 맥스(softmax) 활성화 함수

 

소프트 맥스 활성화 함수 (파이썬 shell에서 확인)

from math import exp as e
(e(1.3)),(e(5.1)),(e(2.2)),(e(0.7)),(e(1.1))
sumY=e(1.3)+e(5.1)+e(2.2)+e(0.7)+e(1.1)
sumY
(e(1.3)/sumY,e(5.1)/sumY,e(2.2)/sumY,e(0.7)/sumY,e(1.1)/sumY)

 

 

---

* 소수점 조절

from math import exp as e
print('%.6f'%e(1.3))
print('{:.6f}'.format(e(1.3)))
print(f'{e(1.3):.6f}')

 

 

---

소프트 맥스 함수의 분모 크기 줄이기  (파이썬 shell에서 확인)

from math import exp as e
(e(1.3-5.1),e(5.1-5.1),e(2.2-5.1),e(0.7-5.1),e(1.1-5.1))
sumY=e(1.3-5.1)+e(5.1-5.1)+e(2.2-5.1)+e(0.7-5.1)+e(1.1-5.1)
sumY
(e(1.3-5.1)/sumY, e(5.1-5.1)/sumY, e(2.2-5.1)/sumY, e(0.7-5.1)/sumY, e(1.1-5.1)/sumY)

 

 

 

---

cross entropy 오차 구현해 보기  (파이썬 shell에서 확인)

from math import log as ln
-(1*ln(0.90))

 

 

---

softmax 활성화 함수/cross entropy 오차 함수 적용하기

딥러닝 공식에 적용하기

from math import exp, log

x1,x2=0.05,0.10
w1,w2=0.15,0.20
w3,w4=0.25,0.30
b1,b2=0.35,0.35
w5,w6=0.40,0.45
w7,w8=0.50,0.55
b3,b4=0.60,0.60
y1T,y2T=0.,1. # 크로스 엔트로피는 하나의 목표값만 1, 나머지는 0
lr=0.01
EPOCH=10000

for epoch in range(EPOCH):
    h1=x1*w1+x2*w2+1*b1
    h2=x1*w3+x2*w4+1*b2
    
    # ReLU feed forward
    h1=h1 if h1>0 else 0
    h2=h2 if h2>0 else 0
    
    y1=h1*w5+h2*w6+1*b3
    y2=h1*w7+h2*w8+1*b4
    
    # softmax feed forward
    yMAX=y1 if y1>y2 else y2
    y1-=yMAX
    y2-=yMAX
    sumY=exp(y1)+exp(y2)
    y1=exp(y1)/sumY
    y2=exp(y2)/sumY
    
    E=-(y1T*log(y1)+y2T*log(y2)) # cross entropy error
    
    y1E=y1-y1T
    y2E=y2-y2T
    # softmax back propagation
    # do nothing
    
    w5E=y1E*h1
    w6E=y1E*h2
    w7E=y2E*h1
    w8E=y2E*h2
    b3E=y1E*1
    b4E=y2E*1
    
    h1E=y1E*w5+y2E*w7
    h2E=y1E*w6+y2E*w8
    
    # ReLU back propagation
    h1E=h1E if h1>0 else 0
    h2E=h2E if h2>0 else 0
    
    w1E=h1E*x1
    w2E=h1E*x2
    w3E=h2E*x1
    w4E=h2E*x2
    b1E=h1E*1
    b2E=h2E*1
    
    w5=w5-lr*w5E
    w6=w6-lr*w6E
    w7=w7-lr*w7E
    w8=w8-lr*w8E
    b3=b3-lr*b3E
    b4=b4-lr*b4E
    
    w1=w1-lr*w1E
    w2=w2-lr*w2E
    w3=w3-lr*w3E
    w4=w4-lr*w4E
    b1=b1-lr*b1E
    b2=b2-lr*b2E
    
    if epoch % 100 == 99:
        print(f'epoch = {epoch}')
        print(f'y1 = {y1:.6f}')
        print(f'y2 = {y2:.6f}')
        
    if E<0.0000001:
        break

print(f'w1,w3 = {w1:.6f},{w3:.6f}')
print(f'w2,w4 = {w2:.6f},{w4:.6f}')
print(f'b1,b2 = {b1:.6f},{b2:.6f}')
print(f'w5,w7 = {w5:.6f},{w7:.6f}')
print(f'w6,w8 = {w6:.6f},{w8:.6f}')
print(f'b3,b4 = {b3:.6f},{b4:.6f}')