1 正则化
1.1 L2正则化函数
def compute_cost_with_regularization(A3, Y, parameters, lambd):
m = Y.shape[1]
W1 = parameters["W1"]
W2 = parameters["W2"]
W3 = parameters["W3"]
cross_entropy_cost = compute_cost(A3, Y)
L2_regularization_cost=lambd/(2*m)*(np.sum(np.square(W1))+np.sum(np.square(W2))+np.sum(np.square(W3)))
cost = cross_entropy_cost + L2_regularization_cost
return cost
1.2 梯度下降
def backward_propagation_with_regularization(X, Y, cache, lambd):
m = X.shape[1]
(Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache
dZ3 = A3 - Y
dW3 = 1./m * np.dot(dZ3, A2.T) + lambd/m*W3
db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)
dA2 = np.dot(W3.T, dZ3)
dZ2 = np.multiply(dA2, np.int64(A2 > 0))
dW2 = 1./m * np.dot(dZ2, A1.T) + lambd/m*W2
db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)
dA1 = np.dot(W2.T, dZ2)
dZ1 = np.multiply(dA1, np.int64(A1 > 0))
dW1 = 1./m * np.dot(dZ1, X.T) + lambd/m*W1
db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)
gradients = {"dZ3": dZ3, "dW3": dW3, "db3": db3,"dA2": dA2,
"dZ2": dZ2, "dW2": dW2, "db2": db2, "dA1": dA1,
"dZ1": dZ1, "dW1": dW1, "db1": db1}
return gradients
2 dropout
2.1 方程式
d3 = np.random.rand( a3.shape[0] , a3.shape[1] ) < keep_prop （生成一个随机矩阵，当d3的元素值小于keep_prop时，为1，否则为0的 0-1矩阵） a3 = np.multiply( a3 , d3 ) 也就是点乘， a3
∗
*
∗
=
=
= d3 a3 = a3 / keep_prop
2.2 函数（forword + backword propagation）
2.2.1 forword propagation with dropout
def forward_propagation_with_dropout(X, parameters, keep_prob = 0.5):
np.random.seed(1)
# retrieve parameters
W1 = parameters["W1"]
b1 = parameters["b1"]
W2 = parameters["W2"]
b2 = parameters["b2"]
W3 = parameters["W3"]
b3 = parameters["b3"]
# LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
Z1 = np.dot(W1, X) + b1
A1 = relu(Z1)
### Step 1: initialize matrix D1 = np.random.rand(..., ...)
D1 = np.random.rand(A1.shape[0],A1.shape[1])
### Step 2: convert entries of D1 to 0 or 1 (using keep_prob as the threshold)
D1 = D1<keep_prob
### Step 3: shut down some neurons of A1
A1 = np.multiply(A1,D1)
### Step 4: scale the value of neurons that haven't been shut down
A1 = A1/keep_prob
Z2 = np.dot(W2, A1) + b2
A2 = relu(Z2)
### Step 1: initialize matrix D2 = np.random.rand(..., ...)
D2 = np.random.rand(A2.shape[0],A2.shape[1])
### Step 2: convert entries of D2 to 0 or 1 (using keep_prob as the threshold)
D2 = D2<keep_prob
### Step 3: shut down some neurons of A2
A2 = np.multiply(A2,D2)
### Step 4: scale the value of neurons that haven't been shut down
A2 = A2/keep_prob
Z3 = np.dot(W3, A2) + b3
A3 = sigmoid(Z3)
cache = (Z1, D1, A1, W1, b1, Z2, D2, A2, W2, b2, Z3, A3, W3, b3)
return A3, cache
2.2.2 backword propagation with dropout
def backward_propagation_with_dropout(X, Y, cache, keep_prob):
m = X.shape[1]
(Z1, D1, A1, W1, b1, Z2, D2, A2, W2, b2, Z3, A3, W3, b3) = cache
dZ3 = A3 - Y
dW3 = 1./m * np.dot(dZ3, A2.T)
db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)
dA2 = np.dot(W3.T, dZ3)
### Step 1: Apply mask D2 to shut down the same neurons as during the forward propagation
dA2 = dA2*D2
### Step 2: Scale the value of neurons that haven't been shut down
dA2 = dA2/keep_prob
dZ2 = np.multiply(dA2, np.int64(A2 > 0))
dW2 = 1./m * np.dot(dZ2, A1.T)
db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)
dA1 = np.dot(W2.T, dZ2)
### Step 1: Apply mask D1 to shut down the same neurons as during the forward propagation
dA1 = dA1*D1
### Step 2: Scale the value of neurons that haven't been shut down
dA1 = dA1/keep_prob
dZ1 = np.multiply(dA1, np.int64(A1 > 0))
dW1 = 1./m * np.dot(dZ1, X.T)
db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)
gradients = {"dZ3": dZ3, "dW3": dW3, "db3": db3,"dA2": dA2,
"dZ2": dZ2, "dW2": dW2, "db2": db2, "dA1": dA1,
"dZ1": dZ1, "dW1": dW1, "db1": db1}
return gradients
3 Gradient Checking 梯度检验
3.1 概念
计算
ε
\varepsilon
ε. 和近似梯度
g
r
a
d
a
p
p
r
o
x
gradapprox
gradapprox
θ
+
=
θ
+
ε
\theta^{+} = \theta + \varepsilon
θ+=θ+ε
θ
−
=
θ
−
ε
\theta^{-} = \theta - \varepsilon
θ−=θ−ε
J
+
=
J
(
θ
+
)
J^{+} = J(\theta^{+})
J+=J(θ+)
J
−
=
J
(
θ
−
)
J^{-} = J(\theta^{-})
J−=J(θ−)
g
r
a
d
a
p
p
r
o
x
=
J
+
−
J
−
2
ε
gradapprox = \frac{J^{+} - J^{-}}{2 \varepsilon}
gradapprox=2εJ+−J− 通过反向传播计算梯度
g
r
a
d
grad
grad计算差异：
(2)
d
i
f
f
e
r
e
n
c
e
=
∣
∣
g
r
a
d
−
g
r
a
d
a
p
p
r
o
x
∣
∣
2
∣
∣
g
r
a
d
∣
∣
2
+
∣
∣
g
r
a
d
a
p
p
r
o
x
∣
∣
2
difference = \frac {\mid\mid grad - gradapprox \mid\mid_2}{\mid\mid grad \mid\mid_2 + \mid\mid gradapprox \mid\mid_2} \tag{2}
difference=∣∣grad∣∣2+∣∣gradapprox∣∣2∣∣grad−gradapprox∣∣2(2) 通过如下三步进行计算:
1’. 计算分子 numerator 使用： np.linalg.norm(…)2’. 计算坟墓 denominator. 使用： np.linalg.norm(…) 两次.3’. 相除. 如果差异很小(小于
1
0
−
7
10^{-7}
10−7), 你可以说你计算的梯度非常正确。否则，在梯度计算中存在一些问题。
3.2 N维神经网络的梯度检查
def forward_propagation_n(X, Y, parameters):
# retrieve parameters
m = X.shape[1]
W1 = parameters["W1"]
b1 = parameters["b1"]
W2 = parameters["W2"]
b2 = parameters["b2"]
W3 = parameters["W3"]
b3 = parameters["b3"]
# LINEAR -> RELU -> LINEAR -> RELU -> LINEAR -> SIGMOID
Z1 = np.dot(W1, X) + b1
A1 = relu(Z1)
Z2 = np.dot(W2, A1) + b2
A2 = relu(Z2)
Z3 = np.dot(W3, A2) + b3
A3 = sigmoid(Z3)
# Cost
logprobs = np.multiply(-np.log(A3),Y) + np.multiply(-np.log(1 - A3), 1 - Y)
cost = 1./m * np.sum(logprobs)
cache = (Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3)
return cost, cache
def backward_propagation_n(X, Y, cache):
m = X.shape[1]
(Z1, A1, W1, b1, Z2, A2, W2, b2, Z3, A3, W3, b3) = cache
dZ3 = A3 - Y
dW3 = 1./m * np.dot(dZ3, A2.T)
db3 = 1./m * np.sum(dZ3, axis=1, keepdims = True)
dA2 = np.dot(W3.T, dZ3)
dZ2 = np.multiply(dA2, np.int64(A2 > 0))
dW2 = 1./m * np.dot(dZ2, A1.T)
db2 = 1./m * np.sum(dZ2, axis=1, keepdims = True)
dA1 = np.dot(W2.T, dZ2)
dZ1 = np.multiply(dA1, np.int64(A1 > 0))
dW1 = 1./m * np.dot(dZ1, X.T)
db1 = 1./m * np.sum(dZ1, axis=1, keepdims = True)
gradients = {"dZ3": dZ3, "dW3": dW3, "db3": db3,
"dA2": dA2, "dZ2": dZ2, "dW2": dW2, "db2": db2,
"dA1": dA1, "dZ1": dZ1, "dW1": dW1, "db1": db1}
return gradients
