3.27【机器学习】第五章作业代码实现

5.1

使用线性函数作为激活函数时，因为在单元层和隐藏层，其单元值仍是输入值X的线性组合，只能表示线性关系，无法表示非线性关系。神经网络的优势在于能够学习和表示复杂的非线性关系。如果使用线性激活函数，神经网络只能表示线性变换，限制了其表达能力
若输出层也用线性函数作为激活函数，达不到“激活”与“筛选”的目的.

5.2

对率回归，是使用Sigmoid函数作为联系函数时的广义线性模型。使用Sigmoid激活函数，每个神经元几乎和对率回归相同，只不过对率回归在 [sigmoid(x)>0.5] 时输出为1，而神经元直接输出 [sigmoid(x)] 。

5.3

见附件

5.4

学习率太低，每次下降得很慢，使得迭代次数增多，时间开销、系统开销等各种开销增大。
学习率太高，会在梯度下降最低点来回震荡，难以得到想要的结果。

5.5

#!/usr/bin/env python
# -*- coding: utf-8 -*-# STANDARD BP-NN & ACCUMULATED BP-NN
import numpy as npclass Data(object):def __init__(self, data):self.data = np.array(data)self.rows = len(self.data[:, 0])self.cols = len(self.data[0, :])  # it include the column of labelsself.__eta = 0.1  # initial eta=0.1self.__in = self.cols - 1  # number of input neuronsself.__out = len(np.unique(self.data[:, -1]))  # number of output neuronsdef set_eta(self, n):self.__eta = ndef get_eta(self):return self.__etadef get_in(self):return self.__indef get_out(self):return self.__out# 标准BP算法def BP_NN(self, q=10, err=0.1):X = self.data[:, :-1]# 为X矩阵左边插入列-1来计算vx-gama,在后面对b操作应该同样加一列，来计算wb-thetaX = np.insert(X, [0], -1, axis=1)Y = np.array([self.data[:, -1], 1 - self.data[:, -1]]).transpose()d, l = self.__in, self.__outv = np.mat(np.random.random((d + 1, q)))  # v_0 = gamaw = np.mat(np.random.random((q + 1, l)))  # w_0 = thetadef f(x):  # sigmoid functions = 1 / (1 + np.exp(-x))return sn = self.__etagap = 1counter = 0while gap > err:  # set E_k<=0.01 to quit the loopcounter += 1for i in range(self.rows):alpha = np.mat(X[i, :]) * v  # 1*q matrixb_init = f(alpha)  # 1*q matrix# 注意把中间变量b_init增加一个b_0，且b_0 = -1,此时成为bb = np.insert(b_init.T, [0], -1, axis=0)  # (q+1)*1 matrixbeta = b.T * w  # 1*l matrixy_cal = np.array(f(beta))  # 1*l arrayg = y_cal * (1 - y_cal) * (Y[i, :] - y_cal)  # 1*l arrayw_g = w[1:, :] * np.mat(g).T  # q*1 matrixe = np.array(b_init) * (1 - np.array(b_init)) * np.array(w_g.T)  # 1*q arrayd_w = n * b * np.mat(g)d_v = n * np.mat(X[i, :]).T * np.mat(e)w += d_wv += d_vgap = 0.5 * np.sum((Y[i, :] - y_cal) ** 2)print('BP_round:', counter)return v, w# l累积BP算法def ABP_NN(self, q=10, err=0.1):X = self.data[:, :-1]# 为X矩阵左边插入列-1来计算vx-gama,在后面对b操作应该同样加一列，来计算wb-thetaX = np.insert(X, [0], -1, axis=1)Y = np.array([self.data[:, -1], 1 - self.data[:, -1]]).transpose()d, l = self.__in, self.__outv = np.mat(np.random.random((d + 1, q)))  # v_0 = gamaw = np.mat(np.random.random((q + 1, l)))  # w_0 = thetadef f(x):  # sigmoid functions = 1 / (1 + np.exp(-x))return sn = self.__etagap = 1counter = 0while gap > err:  # set E_k<=1 to quit the loopd_v, d_w, gap = 0, 0, 0counter += 1for i in range(self.rows):alpha = np.mat(X[i, :]) * v  # 1*q matrixb_init = f(alpha)  # 1*q matrix# 注意把中间变量b_init增加一个b_0，且b_0 = -1,此时成为bb = np.insert(b_init.T, [0], -1, axis=0)  # (q+1)*1 matrixbeta = b.T * w  # 1*l matrixy_cal = np.array(f(beta))  # 1*l arrayg = y_cal * (1 - y_cal) * (Y[i, :] - y_cal)  # 1*l arrayw_g = w[1:, :] * np.mat(g).T  # q*1 matrixe = np.array(b_init) * (1 - np.array(b_init)) * np.array(w_g.T)  # 1*q arrayd_w += n * b * np.mat(g)d_v += n * np.mat(X[i, :]).T * np.mat(e)gap += 0.5 * np.sum((Y[i, :] - y_cal) ** 2)w += d_w / self.rowsv += d_v / self.rowsgap = gap / self.rowsprint('ABP_round:', counter)return v, wdef test_NN(a, v, w):X = a.data[:, :-1]X = np.insert(X, [0], -1, axis=1)Y = np.array([a.data[:, -1], 1 - a.data[:, -1]]).transpose()y_cal = np.zeros((a.rows, 2))def f(x):  # sigmoid functions = 1 / (1 + np.exp(-x))return sfor i in range(a.rows):alpha = np.mat(X[i, :]) * v  # 1*q matrixb_init = f(alpha)  # 1*q matrixb = np.insert(b_init.T, [0], -1, axis=0)  # (q+1)*1 matrixbeta = b.T * w  # 1*l matrixy_cal[i, :] = np.array(f(beta))  # 1*l arrayprint(y_cal)# 对原始数据进行一位有效编码
D = np.array([[1, 1, 1, 1, 1, 1, 0.697, 0.460, 1],[2, 1, 2, 1, 1, 1, 0.774, 0.376, 1],[2, 1, 1, 1, 1, 1, 0.634, 0.264, 1],[1, 1, 2, 1, 1, 1, 0.608, 0.318, 1],[3, 1, 1, 1, 1, 1, 0.556, 0.215, 1],[1, 2, 1, 1, 2, 2, 0.403, 0.237, 1],[2, 2, 1, 2, 2, 2, 0.481, 0.149, 1],[2, 2, 1, 1, 2, 1, 0.437, 0.211, 1],[2, 2, 2, 2, 2, 1, 0.666, 0.091, 0],[1, 3, 3, 1, 3, 2, 0.243, 0.267, 0],[3, 3, 3, 3, 3, 1, 0.245, 0.057, 0],[3, 1, 1, 3, 3, 2, 0.343, 0.099, 0],[1, 2, 1, 2, 1, 1, 0.639, 0.161, 0],[3, 2, 2, 2, 1, 1, 0.657, 0.198, 0],[2, 2, 1, 1, 2, 2, 0.360, 0.370, 0],[3, 1, 1, 3, 3, 1, 0.593, 0.042, 0],[1, 1, 2, 2, 2, 1, 0.719, 0.103, 0]])
a = Data(D)  # 加载数据
v, w = a.ABP_NN(err=0.01)  # 累积BP
v1, w1 = a.BP_NN(err=0.2)  # 标准BP
# v, w = a.ABP_NN(err=0.2)#累积BP
# v1, w1 = a.BP_NN(err=0.01)#标准BP
print("开始计算累计BP")
test_NN(a, v, w)
print("开始计算标准BP")
test_NN(a, v1, w1)

5.7

import numpy as npdef RBF(x, center_i, beta_i):dist = np.sum(pow((x - center_i), 2))rho = np.exp(-beta_i * dist)return rho# 输出为1
class RBF_network:def __init__(self):self.hidden_num = 0self.y = 0def createNN(self, input_num, hidden_num, learning_rate, center):self.input_num = input_numself.hidden_num = hidden_numself.center = centerself.w = np.random.random(self.hidden_num)self.rho = np.zeros(self.hidden_num)self.beta = np.random.random(self.hidden_num)self.lr = learning_ratedef Predict(self, x):self.y = 0for i in range(self.hidden_num):self.rho[i] = RBF(x, self.center[i], self.beta[i])self.y += self.w[i] * self.rho[i]return self.ydef BackPropagate(self, x, y):self.Predict(x)grad = np.zeros(self.hidden_num)for i in range(self.hidden_num):# dE_k/dy_cap = (y_cap-y)# dE_k/dw = (y_cap-y)*rho[i]# dE_k/d_beta = -(y_cap-y)*rho[i]w_i*||x-c_i||grad[i] = (self.y - y) * self.rho[i]self.w[i] -= self.lr * grad[i]self.beta[i] += self.lr * grad[i] * self.w[i] * np.sum(pow((x - center[i]), 2))def trainNN(self, x, y):error_list = []for i in range(len(x)):self.BackPropagate(x[i], y[i])error = (self.y - y[i]) ** 2error_list.append(error / 2)print(error_list)train_x = np.random.randint(0,2,(100,2))
train_y = np.logical_xor(train_x[:,0],train_x[:,1])
test_x = np.random.randint(0,2,(100,2))
test_y = np.logical_xor(test_x[:,0],test_x[:,1])
center = np.array([[0,0],[0,1],[1,0],[1,1]])
rbf = RBF_network()
rbf.createNN(input_num = 2, hidden_num=4 , learning_rate=0.1, center=center)
rbf.trainNN(train_x, train_y)

激活函数是接收输入后产生的信号，不能用线性，不然依旧还是线性组合