前馈全连接层
什么是前馈全连接层:
在Transformer中前馈全连接层就是具有两层线性层的全连接网络
前馈全连接层的作用:
考虑注意力机制可能对复杂过程的拟合程度不够,通过增加两层网络来增强模型的能力
code
# 前馈全连接层
class PositionwiseFeedForward(nn.Module):def __init__(self, d_model, d_ff,dropout=0.1) -> None:""" d_mode :第一个线下层的输入维度d_ff :隐藏层的维度dropout："""super(PositionwiseFeedForward,self).__init__()self.line1 = nn.Linear(d_model,d_ff)self.line2 = nn.Linear(d_ff,d_model)self.dropout = nn.Dropout(dropout)def forward(self,x):return self.line2(self.dropout(F.relu(self.line1(x))))测试：
输出
ff_result.shape = torch.Size([2, 4, 512])
ff_result = tensor([[[-0.0589, -1.3885, -0.8852, ..., -0.4463, -0.9892, 2.7384],[ 0.2426, -1.1040, -1.1298, ..., -0.9296, -1.5262, 1.0632],[ 0.0318, -0.8362, -0.9389, ..., -1.6359, -1.8531, -0.1163],[ 1.1119, -1.2007, -1.5487, ..., -0.8869, 0.1711, 1.7431]],[[-0.2358, -0.9319, 0.8866, ..., -1.2987, 0.2001, 1.5415],[-0.1448, -0.7505, -0.3023, ..., -0.2585, -0.8902, 0.6206],[ 1.8106, -0.8460, 1.6487, ..., -1.1931, 0.0535, 0.8415],[ 0.2669, -0.3897, 1.1560, ..., 0.1138, -0.2795, 1.8780]]],grad_fn=<ViewBackward0>)规范化层
规范化层的作用:
它是所有深层网络模型都需要的标准网络层，因为随着网络层数的增加，通过多层的计算后参数可能开始出现过大或过小的情况，这样可能会导致学习过程出现异常，模型可能收敛非常的慢,因此都会在一定层数后接规范化层进行数值的规范化，使其特征数值在合理范围内.
code
class LayerNorm(nn.Module):def __init__(self,features,eps=1e-6) -> None:# features 词嵌入的维度# eps 足够小的数据，防止除0，放到分母上super(LayerNorm,self).__init__()# 规范化层的参数，后续训练使用的self.w = nn.parameter(torch.ones(features))self.b = nn.Parameter(torch.zeros(features))self.eps = epsdef forward(self, x):mean = x.mean(-1,keepdim=True)stddev = x.std(-1,keepdim=True)# * 代表对应位置进行相乘，不是点积return self.w*(x-mean)/(stddev,self.eps) + self.btest
输出：
ln_result.shape = torch.Size([2, 4, 512])
ln_result = tensor([[[ 1.3255e+00, 7.7968e-02, -1.7036e+00, ..., -1.3097e-01,4.9385e-01, 1.3975e-03],[-1.0717e-01, -1.8999e-01, -1.0603e+00, ..., 2.9285e-01,1.0337e+00, 1.0597e+00],[ 1.0801e+00, -1.5308e+00, -1.6577e+00, ..., -1.0050e-01,-3.7577e-02, 4.1453e-01],[ 4.2174e-01, -1.1476e-01, -5.9897e-01, ..., -8.2557e-01,1.2285e+00, 2.2961e-01]],[[-1.3024e-01, -6.9125e-01, -8.4373e-01, ..., -4.7106e-01,2.3697e-01, 2.4667e+00],[-1.8319e-01, -5.0278e-01, -6.6853e-01, ..., -3.3992e-02,-4.8510e-02, 2.3002e+00],[-5.7036e-01, -1.4439e+00, -2.9533e-01, ..., -4.9297e-01,9.9002e-01, 9.1294e-01],[ 2.8479e-02, -1.2107e+00, -4.9597e-01, ..., -6.0751e-01,3.1257e-01, 1.7796e+00]]], grad_fn=<AddBackward0>)子层连接结构
什么是子层连接结构:
如图所示，输入到每个子层以及规范化层的过程中，还使用了残差链接(跳跃连接)，因此我们把这一部分结构整体叫做子层连接(代表子层及其链接结构)，在每个编码器层中，都有两个子层，这两个子层加上周围的链接结构就形成了两个子层连接结构.
code
# 子层连接结构
class SublayerConnection(nn.Module):def __init__(self,size,dropout=0.1) -> None:"""size : 词嵌入的维度"""super(SublayerConnection,self).__init__()self.norm = LayerNorm(size)self.dropout = nn.Dropout(dropout)def forward(self, x, sublayer):# sublayer 子层结构或者函数# x+ 跳跃层return x+self.dropout(sublayer(self.norm(x)))测试代码 放到最后
输出：
sc_result.sahpe = torch.Size([2, 4, 512])
sc_result = tensor([[[-1.8123e+00, -2.2030e+01, 3.1459e+00, ..., -1.3725e+01,-6.1578e+00, 2.2611e+01],[ 4.1956e+00, 1.9670e+01, 0.0000e+00, ..., 2.3616e+01,3.8118e+00, 6.4224e+01],[-7.8954e+00, 8.5818e+00, -7.8634e+00, ..., 1.5810e+01,3.5864e-01, 1.8220e+01],[-2.5320e+01, -2.8745e+01, -3.6269e+01, ..., -1.8110e+01,-1.7574e+01, 2.9502e+01]],[[-9.3402e+00, 1.0549e+01, -9.0477e+00, ..., 1.5789e+01,2.6289e-01, 1.8317e+01],[-4.0251e+01, 1.5518e+01, 1.9928e+01, ..., -1.4024e+01,-3.4640e-02, 1.8811e-01],[-2.6166e+01, 2.1279e+01, -1.1375e+01, ..., -1.9781e+00,-6.4913e+00, -3.8984e+01],[ 2.1043e+01, -3.5800e+01, 6.4603e+01, ..., 2.2372e+01,3.0018e+01, -3.0919e+01]]], grad_fn=<AddBackward0>)编码器层
编码器层的作用:
作为编码器的组成单元,每个编码器层完成一次对输入的特征提取过程,即编码过程，如下图
code
# 编码器层
class EncoderLayer(nn.Module):def __init__(self,size,self_attn, feed_forward,dropout) -> None:"""size : 词嵌入维度self_attn: 多头自注意力子层实例化对象fee_forward: 前馈全连接层网络"""super(EncoderLayer,self).__init__()self.self_attn = self_attnself.feed_forward = feed_forward# 编码器中有两个子层结构，所以使用clone实现self.sublayer = clones(SublayerConnection(size, dropout),2)self.size = sizedef forward(self,x,mask):x = self.sublayer[0](x, lambda x: self.self_attn(x,x,x,mask))return self.sublayer[1](x,self.feed_forward)测试代码放到最后
输出
el_result.shape = torch.Size([2, 4, 512])
el_result = tensor([[[-1.4834e+01, -4.3176e+00, 1.2379e+01, ..., -1.0715e+01,-6.8350e-01, -5.2663e+00],[-1.7895e+01, -5.9179e+01, 1.0283e+01, ..., 9.7986e+00,2.2730e+01, 1.6832e+01],[ 5.0309e+00, -6.9362e+00, -2.6385e-01, ..., -1.2178e+01,3.1495e+01, -1.9781e-02],[ 1.6883e+00, 3.9012e+01, 3.2095e-01, ..., -6.1469e-01,3.8988e+01, 2.2591e+01]],[[ 4.8033e+00, -5.5316e+00, 1.4400e+01, ..., -1.1599e+01,3.1904e+01, -1.4026e+01],[ 8.6239e+00, 1.3545e+01, 3.9492e+01, ..., -8.3500e+00,2.6721e+01, 4.4794e+00],[-2.0212e+01, 1.6034e+01, -1.9680e+01, ..., -4.7649e+00,-1.1372e+01, -3.3566e+01],[ 1.0816e+01, -1.7987e+01, 2.0039e+01, ..., -4.7768e+00,-1.9426e+01, 2.7683e+01]]], grad_fn=<AddBackward0>)编码器
编码器的作用:
编码器用于对输入进行指定的特征提取过程,也称为编码,由N个编码器层堆叠而成
code
# 编码器
class Encoder(nn.Module):def __init__(self, layer,N) -> None:super(Encoder,self).__init__()self.layers = clones(layer,N)self.norm = LayerNorm(layer.size)def forward(self, x, mask):"""forward函数的输入和编码器层相同，x代表上一层的输出，mask代表掩码张量"""# 首先就是对我们克隆的编码器层进行循环，每次都会得到一个新的x，# 这个循环的过程，就相当于输出的x经过了N个编码器层的处理，# 最后再通过规范化层的对象self.norm进行处理，最后返回结果for layre in self.layers:x = layre(x,mask)return self.norm(x)测试代码在下面：
输出：
en_result.shape : torch.Size([2, 4, 512])
en_result : tensor([[[-2.7477e-01, -1.1117e+00, 6.1682e-02, ..., 6.7421e-01,-6.2473e-02, -4.6477e-02],[-7.7232e-01, -7.6969e-01, -2.0160e-01, ..., 2.6131e+00,-1.9882e+00, 1.3715e+00],[-1.4178e+00, 2.6184e-01, 1.1888e-01, ..., -9.9172e-01,1.3337e-01, 1.3132e+00],[-1.3268e+00, -1.1559e+00, -1.1774e+00, ..., -8.1548e-01,-2.8089e-02, 1.4730e-03]],[[-1.3472e+00, 4.4969e-01, -4.3498e-02, ..., -9.8910e-01,7.4551e-02, 1.1824e+00],[-2.2395e-02, 3.1730e-01, 6.8652e-02, ..., 4.3939e-01,2.8600e+00, 3.2169e-01],[-7.2252e-01, -7.6787e-01, -7.5412e-01, ..., 6.3915e-02,1.2210e+00, -2.3871e+00],[ 1.6294e-02, -4.8995e-02, -2.2887e-02, ..., -7.7798e-01,4.4148e+00, 1.7802e-01]]], grad_fn=<AddBackward0>)测试代码
from inputs import Embeddings,PositionalEncoding
import numpy as np
import torch
import torch.nn.functional as F
import torch.nn as nn
import matplotlib.pyplot as plt
import math
import copy def subsequent_mask(size):"""生成向后遮掩的掩码张量，参数size是掩码张量最后两个维度的大小，他们最好会形成一个方阵"""attn_shape = (1,size,size)# 使用np.ones方法向这个形状中添加1元素，形成上三角阵subsequent_mask = np.triu(np.ones(attn_shape),k=1).astype('uint8')# 最后将numpy类型转化为torch中的tensor，内部做一个1-的操作，# 在这个其实是做了一个三角阵的反转，subsequent_mask中的每个元素都会被1减# 如果是0，subsequent_mask中的该位置由0变成1# 如果是1，subsequent_mask中的该位置由1变成0return torch.from_numpy(1-subsequent_mask)def attention(query, key, value, mask=None, dropout=None):""" 注意力机制的实现，输入分别是query、key、value，mask此时输入的query、key、value的形状应该是 batch * number_token * embeding"""# 获取词嵌入的维度d_k = query.size(-1)# 根据注意力公示，将query和key的转置相乘，然后乘上缩放系数得到评分，这里为什么需要转置？# batch * number_token * embeding X batch * embeding * number_token = batch * number_token * number_token# 结果是个方阵scores = torch.matmul(query,key.transpose(-2,-1))/math.sqrt(d_k)# 判断是否使用maskif mask is not None:scores = scores.masked_fill(mask==0,-1e9)# scores 的最后一维进行softmax操作，为什么是最后一维？# 因为scores是方阵，每行的所有列代表当前token和全体token的相似度值，# 因此需要在列维度上进行softmaxp_attn = F.softmax(scores,dim=-1) # 这个就是最终的注意力张量# 之后判断是否使用dropout 进行随机置0if dropout is not None:p_attn = dropout(p_attn)# 最后，根据公式将p_attn与value张量相乘获得最终的query注意力表示，同时返回注意力张量# 计算后的 维度是多少呢？# batch * number_token * number_token X batch * number_token * embeding = # batch * number_token * embedingreturn torch.matmul(p_attn,value), p_attndef clones(module, N):"""用于生成相同网络层的克隆函数，N代表克隆的数量"""return nn.ModuleList([copy.deepcopy(module) for _ in range(N)])class MultiHeadedAttention(nn.Module):def __init__(self, head,embedding_dim,dropout=0.1) -> None:""" head 代表头的数量， embedding_dim 词嵌入的维度 """super(MultiHeadedAttention,self).__init__()# 因为多头是指对embedding的向量进行切分，因此头的数量需要整除embeddingassert embedding_dim % head == 0# 计算每个头获得分割词向量维度d_kself.d_k = embedding_dim // head# 传入头数self.head = head# 获得线形层对象，因为线性层是不分割词向量的，同时需要保证线性层输出和词向量维度相同# 因此线形层权重是方阵self.linears = clones(nn.Linear(embedding_dim,embedding_dim),4)# 注意力张量self.attn = None# dropoutself.dropout = nn.Dropout(p=dropout)def forward(self,query,key,value,mask=None):if mask is not None:# 拓展维度，因为有多头了mask = mask.unsqueeze(0)batch_size = query.size(0)# 输入先经过线形层,首先使用zip将网络层和输入数据连接一起，# 模型的输出利用view和transpose进行维度和形状的变换# (query,key,value) 分别对应一个线形层，经过线形层输出后，立刻对其进行切分，注意这里切分是对query经过线形层输出后进行切分，key经过线性层进行切分，value进行线性层进行切分，在这里才是多头的由来query,key,value = \[model(x).view(batch_size, -1, self.head,self.d_k).transpose(1,2) for model,x in zip(self.linears,(query,key,value))]# 将每个头的输出传入注意力层x,self.attn = attention(query,key,value,mask,self.dropout)# 得到每个头的计算结果是4维张量，需要进行形状的转换x = x.transpose(1,2).contiguous().view(batch_size,-1,self.head*self.d_k)return self.linears[-1](x)# 前馈全连接层
class PositionwiseFeedForward(nn.Module):def __init__(self, d_model, d_ff,dropout=0.1) -> None:""" d_mode :第一个线下层的输入维度d_ff :隐藏层的维度dropout ："""super(PositionwiseFeedForward,self).__init__()self.line1 = nn.Linear(d_model,d_ff)self.line2 = nn.Linear(d_ff,d_model)self.dropout = nn.Dropout(dropout)def forward(self,x):return self.line2(self.dropout(F.relu(self.line1(x))))# 规范化层
class LayerNorm(nn.Module):def __init__(self,features,eps=1e-6) -> None:# features 词嵌入的维度# eps 足够小的数据，防止除0，放到分母上super(LayerNorm,self).__init__()# 规范化层的参数，后续训练使用的self.w = nn.Parameter(torch.ones(features))self.b = nn.Parameter(torch.zeros(features))self.eps = epsdef forward(self, x):mean = x.mean(-1,keepdim=True)stddev = x.std(-1,keepdim=True)# * 代表对应位置进行相乘，不是点积return self.w*(x-mean)/(stddev+self.eps) + self.b# 子层连接结构
class SublayerConnection(nn.Module):def __init__(self,size,dropout=0.1) -> None:"""size : 词嵌入的维度"""super(SublayerConnection,self).__init__()self.norm = LayerNorm(size)self.dropout = nn.Dropout(dropout)def forward(self, x, sublayer):# sublayer 子层结构或者函数# x+ 跳跃层return x+self.dropout(sublayer(self.norm(x)))# 编码器层
class EncoderLayer(nn.Module):def __init__(self,size,self_attn, feed_forward,dropout) -> None:"""size : 词嵌入维度self_attn: 多头自注意力子层实例化对象fee_forward: 前馈全连接层网络"""super(EncoderLayer,self).__init__()self.self_attn = self_attnself.feed_forward = feed_forward# 编码器中有两个子层结构，所以使用clone实现self.sublayer = clones(SublayerConnection(size, dropout),2)self.size = sizedef forward(self,x,mask):x = self.sublayer[0](x, lambda x: self.self_attn(x,x,x,mask))return self.sublayer[1](x,self.feed_forward)# 编码器
class Encoder(nn.Module):def __init__(self, layer,N) -> None:super(Encoder,self).__init__()self.layers = clones(layer,N)self.norm = LayerNorm(layer.size)def forward(self, x, mask):"""forward函数的输入和编码器层相同，x代表上一层的输出，mask代表掩码张量"""# 首先就是对我们克隆的编码器层进行循环，每次都会得到一个新的x，# 这个循环的过程，就相当于输出的x经过了N个编码器层的处理，# 最后再通过规范化层的对象self.norm进行处理，最后返回结果for layre in self.layers:x = layre(x,mask)return self.norm(x)if __name__ == "__main__":# size = 5# sm = subsequent_mask(size)# print("sm: \n",sm.data.numpy())# plt.figure(figsize=(5,5))# plt.imshow(subsequent_mask(20)[0])# plt.waitforbuttonpress()# 词嵌入dim = 512vocab =1000emb = Embeddings(dim,vocab)x = torch.LongTensor([[100,2,321,508],[321,234,456,324]])embr =emb(x)print("embr.shape = ",embr.shape)# 位置编码pe = PositionalEncoding(dim,0.1) # 位置向量的维度是20，dropout是0pe_result = pe(embr)print("pe_result.shape = ",pe_result.shape)# 获取注意力值# query = key = value = pe_result # attn,p_attn = attention(query,key,value)# print("attn.shape = ",attn.shape)# print("p_attn.shape = ",p_attn.shape)# print("attn: ",attn)# print("p_attn: ",p_attn)# # 带mask# mask = torch.zeros(2,4,4)# attn,p_attn = attention(query,key,value,mask)# print("mask attn.shape = ",attn.shape)# print("mask p_attn.shape = ",p_attn.shape)# print("mask attn: ",attn)# print("mask p_attn: ",p_attn)# 多头注意力测试head = 8embedding_dim = 512dropout = 0.2query = key = value = pe_result# mask 是给计算出来的点积矩阵使用的，这个矩阵是方阵，tokenmask = torch.zeros(8,4,4) mha = MultiHeadedAttention(head,embedding_dim,dropout)mha_result = mha(query,key,value,mask)print("mha_result.shape = ",mha_result.shape)print("mha_result: ",mha_result)# 前馈全连接层 测试x = mha_resultd_model =512d_ff=128dropout = 0.2ff = PositionwiseFeedForward(d_model, d_ff, dropout)ff_result = ff(x)print("ff_result.shape = ",ff_result.shape)print("ff_result = ",ff_result)# 规范化层测试features = d_model = 512esp = 1e-6ln = LayerNorm(features,esp)ln_result = ln(ff_result)print("ln_result.shape = ",ln_result.shape)print("ln_result = ",ln_result)# 子层连接结构 测试size = 512dropout=0.2head=8d_model=512x = pe_resultmask = torch.zeros(8,4,4)self_attn = MultiHeadedAttention(head,d_model,dropout)sublayer = lambda x:self_attn(x,x,x,mask) # 子层函数sc = SublayerConnection(size,dropout)sc_result = sc(x,sublayer)print("sc_result.sahpe = ", sc_result.shape)print("sc_result = ", sc_result)# 编码器层 测试size = 512dropout=0.2head=8d_model=512d_ff = 64x = pe_resultmask = torch.zeros(8,4,4)self_attn = MultiHeadedAttention(head,d_model,dropout)ff = PositionwiseFeedForward(d_model,d_ff,dropout)mask = torch.zeros(8,4,4)el = EncoderLayer(size,self_attn,ff,dropout)el_result = el(x,mask)print("el_result.shape = ", el_result.shape)print("el_result = ", el_result)# 编码器测试size = 512dropout=0.2head=8d_model=512d_ff = 64c = copy.deepcopyx = pe_resultself_attn = MultiHeadedAttention(head,d_model,dropout)ff = PositionwiseFeedForward(d_model,d_ff,dropout)# 编码器层不是共享的，因此需要深度拷贝layer= EncoderLayer(size,c(self_attn),c(ff),dropout)N=8mask = torch.zeros(8,4,4)en = Encoder(layer,N)en_result = en(x,mask)print("en_result.shape : ",en_result.shape)print("en_result : ",en_result)
