经典神经网络架构参考 v1.0-编程知识

一、线性模型

1.1 线性回归

digraph LinearRegression {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize, NFeature]", shape="Mrecord"]ll  [label="Linear\n[NFeature, 1]"]oup [label="输出\n[BatchSise, 1]", shape="Mrecord"]inp -> ll -> oup
}

1.2 逻辑回归

digraph SoftmaxRegression {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp     [label="输入\n[BatchSize, NFeature]", shape="Mrecord"]ll      [label="Linear\n[NFeature, NLabel]"]softmax [label="Softmax"]oup     [label="输出\n[BatchSise, NLabel]", shape="Mrecord"]inp -> ll -> softmax -> oup
}

1.3 Softmax 回归

digraph SoftmaxRegression {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp     [label="输入\n[BatchSize, NFeature]", shape="Mrecord"]ll      [label="Linear\n[NFeature, NLabel]"]softmax [label="Softmax"]oup     [label="输出\n[BatchSise, NLabel]", shape="Mrecord"]inp -> ll -> softmax -> oup
}

二、MLP

digraph MLP {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp     [label="输入\n[BatchSize,\n NFeature(768)]", shape="Mrecord"]ll1     [label="Linear\n[NFeature(768),\n NHidden1(512)]"]relu1   [label="Relu"]ll2     [label="Linear\n[NHidden1(512),\n NHidden2(256)]"]relu2   [label="Relu"]ll3     [label="Linear\n[NHidden2(256),\n NLabels(10)]"]softmax [label="Softmax"]oup     [label="输出\n[BatchSise,\n NLabels(10)]", shape="Mrecord"]inp -> ll1 -> relu1 -> ll2 -> relu2-> ll3 -> softmax -> oup
}

三、卷积神经网络

3.1 LeNet

digraph Lenet {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize,\n W=32, H=32, C=1]", shape="Mrecord"]conv1 [label="Conv2D 1\n[In=1, Out=6, K=5]"]relu1 [label="Relu"]featmap11 [label="[BatchSize,\n W=28, H=28, C=6]", shape="Mrecord"]pool1 [label="MaxPool2D 1\n[K=2, S=2]"]featmap12 [label="[BatchSize,\n W=14, H=14, C=6]", shape="Mrecord"]conv2 [label="Conv2D 2\n[In=6, Out=16, K=5]"]relu2 [label="Relu"]featmap21 [label="[BatchSize,\n W=10, H=10, C=16]", shape="Mrecord"]pool2 [label="MaxPool2D 2\n[K=2, S=2]"]featmap22 [label="[BatchSize,\n W=5, H=5, C=16]", shape="Mrecord"]reshape [label="reshape\n[BatchSize, 16x5x5]"]ll1 [label="Linear1\n[16x5x5, 120]"]relu3 [label="Relu"]ll2 [label="Linear2\n[120, 84]"]relu4 [label="Relu"]ll3  [label="Linear3\n[84, NLabel(10)]"]softmax [label="Softmax"]oup [label="输出\n[BatchSise,\n NLabel(10)]", shape="Mrecord"]inp -> conv1 -> relu1 -> featmap11 -> pool1 -> featmap12 ->conv2 -> relu2 -> featmap21 -> pool2 -> featmap22 ->reshape -> ll1 -> relu3 -> ll2 -> relu4 -> ll3 -> softmax -> oup}

3.2 AlexNet

块 #1：

digraph AlexNetL1 {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize,\n W=227, H=227, C=3]", shape="Mrecord"]conv1 [label="Conv2D#1\n[K=11, S=4,\n In=3, Out=48x2]"]relu1 [label="Relu"]featmap11 [label="[BatchSize,\nW=55, H=55, C=48x2]", shape="Mrecord"]maxpool1 [label="MaxPool2D#1\n[K=3, S=2]"]featmap12 [label="[BatchSize,\nW=27, H=27, C=48x2]", shape="Mrecord"]inp -> conv1 -> relu1 -> featmap11 -> maxpool1 -> featmap12}

块 #2：

digraph AlexNetL2 {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]featmap12 [label="[BatchSize,\nW=27, H=27, C=48x2]", shape="Mrecord"]conv2 [label="Conv2D#2\n[K=5, P=2,\n In=48x2, Out=128x2]"]relu2 [label="Relu"]featmap21 [label="[BatchSize,\nW=27, H=27, C=128x2]", shape="Mrecord"]maxpool2 [label="MaxPool2D#2\n[K=3, S=2]"]featmap22 [label="[BatchSize,\nW=13, H=13, C=128x2]", shape="Mrecord"]featmap12 -> conv2 -> relu2 -> featmap21 -> maxpool2 -> featmap22
}

块 #3 和 #4：

digraph AlexNetL34 {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]featmap22 [label="[BatchSize,\nW=13, H=13, C=128x2]", shape="Mrecord"]conv3 [label="Conv2D#3\n[K=3, P=1,\n In=128x2, Out=192x2]"]relu3 [label="Relu"]featmap3 [label="[BatchSize,\nW=13, H=13, C=192x2]", shape="Mrecord"]conv4 [label="Conv2D#4\n[K=3, P=1,\n In=192x2, Out=192x2]"]relu4 [label="Relu"]featmap4 [label="[BatchSize,\nW=13, H=13, C=192x2]", shape="Mrecord"]featmap22 -> conv3 -> relu3 -> featmap3 -> conv4 -> relu4 -> featmap4
}

块 #5：

digraph AlexNetL5 {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]featmap4 [label="[BatchSize,\nW=13, H=13, C=192x2]", shape="Mrecord"]conv5 [label="Conv2D#5\n[K=3, P=1,\n In=192x2, Out=128x2]"]relu5 [label="Relu"]featmap51 [label="[BatchSize,\nW=13, H=13, C=128x2]", shape="Mrecord"]maxpool5 [label="MaxPool2D#5\n[K=3, S=2]"]featmap52 [label="[BatchSize,\nW=6, H=6, C=128x2]", shape="Mrecord"]featmap4 -> conv5 -> relu5 -> featmap51 -> maxpool5 -> featmap52
}

块 #6 #7 #8：

digraph AlexNetL678 {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]featmap52 [label="[BatchSize,\nW=6, H=6, C=128x2]", shape="Mrecord"]reshape [label="reshape([4096])"]linear6 [label="Linear6\n[4096, 4096]"]relu6 [label="Relu"]dropout6 [label="Dropout"]linear7 [label="Linear7\n[4096, 4096]"]relu7 [label="Relu"]dropout7 [label="Dropout"]linear8 [label="Linear7\n[4096, 1000]"]softmax [label="Softmax"]oup [label="输出\n[BatchSize, 1000]", shape="Mrecord"]featmap52 -> reshape -> linear6 -> relu6 -> dropout6 ->linear7 -> relu7 -> dropout7 -> linear8 -> softmax -> oup}

3.3 VGG16

主体：

digraph VGG16 {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize,\nW=224, H=224, C=3]", shape="Mrecord"]convblock1 [label="ConvBlock1\n[In=3, Out=64]"]featmap1 [label="[BatchSize\nW=112, H=112, C=64]", shape="Mrecord"]convblock2 [label="ConvBlock2\n[In=64, Out=128]"]featmap2 [label="[BatchSize\nW=56, H=56, C=128]", shape="Mrecord"]convblock3 [label="ConvBlock3\n[In=128, Out=256]"]featmap3 [label="[BatchSize\nW=28, H=28, C=256]", shape="Mrecord"]convblock4 [label="ConvBlock4\n[In=256, Out=512]"]featmap4 [label="[BatchSize\nW=14, H=14, C=512]", shape="Mrecord"]convblock5 [label="ConvBlock5\n[In=512, Out=512]"]featmap5 [label="[BatchSize\nW=7, H=7, C=512]", shape="Mrecord"]reshape [label="reshape([7x7x512])"]linear1 [label="Linear1\n[7x7x512, 4096]"]relu1 [label="Relu"]linear2 [label="Linear2\n[4096, 4096]"]relu2 [label="Relu"]linear3 [label="Linear3\n[4096, 1000]"]relu3 [label="Relu"]linear4 [label="Linear4\n[1000, 1000]"]softmax [label="Softmax"]oup [label="输出\n[BatchSize, 1000]", shape="Mrecord"]inp -> convblock1 -> featmap1 -> convblock2 -> featmap2 ->convblock3 -> featmap3 -> convblock4 -> featmap4 -> convblock5 -> featmap5 -> reshape -> linear1 -> relu1 ->linear2 -> relu2 -> linear3 -> relu3 -> linear4 -> softmax ->oup
}

卷积块：

digraph VGG16ConvBlock {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize, W=BlockW,\n H=BlockH, C=BlockIn]", shape="Mrecord"]conv1 [label="Conv2D#1\n[K=3, P=1,\n In=BlockIn,\n Out=BlockOut]"]conv2 [label="Conv2D#2\n[K=3, P=1,\n In=BlockOut,\n Out=BlockOut]"]relu  [label="relu"]maxpool [label="MaxPool2D\n[K=2, S=2]"]oup [label="输出\n[BatchSize,\n W=BlockW/2,\n H=BlockH/2,\n C=BlockOut]", shape="Mrecord"]inp -> conv1 -> conv2 -> relu -> maxpool -> oup}

3.4 ResNet18

主体：

digraph ResNet18 {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize,\nW=224, H=224, C=3]", shape="Mrecord"]conv0 [label="Conv2D#0\n[K=7, S=2, P=3, In=3, Out=64]"]maxpool0 [label="MaxPool2D#0\n[K=3, S=2, P=1]"]featmap0 [label="[BatchSize,\nW=56, H=56, C=64]", shape="Mrecord"]convblock1 [label="ConvBlock1\n[C=64]"]convblock2 [label="ConvBlock2\n[C=64]"]featmap2 [label="[BatchSize,\nW=56, H=56, C=64]", shape="Mrecord"]downconvblock3 [label="DownConvBlock3\n[In=64, Out=128]"]convblock4 [label="ConvBlock4\n[C=128]"]featmap4 [label="[BatchSize,\nW=28, H=28, C=128]", shape="Mrecord"]downconvblock5 [label="DownConvBlock5\n[In=128, Out=256]"]convblock6 [label="ConvBlock6\n[C=256]"]featmap6 [label="[BatchSize,\nW=14, H=14, C=256]", shape="Mrecord"]downconvblock7 [label="DownConvBlock7\n[In=256, Out=512]"]convblock8 [label="ConvBlock8\n[C=512]"]featmap8 [label="[BatchSize,\nW=7, H=7, C=512]", shape="Mrecord"]avgpool [label="AvgPool2D\n[K=7, S=7]"]featmap9 [label="[BatchSize,\nW=1, H=1, C=512]", shape="Mrecord"]reshape [label="reshape([512])"]linear [label="Linear\n[512, 1000]"]softmax [label="Softmax"]oup [label="输出\n[BatchSize, 1000]", shape="Mrecord"]inp -> conv0 -> maxpool0 -> featmap0 -> convblock1 -> convblock2 -> featmap2 -> downconvblock3 -> convblock4 -> featmap4 ->downconvblock5 -> convblock6 -> featmap6 ->downconvblock7 -> convblock8 -> featmap8 -> avgpool ->featmap9 -> reshape -> linear -> softmax -> oup}

卷积块：

digraph ResNet18ConvBlock {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize,\nW=BlockW, \nH=BlockH, C=BlockC]", shape="Mrecord"]conv1 [label="Conv2D#1\n[K=3, P=1,\n In=BlockC,\n Out=BlockC]"]relu1 [label="Relu"]conv2 [label="Conv2D#2\n[K=3, P=1,\n In=BlockC,\n Out=BlockC]"]add   [label="+"]relu2 [label="Relu"]inp -> conv1 -> relu1 -> conv2 -> add -> relu2 -> oupinp -> add
}

下卷积块：

digraph ResNet18DownConvBlock {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize,\nW=BlockW, \nH=BlockH, C=BlockIn]", shape="Mrecord"]conv1 [label="Conv2D#1\n[K=3, S=2, P=1,\n In=BlockIn,\n Out=BlockOut]"]relu1 [label="Relu"]conv2 [label="Conv2D#2\n[K=3, P=1,\n In=BlockOut,\n Out=BlockOut]"]downconv [label="Conv2D#0\n[K=1, S=2,\n In=BlockIn,\n Out=BlockOut]"]add   [label="+"]relu2 [label="Relu"]inp -> conv1 -> relu1 -> conv2 -> add -> relu2 -> oupinp -> downconv -> add
}

3.5 DenseNet

主体：

digraph DenseNet {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=7, height=2,shape="box",]inp [label="输入\n[BatchSize, W=224,\n H=224, C=3]"]conv0 [label="Conv2D#0\n[K=7, S=2, P=3,\nIn=3, Out=64]"]bn0 [label="BatchNorm2D(64)"]relu0 [label="Relu"]maxpool0 [label="MaxPool2D#0\n[K=3, S=2, P=1]"]featmap0 [label="[BatchSize,\nW=56, H=56, C=64"]dense1 [label="DenseBlock1\n[NLayers=6, BNSize=4,\n GrowthRate=32]"]featmap11 [label="[BatchSize, W=56, H=56,\n C=64+6*32=256]"]trans1 [label="Trasition1\n[CompRate=0.5]"]featmap12 [label="[BatchSize, W=28, H=28, C=128]"]dense2 [label="DenseBlock2\n[NLayers=12, BNSize=4,\n GrowthRate=32]"]featmap21 [label="[BatchSize, W=28, H=28,\n C=128+12*32=512]"]trans2 [label="Trasition2\n[CompRate=0.5]"]featmap22 [label="[BatchSize, W=14, H=14, C=256]"]dense3 [label="DenseBlock3\n[NLayers=24, BNSize=4,\n GrowthRate=32]"]featmap31 [label="[BatchSize, W=14, H=14,\n C=256+24*32=1024]"]trans3 [label="Trasition3\n[CompRate=0.5]"]featmap32 [label="[BatchSize, W=7, H=7, C=512]"]dense4 [label="DenseBlock4\n[NLayers=16, BNSize=4,\n GrowthRate=32]"]featmap4 [label="[BatchSize, W=7, H=7,\n C=512+16*32=1024]"]avgpool [label="AvgPool2D\n[K=7, S=7]"]featmap5 [label="[BatchSize, W=1, H=1, C=1024]"]reshape [label="reshape([1024])"]linear [label="Linear\n[1024, 1000"]softmax [label="Softmax"]oup [label="输出\n[BatchSize, 1000]"]inp -> conv0 -> bn0 -> relu0 -> maxpool0 ->  featmap0 ->dense1 -> featmap11 -> trans1 -> featmap12 ->dense2 -> featmap21 -> trans2 -> featmap22 ->dense3 -> featmap31 -> trans3 -> featmap32 ->dense4 -> featmap4 -> avgpool -> featmap5 ->reshape -> linear -> softmax -> oup
}

密集块：

digraph DenseBlock {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=7, height=2,shape="box",]inp [label="输入\n[BatchSize, W=BlockW,\n H=BlockH, C=LayerIn]"]DenseLayers [label="DenseLayer x NLayers\n[In=BlockIN+#Layer*GrowthRate]"]oup [label="输出\n[BatchSize,\n W=BlockW, H=BlockH,\n C=LayerIn+NumLayer*GrowthRate ]"]inp -> DenseLayers -> oup}

密集层：

digraph DenseLayer {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize, W=BlockW,\n H=BlockH, C=LayerIn]"]bn1 [label="BatchNorm1(LayerIn)"]relu1  [label="Relu"]conv1 [label="Conv2D#1\n[K=1, In=LayerIn,\n Out=BNSize*GrowthRate]"]bn2 [label="BatchNorm2\n(BNSize*GrowthRate)"]relu2  [label="Relu"]conv2 [label="Conv2D#1\n[K=3, P=1,\n In=BNSize*GrowthRate,\n Out=GrowthRate]"]cat [label="cat(axis=1)"]oup [label="输出\n[BatchSize,\n W=BlockW, H=BlockH,\n C=LayerIn+GrowthRate]"]inp -> bn1 -> relu1 -> conv1 -> bn2 -> relu2 -> conv2 -> cat -> oupinp -> cat
}

过渡块：

digraph TransitionLayer {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp [label="输入\n[BatchSize, W=LayerW,\n H=LayerH, C=LayerIn]"]bn [label="BatchNorm1(LayerIn)"]relu  [label="Relu"]conv [label="Conv2D\n[K=1, In=LayerIn,\n Out=LayerIn*CompRate]"]maxpool [label="MaxPool2D\n[K=2, S=2]"]oup [label="输出\n[BatchSize,\n W=BlockW/2,\n H=BlockH/2,\n C=LayerIn*CompRate]"]inp -> bn -> relu -> conv -> maxpool -> oup
}

四、自编码器

4.1 栈式自编码器

digraph StackAutoEncoder {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp     [label="输入\n[BatchSize,\n NFeature(768)]", shape="Mrecord"]ll1     [label="Linear\n[NFeature(768),\n NHidden1(512)]"]σ1      [label="Sigmoid"]ll2     [label="Linear\n[NHidden1(512),\n NHidden2(256)]"]σ2      [label="Sigmoid"]ll3     [label="Linear\n[NHidden2(256),\n NHidden1(512)]"]σ3      [label="Sigmoid"]ll4     [label="Linear\n[NHidden1(512),\n NFeature(768)]"]oup     [label="输出\n[BatchSise,\n NFeature(768)]", shape="Mrecord"]inp -> ll1 -> σ1 -> ll2 -> σ2-> ll3 -> σ3 -> ll4 -> oup
}

4.2 去噪自编码器

digraph DenoiseAutoEncoder {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp       [label="输入\n[BatchSize,\n NFeature(768)]", shape="Mrecord"]noise     [label="AddNoise\n+ 0.5 * randn(\nBatchSize,\n NFeature(768))"]ll1       [label="Linear\n[NFeature(768),\n NHidden1(512)]"]σ1        [label="Sigmoid"]ll2       [label="Linear\n[NHidden1(512),\n NHidden2(256)]"]σ2        [label="Sigmoid"]ll3       [label="Linear\n[NHidden2(256),\n NHidden1(512)]"]σ3        [label="Sigmoid"]ll4       [label="Linear\n[NHidden1(512),\n NFeature(768)]"]oup       [label="输出\n[BatchSize,\n NFeature(768)]", shape="Mrecord"]inp -> noise ->  ll1 -> σ1 -> ll2 -> σ2-> ll3 -> σ3 -> ll4 -> oup
}

4.3 变分自编码器

digraph VariationAutoEncoder {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp           [label="输入\n[BatchSize,\n NFeature(768)]", shape="Mrecord"]ll1           [label="Linear\n[NFeature(768),\n NHidden1(512)]"]σ1            [label="Sigmoid"]ll2           [label="Linear\n[NHidden1(512),\n NHidden2(256)]"]σ2            [label="Sigmoid"]ll3_mean      [label="Linear\n[NHidden2(256),\n NHidden3(128)]"]ll3_log_var   [label="Linear\n[NHidden2(256),\n NHidden3(128)]"]z_mean        [label="μ\n[BatchSize,\n NHidden3(128)]", shape="Mrecord"]z_log_var     [label="logσ²\n[BatchSize,\n NHidden3(128)]", shape="Mrecord"]eps           [label="ε\nrandn(BatchSize,\n NHidden3(128))", shape="Mrecord"]sample        [label="μ + exp(logσ² / 2) * ε", width=7]z             [label="z\n[BatchSize,\n NHidden3(128)]", shape="Mrecord"]ll4           [label="Linear\n[NHidden3(128),\n NHidden2(256)]"]σ4            [label="Sigmoid"]ll5           [label="Linear\n[NHidden2(256),\n NHidden1(512)]"]σ5            [label="Sigmoid"]ll6           [label="Linear\n[NHidden1(512),\n NFeature(768)]"]oup           [label="输出\n[BatchSize,\n NFeature(768)]", shape="Mrecord"]inp -> ll1 -> σ1 -> ll2 -> σ2σ2 -> ll3_mean -> z_mean -> sampleσ2 -> ll3_log_var -> z_log_var -> sampleeps -> sample -> z -> ll4 -> σ4 -> ll5 -> σ5 -> ll6 -> oup}

4.4 卷积自编码器

主体：

digraph ConvAutoEncoder {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp         [label="输入\n[BatchSize,\n W=256, H=256, C=3]", shape="Mrecord"]conv1       [label="ConvBlock1\n[In=3, Out=16]"]featmap_e1  [label="[BatchSize,\nW=128, H=128, C=16]", shape="Mrecord"]conv2       [label="ConvBlock2\n[In=16, Out=32]"]featmap_e2  [label="[BatchSize,\nW=64, H=64, C=32]", shape="Mrecord"]conv3       [label="ConvBlock3\n[In=32, Out=32]"]featmap_e3  [label="[BatchSize,\nW=32, H=32, C=32]", shape="Mrecord"]upconv3     [label="UpConvBlock3\n[In=32, Out=32]"]featmap_d2  [label="[BatchSize,\nW=64, H=64, C=32]", shape="Mrecord"]upconv2     [label="UpConvBlock2\n[In=32, Out=16]"]featmap_d1  [label="[BatchSize,\nW=128, H=128, C=16", shape="Mrecord"]upconv1     [label="UpConvBlock1\n[In=16, Out=3]"]tanh        [label="Tanh"]oup         [label="输出\n[BatchSize,\n W=256, H=256, C=3]", shape="Mrecord"]inp -> conv1 -> featmap_e1 -> conv2 -> featmap_e2-> conv3 -> featmap_e3 -> upconv3 -> featmap_d2-> upconv2 -> featmap_d1 -> upconv1-> tanh -> oup 
}

卷积块：

digraph CAEConvBlock {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp         [label="输入\n[BatchSize,\n W=BlockW,\n H=BlockH,\n C=BlockIn]", shape="Mrecord"]conv        [label="Conv2D\n[K=3, P=1,\n In=BlockIn,\n Out=BlockOut]"]batchnorm   [label="BatchNorm(BlockOut)"]relu        [label="Relu"]maxpool     [label="MaxPool2D\n[K=2, S=2]"]oup         [label="输出\n[BatchSize,\n W=BlockW/2,\n H=BlockH/2,\n C=BlockOut]", shape="Mrecord"]inp -> conv -> batchnorm -> relu -> maxpool -> oup 
}

反卷积块：

digraph CAEDeconvBlock {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp         [label="输入\n[BatchSize,\n W=BlockW,\n H=BlockH,\n C=BlockIn]", shape="Mrecord"]upsamp      [label="UpSample x2"]conv        [label="Conv2D\n[K=3, P=1,\n In=BlockIn,\n Out=BlockOut]"]batchnorm   [label="BatchNorm(BlockOut)"]relu        [label="Relu"]oup         [label="输出\n[BatchSize,\n W=BlockW*2,\n H=BlockH*2,\n C=BlockOut]", shape="Mrecord"]inp -> upsamp -> conv -> batchnorm -> relu -> oup 
}

五、循环神经网络

5.1 时序 RNN

digraph TimeRNN {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp      [label="输入\n[BatchSize,\n SeqLen, NFeature]", shape="Mrecord"]rnn1     [label="RNNCell1\n[NFeature, NHidden]"]rnn_rest [label="RNNCell x (NLayers-1)\n[NHidden, NHidden]"]hidst    [label="[BatchSize,\n SeqLen, NHidden]", shape="Mrecord"]ll       [label="Linear\n[NHidden, NLabels]"]oup      [label="输出\n[BatchSize,\n SeqLen, NLabels]", shape="Mrecord"]inp -> rnn1 -> rnn_rest -> hidst -> ll -> oup
}

5.2 NLP RNN

digraph NLPRNN {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]inp      [label="输入（单词ID）\n[BatchSize, SeqLen]", shape="Mrecord"]emb      [label="Embedding\n[VocabSize, NHidden]"]inp_emb  [label="[BatchSize,\n SeqLen, NHidden]", shape="Mrecord"]rnn      [label="RNNCell x NLayers\n[NHidden, NHidden]"]hidst    [label="[BatchSize,\n SeqLen, NHidden]", shape="Mrecord"]emb_w_t  [label="@ Embedding.w.T\n[NHidden, VocabSize]"]oup      [label="输出（单词概率）\n[BatchSize,\n SeqLen, VocabSize]", shape="Mrecord"]inp -> emb -> inp_emb -> rnn -> hidst -> emb_w_t -> oup
}

5.3 RNN 单元

digraph RNNCell {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]x [label="X[t]（输入）\n[BatchSize, NIn]", shape="Mrecord"]his [label="H[t-1]（历史）\n[BatchSize, NOut]", shape="Mrecord"]w_x [label="W（参数）\n[NIn, NOut]", shape="Mrecord"]w_h [label="U（参数）\n[NOut, NOut]", shape="Mrecord"]b   [label="b（参数）\n[1, NOut]", shape="Mrecord"]σ   [label="Sigmoid"]matmul1 [label="@"]matmul2 [label="@"]plus  [label="+"]h   [label="H[t]（输出）\n[BatchSize, NOut]", shape="Mrecord"]x -> matmul1w_x -> matmul1his -> matmul2w_h -> matmul2matmul1 -> plusmatmul2 -> plusb -> plusplus -> σσ -> h}

5.4 LSTM 单元

digraph GRUCell {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]x [label="X[t]（输入）\n[BatchSize, NIn]", shape="Mrecord"]his_h [label="H[t-1]（短期历史）\n[BatchSize, NOut]", shape="Mrecord"]his_c [label="C[t-1]（长期历史）\n[BatchSize, NOut]", shape="Mrecord"]calc_f [label="σ(X[t] @ W[f]\n + H[t-1] @ U[f]\n + b[f])"]calc_i [label="σ(X[t] @ W[i]\n + H[t-1] @ U[i]\n + b[i])"]calc_c_cand [label="tanh(X[t] @ W[c]\n + H[t-1] @ U[c]\n + b[c])"]calc_o [label="σ(X[t] @ W[o]\n + H[t-1] @ U[o]\n + b[o])"]f   [label="F[t]（遗忘门）\n[BatchSize, NOut]", shape="Mrecord"]i   [label="I[t]（记忆门）\n[BatchSize, NOut]", shape="Mrecord"]c_cand   [label="CCand[t]（长期候补）\n[BatchSize, NOut]", shape="Mrecord"]o   [label="O[t]（输出门）\n[BatchSize, NOut]", shape="Mrecord"]calc_c [label="F[t] * C[t-1]\n + I[t] * CCand[t]"]calc_h [label="C[t] * tanh(O[t])"]h [label="H[t]（输出&短期记忆）\n[BatchSize, NOut]", shape="Mrecord"]c [label="C[t]（长期记忆）\n[BatchSize, NOut]", shape="Mrecord"]x -> calc_fx -> calc_ix -> calc_c_candx -> calc_ohis_h -> calc_f -> fhis_h -> calc_i -> ihis_h -> calc_c_cand -> c_candhis_h -> calc_o -> of -> calc_ci -> calc_cc_cand -> calc_ccalc_c -> cc -> calc_ho -> calc_hcalc_h -> h}

5.5 GRU 单元

digraph LSTMCell {rankdir=BTnode [style=filled, color=Blackfontcolor=White, fillcolor="#30638e", fontname="SimHei",fontsize=32,width=5, height=2,shape="box",]x [label="X[t]（输入）\n[BatchSize, NIn]", shape="Mrecord"]his_h [label="H[t-1]（历史）\n[BatchSize, NOut]", shape="Mrecord"]calc_z [label="σ(X[t] @ W[z]\n + H[t-1] @ U[x]\n + b[z])"]calc_r [label="σ(X[t] @ W[r]\n + H[t-1] @ U[r]\n + b[r])"]calc_h_cand [label="tanh(X[t] @ W[h]\n + (H[t-1] * R[t]) @ U[h]\n + b[h])"]z   [label="Z[t]（更新门）\n[BatchSize, NOut]", shape="Mrecord"]r   [label="R[t]（复位门）\n[BatchSize, NOut]", shape="Mrecord"]h_cand   [label="HCand[t]（候补）\n[BatchSize, NOut]", shape="Mrecord"]calc_h [label="(1 - Z[t]) * H[t-1]\n + Z[t] * HCand[t]"]h [label="H[t]（输出）\n[BatchSize, NOut]", shape="Mrecord"]x -> calc_zx -> calc_rx -> calc_h_candhis_h -> calc_z -> zhis_h -> calc_r -> rhis_h -> calc_h_candr -> calc_h_cand -> h_candz -> calc_hh_cand -> calc_hcalc_h -> h
}