【深度学习】深度学习之解构基础网络结构

import torchfrom torch import nnimport torch.nn.functional as Fclass LeNet5(nn.Module):    def __init__(self):        super(LeNet5, self).__init__()        self.conv1 = nn.Conv2d(1, 6, 5)


    
        self.conv2 = nn.Conv2d(6, 16, 5)        self.max_pool_1 = nn.MaxPool2d(2)        self.max_pool_2 = nn.MaxPool2d(2)        self.fc1 = nn.Linear(16*5*5, 120)        self.fc2 = nn.Linear(120, 84)        self.fc3 = nn.Linear(84, 10)    def forward(self, x):        x = self.max_pool_1(F.relu(self.conv1(x)))        x = self.max_pool_2(F.relu(self.conv2(x)))        x = x.view(-1, 16*5*5)        x = self.fc1(x)        x = self.fc2(x)        x = self.fc3(x)        return x
input1 = torch.rand([1,1,32,32])lenet = LeNet5()print(lenet)output = lenet(input1)print(output.shape)

LeNet5(  (conv1): Conv2d(1, 6, kernel_size=(5, 5), stride=(1, 1))  (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))  (max_pool_1): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)  (max_pool_2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)  (fc1): Linear(in_features=400, out_features=120, bias=True)  (fc2): Linear(in_features=120, out_features=84, bias=True)  (fc3): Linear(in_features=84, out_features=10, bias=True))torch.Size([1, 10])

import torchfrom torch import nnimport torch.nn.functional as Fclass AlexNet(nn.Module):


    
    def __init__(self):        super(AlexNet, self).__init__()        self.conv1 = nn.Sequential(nn.Conv2d(3, 96, kernel_size=11, stride=4, padding=2),                                   nn.ReLU(),                                   nn.MaxPool2d(kernel_size=3,stride=2)                                  )        self.conv2 = nn.Sequential(nn.Conv2d(96, 256, kernel_size=5, stride=1, padding=2),                                   nn.ReLU(),                                   nn.MaxPool2d(kernel_size=3,stride=2)                                  )        self.conv3 = nn.Sequential(nn.Conv2d(256, 384, kernel_size=3, padding=1),                                   nn.ReLU()                                  )        self.conv4 = nn.Sequential(nn.Conv2d(384, 384, kernel_size=3, padding=1),                                   nn.ReLU()                                  )        self.conv5 = nn.Sequential(nn.Conv2d(384, 256, kernel_size=3, padding=1),                                   nn.ReLU(),                                   nn.MaxPool2d(2)                                  )        self.fc1 = nn.Linear(256*6*6, 4096)        self.fc2 = nn.Linear(4096, 4096)        self.fc3 = nn.Linear(4096, 1000)                        pass    def forward(self, x):        x = self.conv1(x)        x = self.conv2(x)        x = self.conv3(x)        x = self.conv4(x)        x = self.conv5(x)        x = x.view(x.size(0), -1)        x = self.fc1(x)        x = self.fc2(x)        x = self.fc3(x)        return xinput1 = torch.rand([1, 3, 224, 224])alex_net = AlexNet()print(alex_net)output = alex_net(input1)print(output.shape)

AlexNet(  (conv1): Sequential(


    
    (0): Conv2d(3, 96, kernel_size=(11, 11), stride=(4, 4), padding=(2, 2))    (1): ReLU()    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)  )  (conv2): Sequential(    (0): Conv2d(96, 256, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))    (1): ReLU()    (2): MaxPool2d(kernel_size=3, stride=2, padding=0, dilation=1, ceil_mode=False)  )  (conv3): Sequential(    (0): Conv2d(256, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))    (1): ReLU()  )  (conv4): Sequential(    (0): Conv2d(384, 384, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))    (1): ReLU()  )  (conv5): Sequential(    (0): Conv2d(384, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))    (1): ReLU()    (2): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)  )  (fc1): Linear(in_features=9216, out_features=4096, bias=True)  (fc2): Linear(in_features=4096, out_features=4096, bias=True)  (fc3): Linear(in_features=4096, out_features=1000, bias=True))torch.Size([1, 1000])

import torchfrom torch import nndef make_layers(cfg):    layers = []    in_channels = 3    for v in cfg:        if v == 'M':            layers += [nn.MaxPool2d(kernel_size=2, stride=2)]        else:            v = int(v)            conv2d = nn.Conv2d(in_channels, v, kernel_size=3, padding=1)            layers += [conv2d, nn.ReLU(inplace=True)]                        in_channels = v    return nn.Sequential(*layers)    

class VGG16(nn.Module):    def __init__(self, features, num_classes=1000):        super(VGG16, self).__init__()        self.features = features        self.avgpool = nn.AdaptiveAvgPool2d((7, 7))        self.classifier = nn.Sequential(            nn.Linear(512 * 7 * 7, 4096),            nn.ReLU(True),            nn.Dropout(),            nn.Linear(4096, 4096),            nn.ReLU(True),            nn.Dropout(),            nn.Linear(4096, num_classes),        )        def forward(self, x):        x = self.features(x)        x = self.avgpool(x)        x = torch.flatten(x, 1)        x = self.classifier(x)        return x
cfg = [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M']input1 = torch.rand([


    
1, 3, 224, 224])vgg16 = VGG16(features=make_layers(cfg))print(vgg16)output = vgg16(input1)print(output.shape)

from collections import namedtupleimport warningsimport torchfrom torch import nn, Tensorimport torch.nn.functional as Ffrom typing import Callable, Any, Optional, Tuple, List



    
InceptionOutputs = namedtuple('InceptionOutputs', ['logits', 'aux_logits'])InceptionOutputs.__annotations__ = {'logits': Tensor, 'aux_logits': [Tensor]}
# Script annotations failed with _GoogleNetOutputs = namedtuple ...# _InceptionOutputs set here for backwards compat_InceptionOutputs = InceptionOutputs
class Inception3(nn.Module):
    def __init__(        self,        num_classes: int = 1000,        aux_logits: bool = False,        transform_input: bool = False,        inception_blocks = None,        init_weights = None) -> None:        super(Inception3, self).__init__()        if inception_blocks is None:            inception_blocks = [                BasicConv2d, InceptionA, InceptionB, InceptionC,                InceptionD, InceptionE, InceptionAux            ]        assert len(inception_blocks) == 7        conv_block = inception_blocks[0]        inception_a = inception_blocks[1]        inception_b = inception_blocks[2]        inception_c = inception_blocks[3]        inception_d = inception_blocks[4]        inception_e = inception_blocks[5]        inception_aux = inception_blocks[6]
        self.aux_logits = aux_logits        self.transform_input = transform_input        self.Conv2d_1a_3x3 = conv_block(3, 32, kernel_size=3, stride=2)        self.Conv2d_2a_3x3 = conv_block(32, 32, kernel_size=3)        self.Conv2d_2b_3x3 = conv_block(32, 64, kernel_size=3, padding=1)        self.maxpool1 = nn.MaxPool2d(kernel_size=3, stride=2)        self.Conv2d_3b_1x1 = conv_block(64, 80, kernel_size=1)        self.Conv2d_4a_3x3 = conv_block(80, 192, kernel_size=3)        self.maxpool2 = nn.MaxPool2d(kernel_size=3, stride=2)        self.Mixed_5b = inception_a(192, pool_features=32)


    
        self.Mixed_5c = inception_a(256, pool_features=64)        self.Mixed_5d = inception_a(288, pool_features=64)        self.Mixed_6a = inception_b(288)        self.Mixed_6b = inception_c(768, channels_7x7=128)        self.Mixed_6c = inception_c(768, channels_7x7=160)        self.Mixed_6d = inception_c(768, channels_7x7=160)        self.Mixed_6e = inception_c(768, channels_7x7=192)        self.AuxLogits = None        if aux_logits:            self.AuxLogits = inception_aux(768, num_classes)        self.Mixed_7a = inception_d(768)        self.Mixed_7b = inception_e(1280)        self.Mixed_7c = inception_e(2048)        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))        self.dropout = nn.Dropout()        self.fc = nn.Linear(2048, num_classes)
    def _transform_input(self, x: Tensor) -> Tensor:        if self.transform_input:            x_ch0 = torch.unsqueeze(x[:, 0], 1) * (0.229 / 0.5) + (0.485 - 0.5) / 0.5            x_ch1 = torch.unsqueeze(x[:, 1], 1) * (0.224 / 0.5) + (0.456 - 0.5) / 0.5            x_ch2 = torch.unsqueeze(x[:, 2], 1) * (0.225 / 0.5) + (0.406 - 0.5) / 0.5            x = torch.cat((x_ch0, x_ch1, x_ch2), 1)        return x
    def _forward(self, x: Tensor):        # N x 3 x 299 x 299        x = self.Conv2d_1a_3x3(x)        # N x 32 x 149 x 149        x = self.Conv2d_2a_3x3(x)        # N x 32 x 147 x 147        x = self.Conv2d_2b_3x3(x)        # N x 64 x 147 x 147        x = self.maxpool1(x)        # N x 64 x 73 x 73        x = self.Conv2d_3b_1x1(x)        # N x 80 x 73 x 73        x = self.Conv2d_4a_3x3(x)        # N x 192 x 71 x 71        x = self.maxpool2(x)        # N x 192 x 35 x 35


    
        x = self.Mixed_5b(x)        # N x 256 x 35 x 35        x = self.Mixed_5c(x)        # N x 288 x 35 x 35        x = self.Mixed_5d(x)        # N x 288 x 35 x 35        x = self.Mixed_6a(x)        # N x 768 x 17 x 17        x = self.Mixed_6b(x)        # N x 768 x 17 x 17        x = self.Mixed_6c(x)        # N x 768 x 17 x 17        x = self.Mixed_6d(x)        # N x 768 x 17 x 17        x = self.Mixed_6e(x)        # N x 768 x 17 x 17        aux = None        if self.AuxLogits is not None:            if self.training:                aux = self.AuxLogits(x)        # N x 768 x 17 x 17        x = self.Mixed_7a(x)        # N x 1280 x 8 x 8        x = self.Mixed_7b(x)        # N x 2048 x 8 x 8        x = self.Mixed_7c(x)        # N x 2048 x 8 x 8        # Adaptive average pooling        x = self.avgpool(x)        # N x 2048 x 1 x 1        x = self.dropout(x)        # N x 2048 x 1 x 1        x = torch.flatten(x, 1)        # N x 2048        x = self.fc(x)        # N x 1000 (num_classes)        return x, aux
    def eager_outputs(self, x: Tensor, aux: Optional[Tensor]) -> InceptionOutputs:        if self.training and self.aux_logits:            return InceptionOutputs(x, aux)        else:            return x  # type: ignore[return-value]
    def forward(self, x: Tensor) -> InceptionOutputs:        x = self._transform_input(x)        x, aux = self._forward(x)        aux_defined = self.training and self.aux_logits        if torch.jit.is_scripting():            if not aux_defined:                warnings.warn("Scripted Inception3 always returns Inception3 Tuple")


    
            return InceptionOutputs(x, aux)        else:            return self.eager_outputs(x, aux)

class InceptionA(nn.Module):
    def __init__(        self,        in_channels: int,        pool_features: int,        conv_block = None) -> None:        super(InceptionA, self).__init__()        if conv_block is None:            conv_block = BasicConv2d        self.branch1x1 = conv_block(in_channels, 64, kernel_size=1)
        self.branch5x5_1 = conv_block(in_channels, 48, kernel_size=1)        self.branch5x5_2 = conv_block(48, 64, kernel_size=5, padding=2)
        self.branch3x3dbl_1 = conv_block(in_channels, 64, kernel_size=1)        self.branch3x3dbl_2 = conv_block(64, 96, kernel_size=3, padding=1)        self.branch3x3dbl_3 = conv_block(96, 96, kernel_size=3, padding=1)
        self.branch_pool = conv_block(in_channels, pool_features, kernel_size=1)
    def _forward(self, x: Tensor) -> List[Tensor]:        branch1x1 = self.branch1x1(x)
        branch5x5 = self.branch5x5_1(x)        branch5x5 = self.branch5x5_2(branch5x5)
        branch3x3dbl = self.branch3x3dbl_1(x)        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)        branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)
        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)        branch_pool = self.branch_pool(branch_pool)
        outputs = [branch1x1, branch5x5, branch3x3dbl, branch_pool]        return outputs
    def forward(self, x: Tensor) -> Tensor:        outputs = self._forward(x)        return torch.cat(outputs, 1)

class InceptionB(nn.Module):



    
    def __init__(        self,        in_channels: int,        conv_block = None) -> None:        super(InceptionB, self).__init__()        if conv_block is None:            conv_block = BasicConv2d        self.branch3x3 = conv_block(in_channels, 384, kernel_size=3, stride=2)
        self.branch3x3dbl_1 = conv_block(in_channels, 64, kernel_size=1)        self.branch3x3dbl_2 = conv_block(64, 96, kernel_size=3, padding=1)        self.branch3x3dbl_3 = conv_block(96, 96, kernel_size=3, stride=2)
    def _forward(self, x: Tensor) -> List[Tensor]:        branch3x3 = self.branch3x3(x)
        branch3x3dbl = self.branch3x3dbl_1(x)        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)        branch3x3dbl = self.branch3x3dbl_3(branch3x3dbl)
        branch_pool = F.max_pool2d(x, kernel_size=3, stride=2)
        outputs = [branch3x3, branch3x3dbl, branch_pool]        return outputs
    def forward(self, x: Tensor) -> Tensor:        outputs = self._forward(x)        return torch.cat(outputs, 1)

class InceptionC(nn.Module):
    def __init__(        self,        in_channels: int,        channels_7x7: int,        conv_block = None) -> None:        super(InceptionC, self).__init__()        if conv_block is None:            conv_block = BasicConv2d        self.branch1x1 = conv_block(in_channels, 192, kernel_size=1)
        c7 = channels_7x7        self.branch7x7_1 = conv_block(in_channels, c7, kernel_size=1)        self.branch7x7_2 = conv_block(c7, c7, kernel_size=(1, 7), padding=(0


    
, 3))        self.branch7x7_3 = conv_block(c7, 192, kernel_size=(7, 1), padding=(3, 0))
        self.branch7x7dbl_1 = conv_block(in_channels, c7, kernel_size=1)        self.branch7x7dbl_2 = conv_block(c7, c7, kernel_size=(7, 1), padding=(3, 0))        self.branch7x7dbl_3 = conv_block(c7, c7, kernel_size=(1, 7), padding=(0, 3))        self.branch7x7dbl_4 = conv_block(c7, c7, kernel_size=(7, 1), padding=(3, 0))        self.branch7x7dbl_5 = conv_block(c7, 192, kernel_size=(1, 7), padding=(0, 3))
        self.branch_pool = conv_block(in_channels, 192, kernel_size=1)
    def _forward(self, x: Tensor) -> List[Tensor]:        branch1x1 = self.branch1x1(x)
        branch7x7 = self.branch7x7_1(x)        branch7x7 = self.branch7x7_2(branch7x7)        branch7x7 = self.branch7x7_3(branch7x7)
        branch7x7dbl = self.branch7x7dbl_1(x)        branch7x7dbl = self.branch7x7dbl_2(branch7x7dbl)        branch7x7dbl = self.branch7x7dbl_3(branch7x7dbl)        branch7x7dbl = self.branch7x7dbl_4(branch7x7dbl)        branch7x7dbl = self.branch7x7dbl_5(branch7x7dbl)
        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)        branch_pool = self.branch_pool(branch_pool)
        outputs = [branch1x1, branch7x7, branch7x7dbl, branch_pool]        return outputs
    def forward(self, x: Tensor) -> Tensor:        outputs = self._forward(x)        return torch.cat(outputs, 1)

class InceptionD(nn.Module):
    def __init__(        self,        in_channels: int,        conv_block = None) -> None:        super(InceptionD, self).__init__()        if conv_block is None:            conv_block = BasicConv2d        self.branch3x3_1 = conv_block(in_channels, 192, kernel_size=1)        self.branch3x3_2 = conv_block(192


    
, 320, kernel_size=3, stride=2)
        self.branch7x7x3_1 = conv_block(in_channels, 192, kernel_size=1)        self.branch7x7x3_2 = conv_block(192, 192, kernel_size=(1, 7), padding=(0, 3))        self.branch7x7x3_3 = conv_block(192, 192, kernel_size=(7, 1), padding=(3, 0))        self.branch7x7x3_4 = conv_block(192, 192, kernel_size=3, stride=2)
    def _forward(self, x: Tensor) -> List[Tensor]:        branch3x3 = self.branch3x3_1(x)        branch3x3 = self.branch3x3_2(branch3x3)
        branch7x7x3 = self.branch7x7x3_1(x)        branch7x7x3 = self.branch7x7x3_2(branch7x7x3)        branch7x7x3 = self.branch7x7x3_3(branch7x7x3)        branch7x7x3 = self.branch7x7x3_4(branch7x7x3)
        branch_pool = F.max_pool2d(x, kernel_size=3, stride=2)        outputs = [branch3x3, branch7x7x3, branch_pool]        return outputs
    def forward(self, x: Tensor) -> Tensor:        outputs = self._forward(x)        return torch.cat(outputs, 1)

class InceptionE(nn.Module):
    def __init__(        self,        in_channels: int,        conv_block = None) -> None:        super(InceptionE, self).__init__()        if conv_block is None:            conv_block = BasicConv2d        self.branch1x1 = conv_block(in_channels, 320, kernel_size=1)
        self.branch3x3_1 = conv_block(in_channels, 384, kernel_size=1)        self.branch3x3_2a = conv_block(384, 384, kernel_size=(1, 3), padding=(0, 1))        self.branch3x3_2b = conv_block(384, 384, kernel_size=(3, 1), padding=(1, 0))
        self.branch3x3dbl_1 = conv_block(in_channels, 


    
448, kernel_size=1)        self.branch3x3dbl_2 = conv_block(448, 384, kernel_size=3, padding=1)        self.branch3x3dbl_3a = conv_block(384, 384, kernel_size=(1, 3), padding=(0, 1))        self.branch3x3dbl_3b = conv_block(384, 384, kernel_size=(3, 1), padding=(1, 0))
        self.branch_pool = conv_block(in_channels, 192, kernel_size=1)
    def _forward(self, x: Tensor) -> List[Tensor]:        branch1x1 = self.branch1x1(x)
        branch3x3 = self.branch3x3_1(x)        branch3x3 = [            self.branch3x3_2a(branch3x3),            self.branch3x3_2b(branch3x3),        ]        branch3x3 = torch.cat(branch3x3, 1)
        branch3x3dbl = self.branch3x3dbl_1(x)        branch3x3dbl = self.branch3x3dbl_2(branch3x3dbl)        branch3x3dbl = [            self.branch3x3dbl_3a(branch3x3dbl),            self.branch3x3dbl_3b(branch3x3dbl),        ]        branch3x3dbl = torch.cat(branch3x3dbl, 1)
        branch_pool = F.avg_pool2d(x, kernel_size=3, stride=1, padding=1)        branch_pool = self.branch_pool(branch_pool)
        outputs = [branch1x1, branch3x3, branch3x3dbl, branch_pool]        return outputs
    def forward(self, x: Tensor) -> Tensor:        outputs = self._forward(x)        return torch.cat(outputs, 1)

class InceptionAux(nn.Module):
    def __init__(        self,        in_channels: int,        num_classes: int,        conv_block = None) -> None:        super(InceptionAux, self).__init__()        if conv_block is None:            conv_block = BasicConv2d        self.conv0 = conv_block(in_channels, 128, kernel_size=1)        self.conv1 = conv_block(128, 768, kernel_size=5)


    
        self.conv1.stddev = 0.01  # type: ignore[assignment]        self.fc = nn.Linear(768, num_classes)        self.fc.stddev = 0.001  # type: ignore[assignment]
    def forward(self, x: Tensor) -> Tensor:        # N x 768 x 17 x 17        x = F.avg_pool2d(x, kernel_size=5, stride=3)        # N x 768 x 5 x 5        x = self.conv0(x)        # N x 128 x 5 x 5        x = self.conv1(x)        # N x 768 x 1 x 1        # Adaptive average pooling        x = F.adaptive_avg_pool2d(x, (1, 1))        # N x 768 x 1 x 1        x = torch.flatten(x, 1)        # N x 768        x = self.fc(x)        # N x 1000        return x

class BasicConv2d(nn.Module):
    def __init__(        self,        in_channels: int,        out_channels: int,        **kwargs) -> None:        super(BasicConv2d, self).__init__()        self.conv = nn.Conv2d(in_channels, out_channels, bias=False, **kwargs)        self.bn = nn.BatchNorm2d(out_channels, eps=0.001)
    def forward(self, x: Tensor) -> Tensor:        x = self.conv(x)        x = self.bn(x)        return F.relu(x, inplace=True)
inception3 = Inception3()input1 = torch.rand([1, 3, 299, 299])# print(resnet50)output = inception3(input1)print(output.shape)

import torchfrom torch import nnimport torch.nn.functional as Ffrom


    
 torchsummary import summaryclass BasicBlock(nn.Module):    expansion = 1    def __init__(self, in_channels=64, out_channels=64, stride=1, is_down=False):
        super(BasicBlock, self).__init__()        self.conv1_3x3 = nn.Conv2d(in_channels, out_channels, kernel_size=3, stride=stride, padding=1)        self.bn1 = nn.BatchNorm2d(out_channels)        self.relu1 = nn.ReLU(inplace=True)                self.conv2_3x3 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1)        self.bn2 = nn.BatchNorm2d(out_channels)        self.relu2 = nn.ReLU(inplace=True)                self.is_down = is_down        self.downsample_conv = nn.Sequential(            nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=2),            nn.BatchNorm2d(out_channels))
            def forward(self, x):        identity = x        out = self.conv1_3x3(x)        out = self.bn1(out)        out = self.relu1(out)           out = self.conv2_3x3(out)        out = self.bn2(out)
        # 看conv1 有没有下采样 stride=2，利用1x1进行大小对齐        if self.is_down:            identity = self.downsample_conv(x)                out += identity        out = self.relu2(out)                return out    class Bottleneck(nn.Module):    expansion = 4    def __init__(self, in_channels, out_channels=64, stride=1, is_down=False):        super(Bottleneck, self).__init__()        expansion_channels = out_channels*self.expansion        # print(expansion_channels)        self.in_channels = in_channels        self.out_channels = out_channels        self.conv1_1x1 = nn.Conv2d(in_channels, out_channels, kernel_size=1, stride=stride)        self.bn1 = nn.BatchNorm2d(out_channels)        self.relu1 = nn.ReLU(inplace=True)                self.conv2_3x3 = nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1)        self.bn2 = nn.BatchNorm2d(out_channels)        self.relu2 = nn.ReLU(inplace=True)


    
                self.conv3_1x1 = nn.Conv2d(out_channels, expansion_channels, kernel_size=1, stride=1)        self.bn3 = nn.BatchNorm2d(expansion_channels)        self.relu3 = nn.ReLU(inplace=True)                self.is_down = is_down        self.downsample_conv = nn.Sequential(          nn.Conv2d(in_channels, expansion_channels, kernel_size=1, stride=stride),          nn.BatchNorm2d(expansion_channels))            def forward(self, x):        identity = x        out = self.conv1_1x1(x)        out = self.bn1(out)        out = self.relu1(out)                out = self.conv2_3x3(out)        out = self.bn2(out)        out = self.relu2(out)
        out = self.conv3_1x1(out)        out = self.bn3(out)
                # 看conv1 有没有下采样 stride=2，利用1x1进行大小对齐        if self.is_down | (self.in_channels == self.out_channels):            identity = self.downsample_conv(x)                out += identity        out = self.relu3(out)        return outclass ResNet(nn.Module):    def __init__(self, layers, block, num_classes):        super(ResNet, self).__init__()        self.in_channels = 64        self.conv1 = nn.Conv2d(3, self.in_channels, kernel_size=7, stride=2, padding=3,                               bias=False)                self.bn = nn.BatchNorm2d(self.in_channels)        self.relu = nn.ReLU(inplace=True)        self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)                layer_in_channels = [i * block.expansion for i in [64, 128, 256, 512]]        self.layer1 = self._make_layers(block, layers[0],     self.in_channels, 64, stride=1, is_down=False)        self.layer2 = self._make_layers(block, layers[1], layer_in_channels[0], 128, stride=2, is_down=True)        self.layer3 = self._make_layers(block, layers[2], layer_in_channels[1], 256, stride=2, is_down=True)        self.layer4 = self._make_layers(block, layers[3], layer_in_channels[2], 512, stride=2


    
, is_down=True)                self.avgpool = nn.AdaptiveAvgPool2d((1, 1))        self.fc = nn.Linear(layer_in_channels[3], num_classes)        def _make_layers(self, block, blocks, in_channels, out_channels, stride, is_down):        layers = []        layers.append(block(in_channels, out_channels, stride=stride, is_down=is_down))        for i in range(1, blocks):            layers.append(block(out_channels*block.expansion, out_channels, stride=1, is_down=False))                return nn.Sequential(*layers)            def forward(self, x):        print(x.shape)        x = self.conv1(x)        x = self.bn(x)        x = self.relu(x)        print(x.shape)        x = self.maxpool(x)        print(x.shape)
        x = self.layer1(x)        print(x.shape)        x = self.layer2(x)        print(x.shape)        x = self.layer3(x)        print(x.shape)        x = self.layer4(x)        print(x.shape)                                x = self.avgpool(x)        x = torch.flatten(x, 1)        x = self.fc(x)        return x

layers = [3, 4, 6, 3]resnet50 = ResNet(layers, block=Bottleneck, num_classes=1000)
input1 = torch.rand([1, 3, 224, 224])# print(resnet50)output = resnet50(input1)print(output.shape)
layers = [3, 4, 6, 3]resnet34 = ResNet(layers, block=BasicBlock, num_classes=1000)
input1 = torch.rand([1, 3, 224, 224])# print(resnet34)output = resnet34(input1)print(output.shape)

import torchfrom torch import nnimport torch.nn.functional as Ffrom collections import OrderedDictfrom torchsummary import summary
class _DenseLayer(nn.Module):    def


    
 __init__(self, num_input_features, bn_size, growth_rate):        super(_DenseLayer, self).__init__()        self.bn1 = nn.BatchNorm2d(num_input_features)        self.relu1 = nn.ReLU(inplace=True)        self.conv1 = nn.Conv2d(in_channels=num_input_features, out_channels=bn_size*growth_rate, kernel_size=1, stride=1)                self.bn2 = nn.BatchNorm2d(bn_size*growth_rate)        self.relu2 = nn.ReLU(inplace=True)        self.conv2 = nn.Conv2d(in_channels=bn_size*growth_rate, out_channels=growth_rate, kernel_size=3, stride=1, padding=1)            def forward(self, inputs):        concat_inputs = torch.cat(inputs, 1)        x = self.bn1(concat_inputs)        x = self.relu1(x)        x = self.conv1(x)                x = self.bn2(x)        x = self.relu2(x)        new_features = self.conv2(x)                return new_features
class _DenseBlock(nn.ModuleDict):    def __init__(self, num_layers, num_input_features, bn_size, growth_rate):        super(_DenseBlock, self).__init__()        for i in range(num_layers):            layer = _DenseLayer(num_input_features + i * growth_rate, bn_size, growth_rate)            self.add_module('denselayer%d' % (i + 1), layer)                def forward(self, init_features):                features = [init_features]        for name, _layer in self.items():            new_features = _layer(features)            features.append(new_features)                    return torch.cat(features, 1)
class _TransitionLayer(nn.Module):    def __init__(self, num_input_features, num_output_featurns):        super(_TransitionLayer, self).__init__()        self.bn = nn.BatchNorm2d(num_input_features)        self.relu = nn.ReLU(inplace=True)        self.conv = nn.Conv2d(in_channels=num_input_features, out_channels=num_output_featurns, kernel_size=1, padding=1)        self.avgpool = nn.AvgPool2d(kernel_size=2, stride=2)            def forward(self, x):


    
        x = self.bn(x)        x = self.relu(x)        x = self.conv(x)        x = self.avgpool(x)        return x
class DenseNet(nn.Module):    def __init__(self, in_channels, bn_size, growth_rate, block_config, num_classes=1000):        super(DenseNet, self).__init__()                self.features = nn.Sequential(OrderedDict([            ('conv0', nn.Conv2d(3, in_channels, kernel_size=7, stride=2, padding=3,                               bias=False)),            ('norm0', nn.BatchNorm2d(in_channels)),            ('relu0', nn.ReLU(inplace=True)),            ('pool0',  nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),        ]))        num_input_features = in_channels        for i, num_layers in enumerate(block_config):            block = _DenseBlock(num_layers, num_input_features, bn_size, growth_rate)            self.features.add_module('denseblock%d' % (i + 1), block)                        num_input_features = num_input_features + num_layers * growth_rate                        if i != len(block_config) - 1:                transistion = _TransitionLayer(num_input_features, num_input_features // 2)                self.features.add_module('transistion%d' % (i + 1), transistion)                num_input_features = num_input_features // 2                self.features.add_module('norm5', nn.BatchNorm2d(num_input_features))                self.avgpool = nn.AdaptiveAvgPool2d((1, 1))        self.classifier = nn.Linear(num_input_features, num_classes)            def forward(self, x):                print(x.shape)                features = self.features(x)        out = F.relu(features, inplace=True)        out = self.avgpool(out)        out = torch.flatten(out, 1)        out = self.classifier(out)                return outblock_config = [6, 12, 24, 16]growth_rate = 32in_channels = 64bn_size = 4


    
densenet121 = DenseNet(in_channels, bn_size, growth_rate, block_config)
input1 = torch.rand([1, 3, 224, 224])# print(densenet121)output = densenet121(input1)print(output.shape)# torch.Size([1, 1000]

import torchfrom torch import Tensorfrom torch import nnimport torch.nn.functional as Ffrom collections import OrderedDictfrom torchsummary import summaryimport torchvisionimport copyimport mathimport torchfrom torch import nn, Tensorfrom typing import Any, Callable, List, Optional, Sequence
from functools import partial# from torchvision.ops import StochasticDepthdef _make_divisible(v: float, divisor: int, min_value= None) -> int:    """    This function is taken from the original tf repo.    It ensures that all layers have a channel number that is divisible by 8    It can be seen here:    https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet/mobilenet.py    """    if min_value is None:        min_value = divisor    new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)    # Make sure that round down does not go down by more than 10%.    if new_v < 0.9 * v:        new_v += divisor    return new_vclass ConvNormActivation(torch.nn.Sequential):    """    Configurable block used for Convolution-Normalzation-Activation blocks.    Args:        in_channels (int): Number of channels in the input image        out_channels (int): Number of channels produced by the Convolution-Normalzation-Activation block


    
        kernel_size: (int, optional): Size of the convolving kernel. Default: 3        stride (int, optional): Stride of the convolution. Default: 1        padding (int, tuple or str, optional): Padding added to all four sides of the input. Default: None, in wich case it will calculated as ``padding = (kernel_size - 1) // 2 * dilation``        groups (int, optional): Number of blocked connections from input channels to output channels. Default: 1        norm_layer (Callable[..., torch.nn.Module], optional): Norm layer that will be stacked on top of the convolutiuon layer. If ``None`` this layer wont be used. Default: ``torch.nn.BatchNorm2d``        activation_layer (Callable[..., torch.nn.Module], optinal): Activation function which will be stacked on top of the normalization layer (if not None), otherwise on top of the conv layer. If ``None`` this layer wont be used. Default: ``torch.nn.ReLU``        dilation (int): Spacing between kernel elements. Default: 1        inplace (bool): Parameter for the activation layer, which can optionally do the operation in-place. Default ``True``    """
    def __init__(        self,        in_channels: int,        out_channels: int,        kernel_size: int = 3,        stride: int = 1,        padding: [int] = None,        groups: int = 1,        norm_layer = torch.nn.BatchNorm2d,        activation_layer = torch.nn.ReLU,        dilation: int = 1,        inplace: bool = True,) -> None:        if padding is None:            padding = (kernel_size - 1) // 2 * dilation        layers = [            torch.nn.Conv2d(                in_channels,                out_channels,                kernel_size,                stride,                padding,                dilation=dilation,                groups=groups,                bias=norm_layer is None,            )        ]        if norm_layer is not None:            layers.append(norm_layer(out_channels))        if activation_layer is not None:            layers.append(activation_layer(inplace=inplace))        super().__init__(*layers)        self.out_channels = out_channels

class SqueezeExcitation(torch.nn.Module):    """    This block implements the Squeeze-and-Excitation block from https://arxiv.org/abs/1709.01507 (see Fig. 1).    Parameters ``activation``, and ``scale_activation`` correspond to ``delta`` and ``sigma`` in in eq. 3.    Args:        input_channels (int): Number of channels in the input image        squeeze_channels (int): Number of squeeze channels


    
        activation (Callable[..., torch.nn.Module], optional): ``delta`` activation. Default: ``torch.nn.ReLU``        scale_activation (Callable[..., torch.nn.Module]): ``sigma`` activation. Default: ``torch.nn.Sigmoid``    """
    def __init__(        self,        input_channels: int,        squeeze_channels: int,        activation = torch.nn.ReLU,        scale_activation = torch.nn.Sigmoid,) -> None:        super().__init__()        self.avgpool = torch.nn.AdaptiveAvgPool2d(1)        self.fc1 = torch.nn.Conv2d(input_channels, squeeze_channels, 1)        self.fc2 = torch.nn.Conv2d(squeeze_channels, input_channels, 1)        self.activation = activation()        self.scale_activation = scale_activation()
    def _scale(self, input: Tensor) -> Tensor:        scale = self.avgpool(input)        scale = self.fc1(scale)        scale = self.activation(scale)        scale = self.fc2(scale)        return self.scale_activation(scale)
    def forward(self, input: Tensor) -> Tensor:        scale = self._scale(input)        return scale * input
class MBConvConfig:    # Stores information listed at Table 1 of the EfficientNet paper    def __init__(self,                 expand_ratio: float, kernel: int, stride: int,                 input_channels: int, out_channels: int, num_layers: int,                 width_mult: float, depth_mult: float) -> None:        self.expand_ratio = expand_ratio        self.kernel = kernel        self.stride = stride        self.input_channels = self.adjust_channels(input_channels, width_mult)        self.out_channels = self.adjust_channels(out_channels, width_mult)        self.num_layers = self.adjust_depth(num_layers, depth_mult)
    def __repr__(self) -> str:        s = self.__class__.__name__ + '('        s += 'expand_ratio={expand_ratio}'        s += ', kernel={kernel}'        s += ', stride={stride}'


    
        s += ', input_channels={input_channels}'        s += ', out_channels={out_channels}'        s += ', num_layers={num_layers}'        s += ')'        return s.format(**self.__dict__)
    @staticmethod    def adjust_channels(channels: int, width_mult: float, min_value: [int] = None) -> int:        return _make_divisible(channels * width_mult, 8, min_value)
    @staticmethod    def adjust_depth(num_layers: int, depth_mult: float):        return int(math.ceil(num_layers * depth_mult))

class MBConv(nn.Module):    def __init__(self, cnf: MBConvConfig,                  stochastic_depth_prob: float,                  norm_layer,                 se_layer = SqueezeExcitation) -> None:        super().__init__()
        if not (1 <= cnf.stride <= 2):            raise ValueError('illegal stride value')
        self.use_res_connect = cnf.stride == 1 and cnf.input_channels == cnf.out_channels
        layers = []        activation_layer = nn.ReLU6
        # expand        expanded_channels = cnf.adjust_channels(cnf.input_channels, cnf.expand_ratio)        if expanded_channels != cnf.input_channels:            layers.append(ConvNormActivation(cnf.input_channels, expanded_channels, kernel_size=1,                                             norm_layer=norm_layer, activation_layer=activation_layer))
        # depthwise        layers.append(ConvNormActivation(expanded_channels, expanded_channels, kernel_size=cnf.kernel,                                         stride=cnf.stride, groups=expanded_channels,                                         norm_layer=norm_layer, activation_layer=activation_layer))
        # squeeze and excitation        squeeze_channels = max(1, cnf.input_channels // 4)        layers.append(se_layer(expanded_channels, squeeze_channels, activation=partial(nn.ReLU6, inplace=True)))



    
        # project        layers.append(ConvNormActivation(expanded_channels, cnf.out_channels, kernel_size=1, norm_layer=norm_layer,                                         activation_layer=None))
        self.block = nn.Sequential(*layers)#         self.stochastic_depth = StochasticDepth(stochastic_depth_prob, "row")        self.out_channels = cnf.out_channels
    def forward(self, input: Tensor) -> Tensor:        result = self.block(input)#         if self.use_res_connect:#             result = self.stochastic_depth(result)#             result += input        return result

class EfficientNet(nn.Module):    def __init__(            self,            inverted_residual_setting,            dropout: float,            stochastic_depth_prob: float = 0.2,            num_classes: int = 1000,            block = None,            norm_layer = None,            **kwargs) -> None:        """        EfficientNet main class
        Args:            inverted_residual_setting (List[MBConvConfig]): Network structure            dropout (float): The droupout probability            stochastic_depth_prob (float): The stochastic depth probability            num_classes (int): Number of classes            block (Optional[Callable[..., nn.Module]]): Module specifying inverted residual building block for mobilenet            norm_layer (Optional[Callable[..., nn.Module]]): Module specifying the normalization layer to use        """        super().__init__()
        if not inverted_residual_setting:            raise ValueError("The inverted_residual_setting should not be empty")        elif not (isinstance(inverted_residual_setting, Sequence) and                  all([isinstance(s, MBConvConfig) for s in inverted_residual_setting])):            raise TypeError("The inverted_residual_setting should be List[MBConvConfig]")
        if block is None:            block = MBConv



    
        if norm_layer is None:            norm_layer = nn.BatchNorm2d
        layers = []
        # building first layer        firstconv_output_channels = inverted_residual_setting[0].input_channels        layers.append(ConvNormActivation(3, firstconv_output_channels, kernel_size=3, stride=2, norm_layer=norm_layer,                                         activation_layer=nn.ReLU6))
        # building inverted residual blocks        total_stage_blocks = sum([cnf.num_layers for cnf in inverted_residual_setting])        stage_block_id = 0        for cnf in inverted_residual_setting:            stage = []            for _ in range(cnf.num_layers):                # copy to avoid modifications. shallow copy is enough                block_cnf = copy.copy(cnf)
                # overwrite info if not the first conv in the stage                if stage:                    block_cnf.input_channels = block_cnf.out_channels                    block_cnf.stride = 1
                # adjust stochastic depth probability based on the depth of the stage block                sd_prob = stochastic_depth_prob * float(stage_block_id) / total_stage_blocks
                stage.append(block(block_cnf, sd_prob, norm_layer))                stage_block_id += 1
            layers.append(nn.Sequential(*stage))
        # building last several layers        lastconv_input_channels = inverted_residual_setting[-1].out_channels        lastconv_output_channels = 4 * lastconv_input_channels        layers.append(ConvNormActivation(lastconv_input_channels, lastconv_output_channels, kernel_size=1,                                         norm_layer=norm_layer, activation_layer= nn.ReLU6))
        self.features = nn.Sequential(*layers)        self.avgpool = nn.AdaptiveAvgPool2d(1)        self.classifier = nn.Sequential(            nn.Dropout(p=dropout, inplace=True),            nn.Linear(lastconv_output_channels, num_classes),        )
        for m in self.modules():            if isinstance(m, nn.Conv2d):                nn.init.kaiming_normal_(m.weight, mode='fan_out')                if m.bias is not None:                    nn.init.zeros_(m.bias)            elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):


    
                nn.init.ones_(m.weight)                nn.init.zeros_(m.bias)            elif isinstance(m, nn.Linear):                init_range = 1.0 / math.sqrt(m.out_features)                nn.init.uniform_(m.weight, -init_range, init_range)                nn.init.zeros_(m.bias)
    def _forward_impl(self, x: Tensor) -> Tensor:        x = self.features(x)
        x = self.avgpool(x)        x = torch.flatten(x, 1)
        x = self.classifier(x)
        return x
    def forward(self, x: Tensor) -> Tensor:        return self._forward_impl(x)

def _efficientnet_conf(width_mult: float, depth_mult: float, **kwargs):    bneck_conf = partial(MBConvConfig, width_mult=width_mult, depth_mult=depth_mult)    inverted_residual_setting = [        bneck_conf(1, 3, 1, 32, 16, 1),        bneck_conf(6, 3, 2, 16, 24, 2),        bneck_conf(6, 5, 2, 24, 40, 2),        bneck_conf(6, 3, 2, 40, 80, 3),        bneck_conf(6, 5, 1, 80, 112, 3),        bneck_conf(6, 5, 2, 112, 192, 4),        bneck_conf(6, 3, 1, 192, 320, 1),    ]    return inverted_residual_setting

def _efficientnet_model(    arch: str,    inverted_residual_setting,    dropout: float,    **kwargs) -> EfficientNet:    model = EfficientNet(inverted_residual_setting, dropout, **kwargs)    return


    
 model
inverted_residual_setting = _efficientnet_conf(width_mult=1.0, depth_mult=1.0)efficientnet_b0 = _efficientnet_model("efficientnet_b0", inverted_residual_setting, 0.2)
input1 = torch.rand([1, 3, 224, 224])# print(densenet121)output = efficientnet_b0(input1)print(output.shape)
inverted_residual_setting = _efficientnet_conf(width_mult=1.0, depth_mult=1.1)efficientnet_b1 = _efficientnet_model("efficientnet_b1", inverted_residual_setting, 0.2)input1 = torch.rand([1, 3, 224, 224])# print(densenet121)output = efficientnet_b1(input1)print(output.shape)