Verified Commit 1ebdf0d2 authored by Yaoyao Liu's avatar Yaoyao Liu
Browse files

Upload the code

parent e1966a19
# File types
*.pyc
*.npy
*.tar.gz
# Folders
ckpts
results
data
__pycache__
# File
.DS_Store
bashrc
\ No newline at end of file
......@@ -6,7 +6,7 @@
\[[Project Page](https://class-il.mpi-inf.mpg.de/rmm/)\]
### We will release the code before the conference.
### The code is already uploaded. We will add the instruction for the code soon.
### Citation
......
import matplotlib; matplotlib.use('Agg')
from inclearn import parser, train
import matplotlib
import os
from inclearn import parser
from inclearn.train import train
from inclearn.gpu_tools import occupy_memory, set_gpu
matplotlib.use('Agg')
def main():
args = parser.get_parser().parse_args()
args = vars(args)
set_gpu(str(args["device"][0]))
occupy_memory(args["device"][0])
if args["seed_range"] is not None:
args["seed"] = list(range(args["seed_range"][0], args["seed_range"][1] + 1))
print("Seed range", args["seed"])
for _ in train(args):
pass
if __name__ == "__main__":
main()
from . import (
cifar_resnet, densenet, my_resnet, my_resnet2, my_resnet_brn, my_resnet_mcbn, my_resnet_mtl,
resnet, resnet_mtl, ucir_resnet, vgg
)
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import init
class DownsampleA(nn.Module):
def __init__(self, nIn, nOut, stride):
super(DownsampleA, self).__init__()
assert stride == 2
self.avg = nn.AvgPool2d(kernel_size=1, stride=stride)
def forward(self, x):
x = self.avg(x)
return torch.cat((x, x.mul(0)), 1)
class ResNetBasicblock(nn.Module):
expansion = 1
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(ResNetBasicblock, self).__init__()
self.conv_a = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
self.bn_a = nn.BatchNorm2d(planes)
self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
self.bn_b = nn.BatchNorm2d(planes)
self.downsample = downsample
self.featureSize = 64
def forward(self, x):
residual = x
basicblock = self.conv_a(x)
basicblock = self.bn_a(basicblock)
basicblock = F.relu(basicblock, inplace=True)
basicblock = self.conv_b(basicblock)
basicblock = self.bn_b(basicblock)
if self.downsample is not None:
residual = self.downsample(x)
return F.relu(residual + basicblock, inplace=True)
class CifarResNet(nn.Module):
def __init__(self, block, depth, num_classes, channels=3):
super(CifarResNet, self).__init__()
self.featureSize = 64
assert (depth - 2) % 6 == 0, 'depth should be one of 20, 32, 44, 56, 110'
layer_blocks = (depth - 2) // 6
self.num_classes = num_classes
self.conv_1_3x3 = nn.Conv2d(channels, 16, kernel_size=3, stride=1, padding=1, bias=False)
self.bn_1 = nn.BatchNorm2d(16)
self.inplanes = 16
self.stage_1 = self._make_layer(block, 16, layer_blocks, 1)
self.stage_2 = self._make_layer(block, 32, layer_blocks, 2)
self.stage_3 = self._make_layer(block, 64, layer_blocks, 2)
self.avgpool = nn.AvgPool2d(8)
self.out_dim = 64 * block.expansion
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
init.kaiming_normal(m.weight)
m.bias.data.zero_()
def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = DownsampleA(self.inplanes, planes * block.expansion, stride)
layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
def forward(self, x, feature=False, T=1, labels=False, scale=None, keep=None):
x = self.conv_1_3x3(x)
x = F.relu(self.bn_1(x), inplace=True)
x = self.stage_1(x)
x = self.stage_2(x)
x = self.stage_3(x)
x = self.avgpool(x)
x = x.view(x.size(0), -1)
return x
def forwardFeature(self, x):
pass
def resnet20(num_classes=10):
model = CifarResNet(ResNetBasicblock, 20, num_classes)
return model
def resnet10mnist(num_classes=10):
model = CifarResNet(ResNetBasicblock, 10, num_classes, 1)
return model
def resnet20mnist(num_classes=10):
model = CifarResNet(ResNetBasicblock, 20, num_classes, 1)
return model
def resnet32mnist(num_classes=10, channels=1):
model = CifarResNet(ResNetBasicblock, 32, num_classes, channels)
return model
def resnet32(num_classes=10):
model = CifarResNet(ResNetBasicblock, 32, num_classes)
return model
def resnet44(num_classes=10):
model = CifarResNet(ResNetBasicblock, 44, num_classes)
return model
def resnet56(num_classes=10):
model = CifarResNet(ResNetBasicblock, 56, num_classes)
return model
def resnet110(num_classes=10):
model = CifarResNet(ResNetBasicblock, 110, num_classes)
return model
import re
from collections import OrderedDict
import torch
import torch.nn as nn
import torch.nn.functional as F
__all__ = ['DenseNet', 'densenet121', 'densenet169', 'densenet201', 'densenet161']
model_urls = {
'densenet121': 'https://download.pytorch.org/models/densenet121-a639ec97.pth',
'densenet169': 'https://download.pytorch.org/models/densenet169-b2777c0a.pth',
'densenet201': 'https://download.pytorch.org/models/densenet201-c1103571.pth',
'densenet161': 'https://download.pytorch.org/models/densenet161-8d451a50.pth',
}
class _DenseLayer(nn.Sequential):
def __init__(self, num_input_features, growth_rate, bn_size, drop_rate):
super(_DenseLayer, self).__init__()
self.add_module('norm1', nn.BatchNorm2d(num_input_features)),
self.add_module('relu1', nn.ReLU(inplace=True)),
self.add_module('conv1', nn.Conv2d(num_input_features, bn_size *
growth_rate, kernel_size=1, stride=1,
bias=False)),
self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)),
self.add_module('relu2', nn.ReLU(inplace=True)),
self.add_module('conv2', nn.Conv2d(bn_size * growth_rate, growth_rate,
kernel_size=3, stride=1, padding=1,
bias=False)),
self.drop_rate = drop_rate
def forward(self, x):
new_features = super(_DenseLayer, self).forward(x)
if self.drop_rate > 0:
new_features = F.dropout(new_features, p=self.drop_rate,
training=self.training)
return torch.cat([x, new_features], 1)
class _DenseBlock(nn.Sequential):
def __init__(self, num_layers, num_input_features, bn_size, growth_rate, drop_rate):
super(_DenseBlock, self).__init__()
for i in range(num_layers):
layer = _DenseLayer(num_input_features + i * growth_rate, growth_rate,
bn_size, drop_rate)
self.add_module('denselayer%d' % (i + 1), layer)
class _Transition(nn.Sequential):
def __init__(self, num_input_features, num_output_features):
super(_Transition, self).__init__()
self.add_module('norm', nn.BatchNorm2d(num_input_features))
self.add_module('relu', nn.ReLU(inplace=True))
self.add_module('conv', nn.Conv2d(num_input_features, num_output_features,
kernel_size=1, stride=1, bias=False))
self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2))
class DenseNet(nn.Module):
def __init__(self, growth_rate=32, block_config=(6, 12, 24, 16),
num_init_features=64, bn_size=4, drop_rate=0, num_classes=1000,
**kwargs):
super(DenseNet, self).__init__()
self.features = nn.Sequential(OrderedDict([
('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2,
padding=3, bias=False)),
('norm0', nn.BatchNorm2d(num_init_features)),
('relu0', nn.ReLU(inplace=True)),
('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),
]))
num_features = num_init_features
for i, num_layers in enumerate(block_config):
block = _DenseBlock(num_layers=num_layers, num_input_features=num_features,
bn_size=bn_size, growth_rate=growth_rate,
drop_rate=drop_rate)
self.features.add_module('denseblock%d' % (i + 1), block)
num_features = num_features + num_layers * growth_rate
if i != len(block_config) - 1:
trans = _Transition(num_input_features=num_features,
num_output_features=num_features // 2)
self.features.add_module('transition%d' % (i + 1), trans)
num_features = num_features // 2
self.features.add_module('norm5', nn.BatchNorm2d(num_features))
self.out_dim = num_features
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight)
elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.Linear):
nn.init.constant_(m.bias, 0)
def forward(self, x):
features = self.features(x)
out = F.relu(features, inplace=True)
out = F.adaptive_avg_pool2d(out, (1, 1)).view(features.size(0), -1)
return out
def _load_state_dict(model, model_url, progress):
pass
def _densenet(arch, growth_rate, block_config, num_init_features, pretrained, progress,
**kwargs):
model = DenseNet(growth_rate, block_config, num_init_features, **kwargs)
if pretrained:
_load_state_dict(model, model_urls[arch], progress)
return model
def densenet121(pretrained=False, progress=True, **kwargs):
return _densenet('densenet121', 32, (6, 12, 24, 16), 64, pretrained, progress,
**kwargs)
def densenet161(pretrained=False, progress=True, **kwargs):
return _densenet('densenet161', 48, (6, 12, 36, 24), 96, pretrained, progress,
**kwargs)
def densenet169(pretrained=False, progress=True, **kwargs):
return _densenet('densenet169', 32, (6, 12, 32, 32), 64, pretrained, progress,
**kwargs)
def densenet201(pretrained=False, progress=True, **kwargs):
return _densenet('densenet201', 32, (6, 12, 48, 32), 64, pretrained, progress,
**kwargs)
import logging
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import init
from inclearn.lib import pooling
logger = logging.getLogger(__name__)
class DownsampleStride(nn.Module):
def __init__(self, n=2):
super(DownsampleStride, self).__init__()
self._n = n
def forward(self, x):
return x[..., ::2, ::2]
class DownsampleConv(nn.Module):
def __init__(self, inplanes, planes):
super().__init__()
self.conv = nn.Sequential(
nn.Conv2d(inplanes, planes, stride=2, kernel_size=1, bias=False),
nn.BatchNorm2d(planes),
)
def forward(self, x):
return self.conv(x)
class ResidualBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, increase_dim=False, last_relu=False, downsampling="stride"):
super(ResidualBlock, self).__init__()
self.increase_dim = increase_dim
if increase_dim:
first_stride = 2
planes = inplanes * 2
else:
first_stride = 1
planes = inplanes
self.conv_a = nn.Conv2d(
inplanes, planes, kernel_size=3, stride=first_stride, padding=1, bias=False
)
self.bn_a = nn.BatchNorm2d(planes)
self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
self.bn_b = nn.BatchNorm2d(planes)
if increase_dim:
if downsampling == "stride":
self.downsampler = DownsampleStride()
self._need_pad = True
else:
self.downsampler = DownsampleConv(inplanes, planes)
self._need_pad = False
self.last_relu = last_relu
@staticmethod
def pad(x):
return torch.cat((x, x.mul(0)), 1)
def forward(self, x):
y = self.conv_a(x)
y = self.bn_a(y)
y = F.relu(y, inplace=True)
y = self.conv_b(y)
y = self.bn_b(y)
if self.increase_dim:
x = self.downsampler(x)
if self._need_pad:
x = self.pad(x)
y = x + y
if self.last_relu:
y = F.relu(y, inplace=True)
return y
class PreActResidualBlock(nn.Module):
expansion = 1
def __init__(self, inplanes, increase_dim=False, last_relu=False):
super().__init__()
self.increase_dim = increase_dim
if increase_dim:
first_stride = 2
planes = inplanes * 2
else:
first_stride = 1
planes = inplanes
self.bn_a = nn.BatchNorm2d(inplanes)
self.conv_a = nn.Conv2d(
inplanes, planes, kernel_size=3, stride=first_stride, padding=1, bias=False
)
self.bn_b = nn.BatchNorm2d(planes)
self.conv_b = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
if increase_dim:
self.downsample = DownsampleStride()
self.pad = lambda x: torch.cat((x, x.mul(0)), 1)
self.last_relu = last_relu
def forward(self, x):
y = self.bn_a(x)
y = F.relu(y, inplace=True)
y = self.conv_a(x)
y = self.bn_b(y)
y = F.relu(y, inplace=True)
y = self.conv_b(y)
if self.increase_dim:
x = self.downsample(x)
x = self.pad(x)
y = x + y
if self.last_relu:
y = F.relu(y, inplace=True)
return y
class Stage(nn.Module):
def __init__(self, blocks, block_relu=False):
super().__init__()
self.blocks = nn.ModuleList(blocks)
self.block_relu = block_relu
def forward(self, x):
intermediary_features = []
for b in self.blocks:
x = b(x)
intermediary_features.append(x)
if self.block_relu:
x = F.relu(x)
return intermediary_features, x
class CifarResNet(nn.Module):
def __init__(
self,
n=5,
nf=16,
channels=3,
preact=False,
zero_residual=True,
pooling_config={"type": "avg"},
downsampling="stride",
final_layer=False,
all_attentions=False,
last_relu=False,
**kwargs
):
if kwargs:
raise ValueError("Unused kwargs: {}.".format(kwargs))
self.all_attentions = all_attentions
logger.info("Downsampling type {}".format(downsampling))
self._downsampling_type = downsampling
self.last_relu = last_relu
Block = ResidualBlock if not preact else PreActResidualBlock
super(CifarResNet, self).__init__()
self.conv_1_3x3 = nn.Conv2d(channels, nf, kernel_size=3, stride=1, padding=1, bias=False)
self.bn_1 = nn.BatchNorm2d(nf)
self.stage_1 = self._make_layer(Block, nf, increase_dim=False, n=n)
self.stage_2 = self._make_layer(Block, nf, increase_dim=True, n=n - 1)
self.stage_3 = self._make_layer(Block, 2 * nf, increase_dim=True, n=n - 2)
self.stage_4 = Block(
4 * nf, increase_dim=False, last_relu=False, downsampling=self._downsampling_type
)
if pooling_config["type"] == "avg":
self.pool = nn.AdaptiveAvgPool2d((1, 1))
elif pooling_config["type"] == "weldon":
self.pool = pooling.WeldonPool2d(**pooling_config)
else:
raise ValueError("Unknown pooling type {}.".format(pooling_config["type"]))
self.out_dim = 4 * nf
if final_layer in (True, "conv"):
self.final_layer = nn.Conv2d(self.out_dim, self.out_dim, kernel_size=1, bias=False)
elif isinstance(final_layer, dict):
if final_layer["type"] == "one_layer":
self.final_layer = nn.Sequential(
nn.BatchNorm1d(self.out_dim), nn.ReLU(inplace=True),
nn.Linear(self.out_dim, int(self.out_dim * final_layer["reduction_factor"]))
)
self.out_dim = int(self.out_dim * final_layer["reduction_factor"])
elif final_layer["type"] == "two_layers":
self.final_layer = nn.Sequential(
nn.BatchNorm1d(self.out_dim), nn.ReLU(inplace=True),
nn.Linear(self.out_dim, self.out_dim), nn.BatchNorm1d(self.out_dim),
nn.ReLU(inplace=True),
nn.Linear(self.out_dim, int(self.out_dim * final_layer["reduction_factor"]))
)
self.out_dim = int(self.out_dim * final_layer["reduction_factor"])
else:
raise ValueError("Unknown final layer type {}.".format(final_layer["type"]))
else:
self.final_layer = None
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.Linear):
nn.init.kaiming_normal_(m.weight, mode="fan_out", nonlinearity="relu")
if zero_residual:
for m in self.modules():
if isinstance(m, ResidualBlock):
nn.init.constant_(m.bn_b.weight, 0)
def _make_layer(self, Block, planes, increase_dim=False, n=None):
layers = []
if increase_dim:
layers.append(
Block(