|
- from timm import create_model
- from timm.optim import create_optimizer
- from timm.scheduler import create_scheduler
-
- from pytorch_xcit import get_args_parser
-
- model = create_model('resnet34')
-
- from types import SimpleNamespace
- args = get_args_parser().parse_args()
-
- print(args.opt)
- optimizer = create_optimizer(args, model)
-
- def get_lr_per_epoch(scheduler, num_epoch):
- lr_per_epoch = []
- for epoch in range(num_epoch):
- lr_per_epoch.append(scheduler.get_epoch_values(epoch))
- return lr_per_epoch
-
- num_epoch = 400
- lr_scheduler, _ = create_scheduler(args, optimizer)
- lr_per_epoch = get_lr_per_epoch(lr_scheduler, num_epoch)
- # print(lr_per_epoch)
-
- import numpy as np
- import proplot as pplt
- fig = pplt.figure(share=0)
- ax = fig.subplot(221)
- ax.plot(np.arange(num_epoch), np.array(lr_per_epoch)[:,0], lw=2)
-
- ax.format(title='Pytorch', suptitle='Cosine', xlabel='Epoch', ylabel='Learning Rate', xlim=(0, num_epoch))
- # pplt.show()
-
- ## Mindspore
-
- from config import parse_args
- from mindcv.scheduler import create_scheduler as ms_create_scheduler
- from mindcv.optim import create_optimizer as ms_create_optimizer
- import mindspore as ms
- from mindspore import nn
- class Net(nn.Cell):
- def __init__(self, num_class=10, num_channel=1):
- super(Net, self).__init__()
- self.conv1 = nn.Conv2d(num_channel, 6, 5, pad_mode='valid')
- self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid')
- self.fc1 = nn.Dense(16*5*5, 120, weight_init='ones')
- self.fc2 = nn.Dense(120, 84, weight_init='ones')
- self.fc3 = nn.Dense(84, num_class, weight_init='ones')
- self.relu = nn.ReLU()
- self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2)
- self.flatten = nn.Flatten()
-
- def construct(self, x):
- x = self.max_pool2d(self.relu(self.conv1(x)))
- x = self.max_pool2d(self.relu(self.conv2(x)))
- x = self.flatten(x)
- x = self.relu(self.fc1(x))
- x = self.relu(self.fc2(x))
- x = self.fc3(x)
- return x
- ms_model = Net()
- args = parse_args()
-
- num_batches = 100
- lr_scheduler = ms_create_scheduler(num_batches,
- scheduler=args.scheduler,
- lr=args.lr,
- min_lr=args.min_lr,
- warmup_epochs=args.warmup_epochs,
- warmup_factor=args.warmup_factor,
- decay_epochs=args.decay_epochs,
- decay_rate=args.decay_rate,
- milestones=args.multi_step_decay_milestones,
- num_epochs=args.epoch_size,
- lr_epoch_stair=args.lr_epoch_stair)
- optimizer = ms_create_optimizer(ms_model.trainable_params(),
- opt=args.opt,
- lr=lr_scheduler,
- weight_decay=args.weight_decay,
- momentum=args.momentum,
- nesterov=args.use_nesterov,
- filter_bias_and_bn=args.filter_bias_and_bn,
- loss_scale=args.loss_scale)
- def ms_get_lr_per_epoch(optim, num_epoch):
- lr_per_epoch = []
- for epoch in range(num_epoch):
- lr_per_epoch.append(optim.get_lr().asnumpy())
- return lr_per_epoch
-
-
- lr_per_epoch = ms_get_lr_per_epoch(optimizer, args.epoch_size * num_batches)
- # print(lr_per_epoch)
- ax = fig.subplot(222)
- ax.plot(np.array(lr_per_epoch), lw=2)
- ax.format(title='mindcv-decay=395,warm=5', xlabel='steps', ylabel='Learning Rate')
-
- lr_scheduler = ms_create_scheduler(num_batches,
- scheduler=args.scheduler,
- lr=args.lr,
- min_lr=args.min_lr,
- warmup_epochs=args.warmup_epochs,
- warmup_factor=args.warmup_factor,
- decay_epochs=args.decay_epochs-200,
- decay_rate=args.decay_rate,
- milestones=args.multi_step_decay_milestones,
- num_epochs=args.epoch_size,
- lr_epoch_stair=args.lr_epoch_stair)
- optimizer = ms_create_optimizer(ms_model.trainable_params(),
- opt=args.opt,
- lr=lr_scheduler,
- weight_decay=args.weight_decay,
- momentum=args.momentum,
- nesterov=args.use_nesterov,
- filter_bias_and_bn=args.filter_bias_and_bn,
- loss_scale=args.loss_scale)
- def ms_get_lr_per_epoch(optim, num_epoch):
- lr_per_epoch = []
- for epoch in range(num_epoch):
- lr_per_epoch.append(optim.get_lr().asnumpy())
- return lr_per_epoch
-
-
- lr_per_epoch = ms_get_lr_per_epoch(optimizer, args.epoch_size * num_batches)
- # print(lr_per_epoch)
- ax = fig.subplot(223)
- ax.plot(np.array(lr_per_epoch), lw=2)
- ax.format(title='mindcv-decay=195,warm=5')
- pplt.show()
|