Skip to content

M
Model-Transfer-Adaptability
Commit 625093df by Zhihong Ma
Options

feat: Second Idea for Model Transfer Safety

parent f627ea54
Showing with 3104 additions and 0 deletions
mzh/new_mzh/ALL_2/cfg.py 0 → 100644
# conv: 'C',''/'B'/'BRL'/'BRS',qi,in_ch,out_ch,kernel_size,stirde,padding,bias
# relu: 'RL'
# relu6: 'RS'
# inception: 'Inc'
# maxpool: 'MP',kernel_size,stride,padding
# adaptiveavgpool: 'AAP',output_size
# view: 'VW':
# dafault: x = x.view(x.size(0),-1)
# dropout: 'D'
# MakeLayer: 'ML','BBLK'/'BTNK'/'IRES', ml_idx, blocks
# softmax: 'SM'
# class 100
ResNet_18_cfg_table = [
['C','BRL',True,3,64,3,1,1,False],
['ML','BBLK',0,2],
['ML','BBLK',1,2],
['ML','BBLK',2,2],
['ML','BBLK',3,2],
['AAP',1],
['VW'],
['FC',512,100,True],
]
ResNet_50_cfg_table = [
['C','BRL',True,3,64,3,1,1,False],
['ML','BTNK',0,3],
['ML','BTNK',1,4],
['ML','BTNK',2,6],
['ML','BTNK',3,3],
['AAP',1],
['VW'],
['FC',2048,100,True]
]
ResNet_152_cfg_table = [
['C','BRL',True,3,64,3,1,1,False],
['ML','BTNK',0,3],
['ML','BTNK',1,8],
['ML','BTNK',2,36],
['ML','BTNK',3,3],
['AAP',1],
['VW'],
['FC',2048,100,True]
]
MobileNetV2_cfg_table = [
['C','BRS',True,3,32,1,1,1,True],
['ML','IRES',0,1],
['ML','IRES',1,2],
['ML','IRES',2,3],
['ML','IRES',3,4],
['ML','IRES',4,3],
['ML','IRES',5,3],
['ML','IRES',6,1],
['C','BRS',False,320,1280,1,1,0,True],
['AAP',1],
['VW'],
['FC',1280,100,True]
]
AlexNet_cfg_table = [
['C','',True,3,32,3,1,1,True],
['RL'],
['MP',2,2,0],
['C','',False,32,64,3,1,1,True],
['RL'],
['MP',2,2,0],
['C','',False,64,128,3,1,1,True],
['RL'],
['C','',False,128,256,3,1,1,True],
['RL'],
['C','',False,256,256,3,1,1,True],
['RL'],
['MP',3,2,0],
['VW'],
['D',0.5],
['FC',2304,1024,True],
['RL'],
['D',0.5],
['FC',1024,512,True],
['RL'],
['FC',512,100,True]
]
AlexNet_BN_cfg_table = [
['C','BRL',True,3,32,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,32,64,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,64,128,3,1,1,True],
['C','BRL',False,128,256,3,1,1,True],
['C','BRL',False,256,256,3,1,1,True],
['MP',3,2,0],
['VW'],
['D',0.5],
['FC',2304,1024,True],
['RL'],
['D',0.5],
['FC',1024,512,True],
['RL'],
['FC',512,100,True]
]
VGG_16_cfg_table = [
['C','BRL',True,3,64,3,1,1,True],
['C','BRL',False,64,64,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,64,128,3,1,1,True],
['C','BRL',False,128,128,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,128,256,3,1,1,True],
['C','BRL',False,256,256,3,1,1,True],
['C','BRL',False,256,256,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,256,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,512,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['MP',2,2,0],
['VW'],
['FC',512,4096,True],
['RL'],
['D',0.5],
['FC',4096,4096,True],
['RL'],
['D',0.5],
['FC',4096,100,True]
]
VGG_19_cfg_table = [
['C','BRL',True,3,64,3,1,1,True],
['C','BRL',False,64,64,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,64,128,3,1,1,True],
['C','BRL',False,128,128,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,128,256,3,1,1,True],
['C','BRL',False,256,256,3,1,1,True],
['C','BRL',False,256,256,3,1,1,True],
['C','BRL',False,256,256,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,256,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['MP',2,2,0],
['C','BRL',False,512,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['C','BRL',False,512,512,3,1,1,True],
['MP',2,2,0],
['VW'],
['FC',512,4096,True],
['RL'],
['D',0.5],
['FC',4096,4096,True],
['RL'],
['D',0.5],
['FC',4096,100,True]
]
Inception_BN_cfg_table = [
['C','',True,3,64,3,1,1,True],
['RL'],
['C','',False,64,64,3,1,1,True],
['RL'],
['Inc',0],
['Inc',1],
['MP',3,2,1],
['Inc',2],
['Inc',3],
['Inc',4],
['Inc',5],
['Inc',6],
['MP',3,2,1],
['Inc',7],
['Inc',8],
['AAP',1],
['C','',False,1024,100,1,1,0,True],
['VW']
]
model_cfg_table = {
'AlexNet' : AlexNet_cfg_table,
'AlexNet_BN' : AlexNet_BN_cfg_table,
'VGG_16' : VGG_16_cfg_table,
'VGG_19' : VGG_19_cfg_table,
'Inception_BN' : Inception_BN_cfg_table,
'ResNet_18' : ResNet_18_cfg_table,
'ResNet_50' : ResNet_50_cfg_table,
'ResNet_152' : ResNet_152_cfg_table,
'MobileNetV2' : MobileNetV2_cfg_table
}
#每行对应一个Inc结构(channel)的参数表
inc_ch_table=[
[ 64, 64, 96,128, 16, 32, 32],#3a
[256,128,128,192, 32, 96, 64],#3b
[480,192, 96,208, 16, 48, 64],#4a
[512,160,112,224, 24, 64, 64],#4b
[512,128,128,256, 24, 64, 64],#4c
[512,112,144,288, 32, 64, 64],#4d
[528,256,160,320, 32,128,128],#4e
[832,256,160,320, 32,128,128],#5a
[832,384,192,384, 48,128,128] #5b
]
# br0,br1,br2,br3 <- br1x1,br3x3,br5x5,brM
# 每个子数组对应Inc结构中一个分支的结构，均默认含'BRL'参数，bias为False
# Conv层第2、3个参数是对应Inc结构(即ch_table中的一行)中的索引
# 由于每个Inc结构操作一致，只有权重不同，使用索引而非具体值，方便复用
# 各分支后还有Concat操作，由于只有唯一结构，未特殊说明
# conv: 'C', ('BRL' default), in_ch_idex, out_ch_idx, kernel_size, stride, padding, (bias: True default)
# maxpool: 'MP', kernel_size, stride, padding
# relu: 'RL'
inc_cfg_table = [
[
['C',0,1,1,1,0]
],
[
['C',0,2,1,1,0],
['C',2,3,3,1,1]
],
[
['C',0,4,1,1,0],
['C',4,5,5,1,2]
],
[
['MP',3,1,1],
['RL'],
['C',0,6,1,1,0]
]
]
# ml_cfg_table = []
#BasicBlock
#value: downsample,inplanes,planes,planes*expansion,stride,1(dafault stride and group)
bblk_ch_table = [
[False, 64, 64, 64,1,1], #layer1,first
[False, 64, 64, 64,1,1], # other
[True, 64,128,128,2,1], #layer2
[False,128,128,128,1,1],
[True, 128,256,256,2,1], #layer3
[False,256,256,256,1,1],
[True, 256,512,512,2,1], #layer4
[False,512,512,512,1,1]
]
#conv: 'C','B'/'BRL'/'BRS', in_ch_idx, out_ch_idx, kernel_sz, stride_idx, padding, groups_idx,bias
#add: 'AD', unconditonal. unconditonal为true或flag为true时将outs中两元素相加
bblk_cfg_table = [
[
['C','BRL',1,2,3,4,1,5,False],
['C','B' ,2,3,3,5,1,5,False],
],
# downsample, 仅当downsample传入为True时使用
[
['C','B' ,1,3,1,4,0,5,False]
],
# 分支交汇后动作
[
['AD',True],
['RL']
]
]
#BottleNeck
#value: downsample,inplanes,planes,planes*expansion,stride,1(dafault stride and group)
btnk_ch_table = [
[True, 64, 64, 256,1,1], #layer1,first
[False, 256, 64, 256,1,1], # other
[True, 256,128, 512,2,1], #layer2
[False, 512,128, 512,1,1],
[True, 512,256,1024,2,1], #layer3
[False,1024,256,1024,1,1],
[True, 1024,512,2048,2,1], #layer4
[False,2048,512,2048,1,1]
]
#conv: 'C','B'/'BRL'/'BRS', in_ch_idx, out_ch_idx, kernel_sz, stride_idx, padding, groups_idx (bias: True default)
#add: 'AD', unconditonal. unconditonal为true或flag为true时将outs中两元素相加
btnk_cfg_table = [
[
['C','BRL',1,2,1,5,0,5,False],
['C','BRL',2,2,3,4,1,5,False],
['C','B' ,2,3,1,5,0,5,False]
],
# downsample, 仅当downsample传入为True时使用
[
['C','B' ,1,3,1,4,0,5,False]
],
# 分支交汇后动作
[
['AD',True],
['RL']
]
]
#InvertedResidual/LinearBottleNeck
#value: identity_flag, in_ch, out_ch, in_ch*expand_ratio, stride, 1(dafault stride and group)
ires_ch_table = [
[False, 32, 16, 32,1,1], #layer1,first
[ True, 16, 16, 16,1,1], # other
[False, 16, 24, 96,2,1], #layer2
[ True, 24, 24, 144,1,1],
[False, 24, 32, 144,2,1], #layer3
[ True, 32, 32, 192,1,1],
[False, 32, 64, 192,2,1], #layer4
[ True, 64, 64, 384,1,1],
[False, 64, 96, 384,1,1], #layer5
[ True, 96, 96, 576,1,1],
[False, 96,160, 576,1,1], #layer6
[ True,160,160, 960,1,1],
[False,160,320, 960,1,1], #layer6
[ True,320,320,1920,1,1]
]
#conv: 'C','B'/'BRL'/'BRS', in_ch_idx, out_ch_idx, kernel_sz, stride_idx, padding, groups_idx (bias: True default)
#add: 'AD', unconditonal. unconditonal为true或flag为true时将outs中两元素相加
ires_cfg_table = [
[
['C','BRS',1,3,1,5,0,5,True],
['C','BRS',3,3,3,4,1,3,True],
['C','B' ,3,2,1,5,0,5,True]
],
# identity_br empty
[
],
# 分支汇合后操作
[
['AD',False] #有条件的相加
]
]
\ No newline at end of file
mzh/new_mzh/ALL_2/dataloader.py 0 → 100644
import os
import torch
import torchvision.datasets as dsets
import torchvision.transforms as transforms
class DataLoader(object):
"""
data loader for CV data sets
"""
def __init__(self, dataset, batch_size):
"""
create data loader for specific data set
:params n_treads: number of threads to load data, default: 4
:params data_path_root: root path to data set, default: /lustre/datasets/
"""
self.dataset = dataset
self.batch_size = batch_size
self.n_threads = 4 #num_workers
self.data_path_root = '/lustre/datasets/'
if self.dataset in ["cifar100","cifar10"]:
self.train_loader, self.val_loader, self.test_loader = self.cifar(
dataset=self.dataset)
else:
assert False, "invalid data set"
def getloader(self):
"""d
get train_loader and test_loader
"""
return self.train_loader, self.val_loader, self.test_loader
def cifar(self, dataset):
"""
dataset: cifar
"""
if dataset == "cifar10":
norm_mean = [0.49139968, 0.48215827, 0.44653124]
norm_std = [0.24703233, 0.24348505, 0.26158768]
elif dataset == "cifar100":
norm_mean = [0.50705882, 0.48666667, 0.44078431]
norm_std = [0.26745098, 0.25568627, 0.27607843]
else:
assert False, "Invalid cifar dataset"
train_transfrom = transforms.Compose([
transforms.RandomCrop(32, padding=2),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std)
])
eval_transfrom = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(norm_mean, norm_std)
])
if self.dataset == "cifar10":
data_path = self.data_path_root + 'CIFAR10'
alltrainset = dsets.CIFAR10(root=data_path,train=True,download=False,
transform=train_transfrom)
testset = dsets.CIFAR10(data_path, train=False, download=False,
transform=eval_transfrom)
elif self.dataset == "cifar100":
data_path = self.data_path_root + 'CIFAR100'
alltrainset = dsets.CIFAR100(root=data_path,train=True,download=False,
transform=train_transfrom)
testset = dsets.CIFAR100(data_path, train=False, download=False,
transform=eval_transfrom)
else:
assert False, "invalid data set"
train_size = (int)(0.8 * len(alltrainset))
val_size = (int)(0.2 * len(alltrainset))
train_idx, val_idx = torch.utils.data.random_split(range(train_size+val_size),[train_size,val_size])
trainset = torch.utils.data.Subset(alltrainset,train_idx)
valset = torch.utils.data.Subset(alltrainset,val_idx)
train_loader = torch.utils.data.DataLoader(
trainset,
batch_size=self.batch_size, shuffle=True, num_workers=self.n_threads, pin_memory=True
)
val_loader = torch.utils.data.DataLoader(
valset,
batch_size=self.batch_size, shuffle=False, num_workers=self.n_threads, pin_memory=True
)
test_loader = torch.utils.data.DataLoader(
testset,
batch_size=self.batch_size, shuffle=False, num_workers=self.n_threads, pin_memory=True
)
return train_loader, val_loader, test_loader
mzh/new_mzh/ALL_2/div.py 0 → 100644
import openpyxl
from mia_utils import *
import module
import gol
import argparse
import numpy as np
import torch
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='DIV_TrajectoryMIA')
parser.add_argument('--model', type=str, default='resnet18', help=['AlexNet','AlexNet_BN','VGG_16','VGG_19','Inception_BN','ResNet_18','ResNet_50','ResNet_152','MobileNetV2'])
parser.add_argument('--data', type=str, default='cifar10', help=['cinic10', 'cifar10', 'cifar100'])
args = parser.parse_args()
data_path = f'mia_ckpt/0/target/{args.data}_{args.model}/trajectory_test_data.npy'
# 统一计算所有的js
gol._init()
quant_type_list = ['INT','POT']
filename =f'{args.model}_mia_result.xlsx'
workbook = openpyxl.load_workbook(filename)
worksheet = workbook[args.data]
for quant_type in quant_type_list:
num_bit_list = numbit_list(quant_type)
for num_bits in num_bit_list:
e_bit_list = ebit_list(quant_type,num_bits)
for e_bits in e_bit_list:
if quant_type == 'FLOAT':
title = '%s_%d_E%d' % (quant_type, num_bits, e_bits)
else:
title = '%s_%d' % (quant_type, num_bits)
model_name_ptq = f'{args.data}_{args.model}_{title}'
p_data_path = f'mia_ckpt/0/target/{model_name_ptq}/trajectory_test_data.npy'
dataSet = np.load(data_path, allow_pickle=True).item()
p_dataSet = np.load(p_data_path, allow_pickle=True).item()
data = torch.from_numpy(np.array(dataSet['model_trajectory'], dtype='f')).transpose(0,1)
p_data = torch.from_numpy(np.array(p_dataSet['model_trajectory'], dtype='f')).transpose(0,1)
div = module.js_div(data,p_data)
div = div.item()
if div<0:
div = 0
print(f"js div of {model_name_ptq}: {div}")
idx = GlobalVariables.title_list.index(title)
idx += 4
worksheet.cell(row=idx,column=2,value=div)
workbook.save(filename)
mzh/new_mzh/ALL_2/extract_ratio.py 0 → 100644
import sys
import os
# 从get_param.py输出重定向文件val.txt中提取参数量和计算量
def extract_ratio(model_name,dataset):
fr = open('param_flops/'+dataset+'/'+model_name+'.txt','r')
lines = fr.readlines()
#跳过warning
for i in range(len(lines)):
if 'Model' in lines[i]:
head = i+1
break
Mac = lines[head].split('Mac,')[0].split(',')[-1]
if 'M' in Mac:
Mac = Mac.split('M')[0]
Mac = float(Mac)
elif 'G' in Mac:
Mac = Mac.split('G')[0]
Mac = float(Mac)
Mac *= 1024
Param = lines[head].split(',')[0]
if 'M' in Param:
Param = Param.split('M')[0]
Param = float(Param)
elif 'k' in Param:
Param = Param.split('k')[0]
Param = float(Param)
Param /= 1024
layer = []
par_ratio = []
flop_ratio = []
for line in lines:
if '(' in line and ')' in line:
layer.append(line.split(')')[0].split('(')[1])
r1 = line.split('%')[0].split(',')[-1]
r1 = float(r1)
par_ratio.append(r1)
r2 = line.split('%')[-2].split(',')[-1]
r2 = float(r2)
flop_ratio.append(r2)
return Mac, Param, layer, par_ratio, flop_ratio
\ No newline at end of file
mzh/new_mzh/ALL_2/function.py 0 → 100644
from torch.autograd import Function
class FakeQuantize(Function):
@staticmethod
def forward(ctx, x, qparam):
x = qparam.quantize_tensor(x)
x = qparam.dequantize_tensor(x)
return x
@staticmethod
def backward(ctx, grad_output):
return grad_output, None
\ No newline at end of file
mzh/new_mzh/ALL_2/gen_one.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#SBATCH --constraint="Ampere|RTX8000"
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
# module load cuda-cudnn/10.2-7.6.5
# module load cuda-cudnn/11.2-8.2.1
module load cuda-cudnn/11.1-8.2.1
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
if [ $Dataset = 'cifar10' ]; then
Label=2
elif [ $Dataset = 'cifar100' ]; then
Label=10
else
echo "Invalid Dataset $Dataset"
exit
fi
if [ $Quant = 'True' ]; then
python gen_one.py --model $Model --dataset $Dataset --quant --multi_label_prob 0.4 --multi_label_num $Label
else
python gen_one.py --model $Model --dataset $Dataset --multi_label_prob 0.4 --multi_label_num $Label
fi
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/gen_options.py 0 → 100644
import os
import os.path as osp
class GenOption(object):
def __init__(self, args):
self.model = args.model
self.dataset = args.dataset
self.batchSize = 128
self.quant = args.quant
if self.dataset == "cifar10":
self.nClasses = 10
elif self.dataset == "cifar100":
self.nClasses = 100
else:
assert False, "invalid dataset"
# ----------Generator options ---------------------------------------------
# self.nEpochs = 100
self.nEpochs = 40
#每个epoch训练多少轮，和batchsize无关
self.iters = 200
#冻结embedding层权重
self.freeze = args.freeze
self.randemb = args.randemb
# 如果不为randomemb，需要根据weight_t调整
self.latent_dim = 64
# 针对imagenet等数据集需要调整
self.img_size = 32
self.channels = 3
self.lr_G = 0.001
# self.milestones_G = [40,60,80]
self.milestones_G = [20,30]
self.gamma_G = 0.1
self.b1 = 0.5
self.b2 = 0.999
# ----------More option ---------------------------------------------
self.multi_label_prob = args.multi_label_prob
self.multi_label_num = args.multi_label_num
self.no_DM = args.no_DM
self.noise_scale = args.noise_scale
self.intermediate_dim = 100
# if self.network == "resnet20":
# self.intermediate_dim = 64
def set(self,quant_type=None,num_bits=None,e_bits=None):
if self.quant:
self.quant_type = quant_type
self.num_bits = num_bits
self.e_bits = e_bits
if quant_type == 'FLOAT':
title = '%s_%d_E%d' % (quant_type, num_bits, e_bits)
else:
title = '%s_%d' % (quant_type, num_bits)
self.teacher_file = 'ckpt_quant/'+self.dataset+'/'+self.model+'/'+title+'.pt'
gen_path = 'ckpt_quant_gen/'+self.dataset+'/'+self.model
self.gen_file = gen_path + '/' + title + '.pt'
else:
self.teacher_file = 'ckpt_full/'+self.dataset+'/'+self.model+'.pt'
gen_path = 'ckpt_full_gen/'+self.dataset
self.gen_file = gen_path +'/'+ self.model+'.pt'
if not osp.exists(self.teacher_file):
assert False, "Empty teacher file"
if not osp.exists(gen_path):
os.makedirs(gen_path)
mzh/new_mzh/ALL_2/generator.py 0 → 100644
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn import init
class Generator(nn.Module):
def __init__(self, options=None, teacher_weight=None, freeze=True):
super(Generator, self).__init__()
self.settings = options
# 注意这里有embedding层，两个分别是词典大小和向量长度
# 用于将标签映射为向量
if teacher_weight==None:
self.label_emb = nn.Embedding(self.settings.nClasses, self.settings.latent_dim)
else:
#当randemb为False时，要求latentdim与输出层输入通道一致
self.label_emb = nn.Embedding.from_pretrained(teacher_weight, freeze=freeze)
self.embed_normalizer = nn.BatchNorm1d(self.label_emb.weight.T.shape,affine=False,track_running_stats=False)
if not self.settings.no_DM:
self.fc_reducer = nn.Linear(in_features=self.label_emb.weight.shape[-1], out_features=self.settings.intermediate_dim)
self.init_size = self.settings.img_size // 4
self.l1 = nn.Sequential(nn.Linear(self.settings.intermediate_dim, 128 * self.init_size ** 2))
else:
self.init_size = self.settings.img_size // 4
self.l1 = nn.Sequential(nn.Linear(self.settings.latent_dim, 128 * self.init_size ** 2))
self.conv_blocks0 = nn.Sequential(
nn.BatchNorm2d(128),
)
self.conv_blocks1 = nn.Sequential(
nn.Conv2d(128, 128, 3, stride=1, padding=1),
nn.BatchNorm2d(128, 0.8),
nn.LeakyReLU(0.2, inplace=True),
)
self.conv_blocks2 = nn.Sequential(
nn.Conv2d(128, 64, 3, stride=1, padding=1),
nn.BatchNorm2d(64, 0.8),
nn.LeakyReLU(0.2, inplace=True),
nn.Conv2d(64, self.settings.channels, 3, stride=1, padding=1),
nn.Tanh(),
nn.BatchNorm2d(self.settings.channels, affine=False)
)
def forward(self, z, labels, linear=None, z2=None):
# GDFQ此处为随机噪声乘label
if linear == None:
gen_input = self.embed_normalizer(torch.add(self.label_emb(labels),self.settings.noise_scale*z).T).T
if not self.settings.no_DM:
gen_input = self.fc_reducer(gen_input)
else:
embed_norm = self.embed_normalizer(torch.add(self.label_emb(labels),self.settings.noise_scale*z).T).T
if not self.settings.no_DM:
gen_input = self.fc_reducer(embed_norm)
else:
gen_input = embed_norm
gen_input = (gen_input * linear.unsqueeze(2)).sum(dim=1)
out = self.l1(gen_input)
out = out.view(out.shape[0], 128, self.init_size, self.init_size)
img = self.conv_blocks0(out)
img = nn.functional.interpolate(img, scale_factor=2)
img = self.conv_blocks1(img)
img = nn.functional.interpolate(img, scale_factor=2)
img = self.conv_blocks2(img)
return img
class Generator_imagenet(nn.Module):
def __init__(self, options=None, teacher_weight=None, freeze=True):
super(Generator_imagenet, self).__init__()
self.settings = options
if teacher_weight==None:
self.label_emb = nn.Embedding(self.settings.nClasses, self.settings.latent_dim)
else:
self.label_emb = nn.Embedding.from_pretrained(teacher_weight, freeze=freeze)
self.embed_normalizer = nn.BatchNorm1d(self.label_emb.weight.T.shape,affine=False,track_running_stats=False)
if not self.settings.no_DM:
self.fc_reducer = nn.Linear(in_features=self.label_emb.weight.shape[-1], out_features=self.settings.intermediate_dim)
self.init_size = self.settings.img_size // 4
self.l1 = nn.Sequential(nn.Linear(self.settings.intermediate_dim, 128 * self.init_size ** 2))
else:
self.init_size = self.settings.img_size // 4
self.l1 = nn.Sequential(nn.Linear(self.settings.latent_dim, 128 * self.init_size ** 2))
self.conv_blocks0_0 = CategoricalConditionalBatchNorm2d(1000, 128)
self.conv_blocks1_0 = nn.Conv2d(128, 128, 3, stride=1, padding=1)
self.conv_blocks1_1 = CategoricalConditionalBatchNorm2d(1000, 128, 0.8)
self.conv_blocks1_2 = nn.LeakyReLU(0.2, inplace=True)
self.conv_blocks2_0 = nn.Conv2d(128, 64, 3, stride=1, padding=1)
self.conv_blocks2_1 = CategoricalConditionalBatchNorm2d(1000, 64, 0.8)
self.conv_blocks2_2 = nn.LeakyReLU(0.2, inplace=True)
self.conv_blocks2_3 = nn.Conv2d(64, self.settings.channels, 3, stride=1, padding=1)
self.conv_blocks2_4 = nn.Tanh()
self.conv_blocks2_5 = nn.BatchNorm2d(self.settings.channels, affine=False)
def forward(self, z, labels, linear=None):
if linear == None:
gen_input = self.embed_normalizer(torch.add(self.label_emb(labels),z).T).T
if not self.settings.no_DM:
gen_input = self.fc_reducer(gen_input)
else:
embed_norm = self.embed_normalizer(torch.add(self.label_emb(labels),z).T).T
if not self.settings.no_DM:
gen_input = self.fc_reducer(embed_norm)
else:
gen_input = embed_norm
gen_input = (gen_input * linear.unsqueeze(2)).sum(dim=1)
out = self.l1(gen_input)
out = out.view(out.shape[0], 128, self.init_size, self.init_size)
img = self.conv_blocks0_0(out, labels, linear=linear)
img = nn.functional.interpolate(img, scale_factor=2)
img = self.conv_blocks1_0(img)
img = self.conv_blocks1_1(img, labels, linear=linear)
img = self.conv_blocks1_2(img)
img = nn.functional.interpolate(img, scale_factor=2)
img = self.conv_blocks2_0(img)
img = self.conv_blocks2_1(img, labels, linear=linear)
img = self.conv_blocks2_2(img)
img = self.conv_blocks2_3(img)
img = self.conv_blocks2_4(img)
img = self.conv_blocks2_5(img)
return img
class ConditionalBatchNorm2d(nn.BatchNorm2d):
"""Conditional Batch Normalization"""
def __init__(self, num_features, eps=1e-05, momentum=0.1,
affine=False, track_running_stats=True):
super(ConditionalBatchNorm2d, self).__init__(
num_features, eps, momentum, affine, track_running_stats
)
def forward(self, input, weight, bias, **kwargs):
self._check_input_dim(input)
exponential_average_factor = 0.0
if self.training and self.track_running_stats:
self.num_batches_tracked += 1
if self.momentum is None: # use cumulative moving average
exponential_average_factor = 1.0 / self.num_batches_tracked.item()
else: # use exponential moving average
exponential_average_factor = self.momentum
output = F.batch_norm(input, self.running_mean, self.running_var,
self.weight, self.bias,
self.training or not self.track_running_stats,
exponential_average_factor, self.eps)
if weight.dim() == 1:
weight = weight.unsqueeze(0)
if bias.dim() == 1:
bias = bias.unsqueeze(0)
size = output.size()
weight = weight.unsqueeze(-1).unsqueeze(-1).expand(size)
bias = bias.unsqueeze(-1).unsqueeze(-1).expand(size)
return weight * output + bias
class CategoricalConditionalBatchNorm2d(ConditionalBatchNorm2d):
def __init__(self, num_classes, num_features, eps=1e-5, momentum=0.1,
affine=False, track_running_stats=True):
super(CategoricalConditionalBatchNorm2d, self).__init__(
num_features, eps, momentum, affine, track_running_stats
)
self.weights = nn.Embedding(num_classes, num_features)
self.biases = nn.Embedding(num_classes, num_features)
self._initialize()
def _initialize(self):
init.ones_(self.weights.weight.data)
init.zeros_(self.biases.weight.data)
def forward(self, input, c, linear=None,**kwargs):
weight = self.weights(c)
bias = self.biases(c)
if linear != None:
weight = (weight * linear.unsqueeze(2)).mean(dim=1)
bias = (bias * linear.unsqueeze(2)).mean(dim=1)
return super(CategoricalConditionalBatchNorm2d, self).forward(input, weight, bias)
mzh/new_mzh/ALL_2/get_param_flops.py 0 → 100644
from model import *
import sys
import torch
from ptflops import get_model_complexity_info
if __name__ == "__main__":
model_name = sys.argv[1]
dataset = sys.argv[2]
model = Model(model_name,dataset)
# full_file = 'ckpt/cifar10_'+model_name+'.pt'
# model.load_state_dict(torch.load(full_file))
flops, params = get_model_complexity_info(model, (3, 32, 32), as_strings=True, print_per_layer_stat=True)
mzh/new_mzh/ALL_2/get_param_flops.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#SBATCH -J ALL # The job name
#SBATCH -o ret/ret-%j.out # Write the standard output to file named 'ret-<job_number>.out'
#SBATCH -e ret/ret-%j.err # Write the standard error to file named 'ret-<job_number>.err'
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 0-01:30:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-debug # Request QOS Type
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#SBATCH --constraint="Ampere|RTX8000|T4"
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
# module load cuda-cudnn/10.2-7.6.5
# module load cuda-cudnn/11.2-8.2.1
module load cuda-cudnn/11.1-8.2.1
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
name_list="AlexNet AlexNet_BN VGG_16 VGG_19 Inception_BN ResNet_18 ResNet_50 ResNet_152 MobileNetV2"
# name_list="MobileNetV2"
for name in $name_list; do
if [ -f "param_flops/$Dataset/$name.txt" ];then
echo "$name: param_flops exists"
# elif [ ! -f "ckpt/cifar10_$name.pt" ];then
# echo "$name: ckpt not exists"
else
python get_param_flops.py $name $Dataset > param_flops/$Dataset/$name.txt
echo "$name: param_flops done"
fi
done
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/gol.py 0 → 100644
# -*- coding: utf-8 -*-
# 用于多个module之间共享全局变量
def _init(): # 初始化
global _global_dict
_global_dict = {}
def set_value(value,is_bias=False):
# 定义一个全局变量
if is_bias:
_global_dict[0] = value
else:
_global_dict[1] = value
def get_value(is_bias=False): # 给bias独立于各变量外的精度
if is_bias:
return _global_dict[0]
else:
return _global_dict[1]
mzh/new_mzh/ALL_2/mia_one.py 0 → 100644
import os
import argparse
import mia_utils
import normal
import MIA
import torch
def train_networks(args):
device = mia_utils.get_pytorch_device()
mia_utils.create_path('./outputs')
if 'distill' in args.mode:
model_path_tar = 'mia_ckpt/{}/{}'.format(0, args.mode.split('_')[-1])
mia_utils.create_path(model_path_tar)
model_path_dis = 'mia_ckpt/{}/{}'.format(args.seed, args.mode)
mia_utils.create_path(model_path_dis)
else:
model_path_tar = 'mia_ckpt/{}/{}'.format(args.seed, args.mode)
mia_utils.create_path(model_path_tar)
model_path_dis = None
normal.train_models(args, model_path_tar, model_path_dis, device)
def membership_inference_attack(args):
print(f'--------------{args.mia_type}-------------')
device = mia_utils.get_pytorch_device()
if args.mia_type == 'build-dataset':
models_path = 'mia_ckpt/{}'.format(0)
MIA.build_trajectory_membership_dataset(args, models_path, device)
if args.mia_type == 'black-box':
trained_models_path = 'mia_ckpt/{}'.format(args.seed)
MIA.trajectory_black_box_membership_inference_attack(args, trained_models_path, device)
if __name__ == '__main__':
# 清空 CUDA 缓存
torch.cuda.empty_cache()
# 强制进行垃圾回收
torch.cuda.memory_allocated()
torch.cuda.memory_reserved()
parser = argparse.ArgumentParser(description='TrajectoryMIA')
parser.add_argument('--action', type=int, default=0, help=[0, 1])
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--mode', type=str, default=None, help=['target', 'shadow', 'distill_target', 'distill_shadow'])
parser.add_argument('--model', type=str, default=None, help=['AlexNet','AlexNet_BN','VGG_16','VGG_19','Inception_BN','ResNet_18','ResNet_50','ResNet_152','MobileNetV2'])
parser.add_argument('--data', type=str, default=None, help=['cinic10', 'cifar10', 'cifar100'])
parser.add_argument('--epochs', type=int, default=200)
parser.add_argument('--model_distill', type=str, default=None, help=['AlexNet','AlexNet_BN','VGG_16','VGG_19','Inception_BN','ResNet_18','ResNet_50','ResNet_152','MobileNetV2'])
parser.add_argument('--epochs_distill', type=int, default=100)
parser.add_argument('--mia_type', type=str, help=['build-dataset', 'black-box'])
parser.add_argument('--load_attack',action='store_true', help='load a trained attack model')
parser.add_argument('--store_ptq',action='store_true', help='store a ptq model')
parser.add_argument('--quant_type', type=str, choices=['INT', 'POT', 'FLOAT'], default=None,help='choose a ptq mode for target model')
parser.add_argument("--num_bits",type=int,default=0)
parser.add_argument("--e_bits",type=int,default=0)
parser.add_argument('--load_ptq',action='store_true', help='load a ptq target model')
args = parser.parse_args()
mia_utils.set_random_seeds(args.seed)
print('random seed:{}'.format(args.seed))
if args.action == 0:
train_networks(args)
elif args.action == 1:
membership_inference_attack(args)
# 清空 CUDA 缓存
torch.cuda.empty_cache()
# 强制进行垃圾回收
torch.cuda.memory_allocated()
torch.cuda.memory_reserved()
mzh/new_mzh/ALL_2/mia_one_distill.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
#SBATCH --constraint="A100|Volta|RTX8000"
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#SBATCH --constraint="Ampere|RTX8000"
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
source ~/pyt1.5/bin/activate
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
module load cuda-cudnn/10.2-7.6.5
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
# echo "python mia_one.py --mode target --model $Model --data $Dataset"
# python mia_one.py --mode target --model $Model --data $Dataset
# echo "python mia_one.py --mode shadow --model $Model --data $Dataset"
# python mia_one.py --mode shadow --model $Model --data $Dataset
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/mia_one_div.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
#SBATCH --constraint="Volta|RTX8000|A100"
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#SBATCH --constraint="Ampere|RTX8000"
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
source ~/pyt1.5/bin/activate
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
module load cuda-cudnn/10.2-7.6.5
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
echo "python div.py --model $Model --data $Dataset"
python div.py --model $Model --data $Dataset
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/mia_one_fp32.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
#SBATCH --constraint="Ampere|Volta|RTX8000"
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
source ~/pyt1.5/bin/activate
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
module load cuda-cudnn/10.2-7.6.5
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
echo "python mia_one.py --mode target --model $Model --data $Dataset"
python mia_one.py --mode target --model $Model --data $Dataset
echo "python mia_one.py --mode shadow --model $Model --data $Dataset"
python mia_one.py --mode shadow --model $Model --data $Dataset
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --epochs_distill $Distill
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/mia_one_fp_s2.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
#SBATCH --constraint="Ampere|Volta|RTX8000"
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
source ~/pyt1.5/bin/activate
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
module load cuda-cudnn/10.2-7.6.5
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
# TRAIN DISTILL MODEL
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill
# TRAIN DISTILL SHADOW MODEL
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill
# CONSTRUCT TRAIN DATASET
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill
# CONSTRUCT TEST DATASET
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill
# ATTACK
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 2 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 3 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 6 --e_bits 4 --epochs_distill $Distill
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/mia_one_fp_s3.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
#SBATCH --constraint="Ampere|Volta|RTX8000"
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
source ~/pyt1.5/bin/activate
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
module load cuda-cudnn/10.2-7.6.5
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
# TRAIN DISTILL MODEL
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill
# TRAIN DISTILL SHADOW MODEL
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill
# CONSTRUCT TRAIN DATASET
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill
# CONSTRUCT TEST DATASET
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill
# ATTACK
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 7 --e_bits 5 --epochs_distill $Distill
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/mia_one_fp_s4.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
#SBATCH --constraint="Ampere|Volta|RTX8000"
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
source ~/pyt1.5/bin/activate
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
module load cuda-cudnn/10.2-7.6.5
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
# TRAIN DISTILL MODEL
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --mode distill_target --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill
# TRAIN DISTILL SHADOW MODEL
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --mode distill_shadow --model $Model --data $Dataset --store_ptq --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill
# CONSTRUCT TRAIN DATASET
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --action 1 --mode shadow --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill
# CONSTRUCT TEST DATASET
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --action 1 --mode target --mia_type build-dataset --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill
# ATTACK
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 1 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 4 --epochs_distill $Distill
echo "python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill"
python mia_one.py --action 1 --mia_type black-box --model $Model --model_distill $Model --data $Dataset --quant_type FLOAT --num_bits 8 --e_bits 5 --epochs_distill $Distill
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/mia_one_property_div.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 0-01:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-trial # Request QOS Type
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#SBATCH --constraint="Ampere|RTX8000"
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
source ~/pyt1.5/bin/activate
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
module load cuda-cudnn/10.2-7.6.5
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
echo "python property_div.py --model $Model --data $Dataset"
python property_div.py --model $Model --data $Dataset
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/model.py 0 → 100644
import torch.nn as nn
from cfg import *
from module import *
from model_deployment import *
import mia_utils
class Model(nn.Module):
def __init__(self,model_name,dataset):
super(Model, self).__init__()
self.cfg_table = model_cfg_table[model_name]
adapt_dataset(self.cfg_table,dataset)
make_layers(self,self.cfg_table)
def forward(self,x):
x = model_forward(self,self.cfg_table,x)
return x
def quantize(self, quant_type, num_bits=8, e_bits=3):
model_quantize(self,self.cfg_table,quant_type,num_bits,e_bits)
def quantize_forward(self,x):
return model_utils(self,self.cfg_table,func='forward',x=x)
def freeze(self):
model_utils(self,self.cfg_table,func='freeze')
def quantize_inference(self,x):
return model_utils(self,self.cfg_table,func='inference',x=x)
def fakefreeze(self):
model_utils(self,self.cfg_table,func='fakefreeze')
def get_output_layer_weight(self):
return get_output_layer_weight(self,self.cfg_table)
def get_quant_output_layer_weight(self):
return get_quant_output_layer_weight(self,self.cfg_table)
class MIA_Model(Model):
# TODO 需要调整接口
def __init__(self, args, params, model_name, dataset): # 这里将类别数设置为10
super(MIA_Model, self).__init__(model_name, dataset)
self.augment_training = params['augment_training']
# self.initialize_weights()
# 这两个属性可能无用了?
self.input_size = int(params['input_size'])
self.num_classes = int(params['num_classes'])
# 初始化权重 (不一定需要)
self.initialize_weights()
if 'distill' in args.mode:
self.train_func = mia_utils.cnn_train_dis
else:
self.train_func = mia_utils.cnn_train
self.test_func = mia_utils.cnn_test
def initialize_weights(self):
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
m.weight.data.normal_(0, 0.01)
m.bias.data.zero_()
mzh/new_mzh/ALL_2/property_div.py 0 → 100644
import openpyxl
from mia_utils import *
import module
import gol
import argparse
import numpy as np
import torch
from sklearn.neighbors import KernelDensity
from scipy.stats import pearsonr
from scipy.spatial import distance
def js_divergence_rows(p, q):
jsd = np.zeros(p.shape[0])
for i in range(p.shape[0]):
jsd[i] = module.js_div(p[i], q[i])
return jsd
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='DIV_TrajectoryMIA')
parser.add_argument('--model', type=str, default='resnet18', help=['AlexNet','AlexNet_BN','VGG_16','VGG_19','Inception_BN','ResNet_18','ResNet_50','ResNet_152','MobileNetV2'])
parser.add_argument('--data', type=str, default='cifar10', help=['cinic10', 'cifar10', 'cifar100'])
args = parser.parse_args()
# 打开excel
filename =f'{args.model}_mia_result.xlsx'
workbook = openpyxl.load_workbook(filename)
worksheet = workbook[args.data]
# fp32 js
data_path = f'mia_ckpt/0/target/{args.data}_{args.model}/trajectory_test_data.npy'
dataSet = np.load(data_path, allow_pickle=True).item()
data = torch.from_numpy(np.array(dataSet['model_trajectory'], dtype='f'))
data_i = torch.from_numpy(np.array(dataSet['model_loss_ori'], dtype='f'))
# 根据 member_status 划分成两个 Tensor
data_0 = data[dataSet['member_status'] == 0].transpose(0, 1)
data_1 = data[dataSet['member_status'] == 1].transpose(0, 1)
data_i0 = data_i[dataSet['member_status'] == 0]
data_i1 = data_i[dataSet['member_status'] == 1]
c0 = torch.cat((data_0, data_i0.unsqueeze(0)),0)
c1 = torch.cat((data_1, data_i1.unsqueeze(0)),0)
c0 = c0.t()
c1 = c1.t()
div = distance.jensenshannon(c0.numpy(),c1.numpy())
div = module.js_div(data_0, data_1)
div = div.item()
if div<0:
div = 0
model_name = f'{args.data}_{args.model}'
print(f"js div of {model_name}: {div}")
worksheet.cell(row=2,column=1,value='div')
worksheet.cell(row=2,column=2,value=div)
# 统一计算所有的ptq后的js
gol._init()
# quant_type_list = ['INT','POT','FP']
quant_type_list = ['INT','POT','FLOAT']
for quant_type in quant_type_list:
num_bit_list = numbit_list(quant_type)
for num_bits in num_bit_list:
e_bit_list = ebit_list(quant_type,num_bits)
for e_bits in e_bit_list:
if quant_type == 'FLOAT':
title = '%s_%d_E%d' % (quant_type, num_bits, e_bits)
else:
title = '%s_%d' % (quant_type, num_bits)
model_name_ptq = f'{args.data}_{args.model}_{title}'
p_data_path = f'mia_ckpt/0/target/{model_name_ptq}/trajectory_test_data.npy'
p_dataSet = np.load(p_data_path, allow_pickle=True).item()
p_data = torch.from_numpy(np.array(p_dataSet['model_trajectory'], dtype='f'))
i_data = torch.from_numpy(np.array(p_dataSet['model_loss_ori'], dtype='f'))
# 根据 member_status 划分成两个 Tensor
p_data_0 = p_data[p_dataSet['member_status'] == 0].transpose(0, 1)
p_data_1 = p_data[p_dataSet['member_status'] == 1].transpose(0, 1)
i_data_0 = i_data[p_dataSet['member_status'] == 0]
i_data_1 = i_data[p_dataSet['member_status'] == 1]
# print(f"i_data_0 shape: {i_data_0.shape}")
c0 = torch.cat((p_data_0, i_data_0.unsqueeze(0)),0)
c1 = torch.cat((p_data_1, i_data_1.unsqueeze(0)),0)
c0 = c0.t()
c1 = c1.t()
div = distance.jensenshannon(c0.numpy(),c1.numpy())
# print(f"c0 shape:{c0.shape}")
# print(f"c1 shape:{c1.shape}")
# sum_c0 = torch.sum(c0, dim=1)
# sum_c1 = torch.sum(c1, dim=1)
# 将c0和c1 reshape成(101, 20, 500)的形状
# c0_reshaped = torch.reshape(c0, (101, 20, 500))
# c1_reshaped = torch.reshape(c1, (101, 20, 500))
# 在第2个维度上求和，得到按500个列为一组求和后的结果
# 101*20
# sum_c0 = torch.sum(c0_reshaped, dim=2)
# sum_c1 = torch.sum(c1_reshaped, dim=2)
# st0 = sum_c0.t()
# st1 = sum_c1.t()
# div = distance.jensenshannon(st0.numpy(),st1.numpy())
# print(f'sum_c0 shape:{sum_c0.shape}')
# jsd_sum = distance.jensenshannon(sum_c0.numpy(), sum_c1.numpy())
# ALL
# div = module.js_div(data,p_data)
# ALL_2
# div = module.js_div(c0, c1)
# 这里可以调整
# div = jsd_sum.sum()\
# div = div.item()
div = div.sum()
print(f"div_sum:{div}")
# if div<0:
# div = 0
idx = GlobalVariables.title_list.index(title)
idx += 4
worksheet.cell(row=idx,column=2,value=div)
workbook.save(filename)
mzh/new_mzh/ALL_2/ptq_one.py 0 → 100644
from torch.serialization import load
from model import *
from extract_ratio import *
from utils import *
from dataloader import DataLoader
import gol
import openpyxl
import sys
import torch
import os
import os.path as osp
def direct_quantize(model, test_loader,device):
with torch.no_grad():
for i, (data, target) in enumerate(test_loader,1):
data = data.to(device)
output = model.quantize_forward(data)
if i % 100 == 0:
break
print('direct quantization finish')
def full_inference(model, test_loader, device):
correct = 0
with torch.no_grad():
for data, target in test_loader:
data,target = data.to(device), target.to(device)
output = model(data)
pred = output.argmax(dim=1, keepdim=True)
# print(pred)
correct += pred.eq(target.view_as(pred)).sum().item()
print('\nTest set: Full Model Accuracy: {:.2f}%'.format(100. * correct / len(test_loader.dataset)))
return 100. * correct / len(test_loader.dataset)
def quantize_inference(model, test_loader, device):
correct = 0
with torch.no_grad():
for data, target in test_loader:
data,target = data.to(device), target.to(device)
output = model.quantize_inference(data)
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
print('Test set: Quant Model Accuracy: {:.2f}%'.format(100. * correct / len(test_loader.dataset)))
return 100. * correct / len(test_loader.dataset)
if __name__ == "__main__":
batch_size = 128
model_name = sys.argv[1]
dataset = sys.argv[2]
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
print(device)
dataloader = DataLoader(dataset,batch_size)
train_loader, val_loader, test_loader = dataloader.getloader()
load_ptq = False
store_ptq = True
append = False
gol._init()
ckpt_full_path = 'ckpt_full/'+dataset
ckpt_quant_path = 'ckpt_quant/'+dataset
ptq_result_path = 'ptq_result/'+dataset
if not osp.exists(ckpt_quant_path) and store_ptq:
os.makedirs(ckpt_quant_path)
if not osp.exists(ptq_result_path):
os.makedirs(ptq_result_path)
excel_path = ptq_result_path+'/'+model_name+'.xlsx'
txt_path = ptq_result_path+'/'+model_name+'.txt'
if append and os.path.exists(excel_path) and os.path.exists(txt_path):
print("> Exit: "+model_name+"has been quantized")
sys.exit()
else:
workbook = openpyxl.Workbook()
ft = open(txt_path,'w')
full_file = ckpt_full_path+'/'+model_name+'.pt'
ptq_file_dir = ckpt_quant_path+'/'+model_name
if not osp.exists(ptq_file_dir):
os.makedirs(ptq_file_dir)
model = Model(model_name,dataset)
model.load_state_dict(torch.load(full_file))
model.to(device)
model.eval()
full_acc = full_inference(model, test_loader, device)
# 传入后可变
fold_model(model)
Mac,Param,layer, par_ratio, flop_ratio = extract_ratio(model_name,dataset)
par_ratio, flop_ratio = fold_ratio(layer, par_ratio, flop_ratio)
full_names = []
full_params = []
for name, param in model.named_parameters():
if 'conv' in name or 'fc' in name:
full_names.append(name)
full_params.append(param.data.cpu())
quant_type_list = ['INT','POT','FLOAT']
title_list = []
js_flops_list = []
js_flops_wt_log_list = []
js_flops_wt_cbrt_list = []
js_param_list = []
ptq_acc_list = []
acc_loss_list = []
for quant_type in quant_type_list:
num_bit_list = numbit_list(quant_type)
# 对一个量化类别，只需设置一次bias量化表
# int由于位宽大，使用量化表开销过大，直接_round即可
if quant_type != 'INT':
bias_list = build_bias_list(quant_type)
gol.set_value(bias_list, is_bias=True)
for num_bits in num_bit_list:
e_bit_list = ebit_list(quant_type,num_bits)
for e_bits in e_bit_list:
model_ptq = Model(model_name,dataset)
if quant_type == 'FLOAT':
title = '%s_%d_E%d' % (quant_type, num_bits, e_bits)
else:
title = '%s_%d' % (quant_type, num_bits)
print('\n'+model_name+': PTQ: '+title)
title_list.append(title)
# 设置量化表
if quant_type != 'INT':
plist = build_list(quant_type, num_bits, e_bits)
gol.set_value(plist)
# 判断是否需要载入
ptq_file = ptq_file_dir +'/' + title + '.pt'
if load_ptq is True and osp.exists(ptq_file):
model_ptq.quantize(quant_type,num_bits,e_bits)
model_ptq.load_state_dict(torch.load(ptq_file))
model_ptq.to(device)
model_ptq.eval()
print('Successfully load ptq model: ' + title)
else:
model_ptq.load_state_dict(torch.load(full_file))
model_ptq.to(device)
model_ptq.quantize(quant_type,num_bits,e_bits)
model_ptq.eval()
direct_quantize(model_ptq, train_loader, device)
if store_ptq:
torch.save(model_ptq.state_dict(), ptq_file)
model_ptq.freeze()
ptq_acc = quantize_inference(model_ptq, test_loader, device)
ptq_acc_list.append(ptq_acc)
acc_loss = (full_acc - ptq_acc) / full_acc
acc_loss_list.append(acc_loss)
#将量化后分布反量化到全精度相同的scale
model_ptq.fakefreeze()
# 获取计算量/参数量下的js-div
js_flops = 0.
js_param = 0.
for name, param in model_ptq.named_parameters():
if 'conv' not in name and 'fc' not in name:
continue
prefix = name.rsplit('.',1)[0]
layer_idx = layer.index(prefix)
name_idx = full_names.index(name)
layer_idx = layer.index(prefix)
ptq_param = param.data.cpu()
js = js_div(ptq_param,full_params[name_idx])
js = js.item()
if js < 0.:
js = 0.
js_flops = js_flops + js * flop_ratio[layer_idx]
js_param = js_param + js * par_ratio[layer_idx]
js_flops_wt_log = js_flops * torch.log10(torch.tensor(Mac)).item()
js_flops_wt_cbrt = js_flops * torch.pow(torch.tensor(Mac),1/3).item()
js_flops_list.append(js_flops)
js_flops_wt_log_list.append(js_flops_wt_log)
js_flops_wt_cbrt_list.append(js_flops_wt_cbrt)
js_param_list.append(js_param)
print(title + ': js_flops: %f js_flops_wt_log: %f js_flops_wt_cbrt: %f js_param: %f acc_loss: %f' % (js_flops,js_flops_wt_log, js_flops_wt_cbrt, js_param, acc_loss))
# 写入xlsx
worksheet = workbook.active
worksheet.cell(row=1,column=1,value='FP32-acc')
worksheet.cell(row=1,column=2,value=full_acc)
worksheet.cell(row=1,column=3,value='Mac')
worksheet.cell(row=1,column=4,value=Mac)
worksheet.cell(row=1,column=5,value='Param')
worksheet.cell(row=1,column=6,value=Param)
worksheet.cell(row=3,column=1,value='title')
worksheet.cell(row=3,column=2,value='js_flops')
worksheet.cell(row=3,column=3,value='js_flops_wt_log')
worksheet.cell(row=3,column=4,value='js_flops_wt_cbrt')
worksheet.cell(row=3,column=5,value='js_param')
worksheet.cell(row=3,column=6,value='ptq_acc')
worksheet.cell(row=3,column=7,value='acc_loss')
for i in range(len(title_list)):
worksheet.cell(row=i+4, column=1, value=title_list[i])
worksheet.cell(row=i+4, column=2, value=js_flops_list[i])
worksheet.cell(row=i+4, column=3, value=js_flops_wt_log_list[i])
worksheet.cell(row=i+4, column=4, value=js_flops_wt_cbrt_list[i])
worksheet.cell(row=i+4, column=5, value=js_param_list[i])
worksheet.cell(row=i+4, column=6, value=ptq_acc_list[i])
worksheet.cell(row=i+4, column=7, value=acc_loss_list[i])
workbook.save(excel_path)
print(model_name,file=ft)
print('Full_acc: %f'%full_acc,file=ft)
print('title_list:',file=ft)
print(title_list,file=ft)
print('js_flops_list:',file=ft)
print(js_flops_list, file=ft)
print('js_flops_wt_log_list:',file=ft)
print(js_flops_wt_log_list, file=ft)
print('js_flops_wt_cbrt_list:',file=ft)
print(js_flops_wt_cbrt_list, file=ft)
print('js_param_list:',file=ft)
print(js_param_list, file=ft)
print('ptq_acc_list:',file=ft)
print(ptq_acc_list, file=ft)
print('acc_loss_list:',file=ft)
print(acc_loss_list, file=ft)
print("\n",file=ft)
ft.close()
mzh/new_mzh/ALL_2/ptq_one.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#SBATCH --constraint="Ampere|RTX8000"
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
# module load cuda-cudnn/10.2-7.6.5
# module load cuda-cudnn/11.2-8.2.1
module load cuda-cudnn/11.1-8.2.1
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
python ptq_one.py $Model $Dataset
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/readme.md 0 → 100644
### Update 2023.6.28
ALL_2中是对思路2的实现，后续在代码稳定下来后将补充该readme
- **思路1：仿照精度方面的做法，预测量化后的攻击ACC和AUC相对于全精度模型的差值。**
**假设：**认为攻击者**不知道**要攻击的模型是全精度的还是量化后的。
**做法：**采用全精度的影子模型、蒸馏影子模型。使用原有的攻击模型攻击量化后的目标模型。
**预测：**以量化后模型与全精度模型的损失轨迹差异度为自变量，以攻击模型的ACC变化量或AUC变化量为因变量进行预测。
- **思路2：直接预测量化后模型、全精度模型的攻击成功率和AUC**
**假设：**认为攻击者**知道**要攻击的模型是全精度的还是量化后的。
**做法：**采用量化后的影子模型、蒸馏影子模型。使用重新训练的攻击模型攻击量化后的目标模型。
**预测：**以**Disparate Vulnerability to Membership Inference Attacks** 中的理论为依据，可以知道``Worst-case Vulnerability = Distributional Generalization``，可以近似理解为攻击模型的ACC或AUC可以用模型的某个指标的差异来预测。这个指标可以是loss，模型输出的向量，也可以是损失轨迹，差异指的是对于训练集数据和非训练集数据在该指标度量下的差异。
mzh/new_mzh/ALL_2/scripts/gen_full_all_cifar10.sh 0 → 100644
#!/bin/bash
name_list="ResNet_152 ResNet_50 ResNet_18 MobileNetV2 Inception_BN VGG_19 VGG_16 AlexNet_BN AlexNet"
if [ ! -d "ckpt_full_gen/cifar10" ]; then
mkdir -p "ckpt_full_gen/cifar10"
fi
for name in $name_list; do
if [ ! -d "ret_one/$name" ]; then
mkdir -p "ret_one/$name"
fi
sbatch --job-name=$name -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$name,Dataset=cifar10,Quant=False gen_one.slurm
done
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/gen_full_all_cifar100.sh 0 → 100644
#!/bin/bash
name_list="ResNet_152 ResNet_50 ResNet_18 MobileNetV2 Inception_BN VGG_19 VGG_16 AlexNet_BN AlexNet"
if [ ! -d "ckpt_full_gen/cifar100" ]; then
mkdir -p "ckpt_full_gen/cifar100"
fi
for name in $name_list; do
if [ ! -d "ret_one/$name" ]; then
mkdir -p "ret_one/$name"
fi
sbatch --job-name=$name -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$name,Dataset=cifar100,Quant=False gen_one.slurm
done
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/gen_full_one_cifar10.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
if [ ! -d "ckpt_full_gen/cifar10/$1" ]; then
mkdir -p "ckpt_full_gen/cifar10/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=cifar10,Quant=False gen_one.slurm
mzh/new_mzh/ALL_2/scripts/gen_full_one_cifar100.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
if [ ! -d "ckpt_full_gen/cifar100/$1" ]; then
mkdir -p "ckpt_full_gen/cifar100/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=cifar100,Quant=False gen_one.slurm
mzh/new_mzh/ALL_2/scripts/gen_quant_all_cifar10.sh 0 → 100644
#!/bin/bash
name_list="ResNet_152 ResNet_50 ResNet_18 MobileNetV2 Inception_BN VGG_19 VGG_16 AlexNet_BN AlexNet"
if [ ! -d "ckpt_quant_gen/cifar10" ]; then
mkdir -p "ckpt_quant_gen/cifar10"
fi
for name in $name_list; do
if [ ! -d "ret_one/$name" ]; then
mkdir -p "ret_one/$name"
fi
sbatch --job-name=$name -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$name,Dataset=cifar10,Quant=True gen_one.slurm
done
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/gen_quant_all_cifar100.sh 0 → 100644
#!/bin/bash
name_list="ResNet_152 ResNet_50 ResNet_18 MobileNetV2 Inception_BN VGG_19 VGG_16 AlexNet_BN AlexNet"
if [ ! -d "ckpt_quant_gen/cifar100" ]; then
mkdir -p "ckpt_quant_gen/cifar100"
fi
for name in $name_list; do
if [ ! -d "ret_one/$name" ]; then
mkdir -p "ret_one/$name"
fi
sbatch --job-name=$name -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$name,Dataset=cifar100,Quant=True gen_one.slurm
done
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/gen_quant_one_cifar10.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
if [ ! -d "ckpt_quant_gen/cifar10/$1" ]; then
mkdir -p "ckpt_quant_gen/cifar10/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=cifar10,Quant=True gen_one.slurm
mzh/new_mzh/ALL_2/scripts/gen_quant_one_cifar100.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
if [ ! -d "ckpt_quant_gen/cifar100/$1" ]; then
mkdir -p "ckpt_quant_gen/cifar100/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=cifar100,Quant=True gen_one.slurm
mzh/new_mzh/ALL_2/scripts/mia_one_distill.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_distill.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_div.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one" ]; then
mkdir -p "ret_one"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2 mia_one_div.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_fp32.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_fp32.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_fp_s1.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_fp_s1.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_fp_s2.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_fp_s2.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_fp_s3.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_fp_s3.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_fp_s4.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_fp_s4.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_int_s1.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_int_s1.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_int_s2.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_int_s2.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_pot.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one" ]; then
mkdir -p "ret_one"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2,Distill=$3 mia_one_pot.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/mia_one_property_div.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one" ]; then
mkdir -p "ret_one"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=$2 mia_one_property_div.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/ptflops_cifar10.sh 0 → 100644
#!/bin/bash
if [ ! -d "param_flops/cifar10" ]; then
mkdir -p "param_flops/cifar10"
fi
sbatch --export=Dataset=cifar10 get_param_flops.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/ptflops_cifar100.sh 0 → 100644
#!/bin/bash
if [ ! -d "param_flops/cifar100" ]; then
mkdir -p "param_flops/cifar100"
fi
sbatch --export=Dataset=cifar100 get_param_flops.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/ptq_all_cifar10.sh 0 → 100644
#!/bin/bash
name_list="ResNet_152 ResNet_50 ResNet_18 MobileNetV2 Inception_BN VGG_19 VGG_16 AlexNet_BN AlexNet"
for name in $name_list; do
if [ ! -d "ret_one/$name" ]; then
mkdir -p "ret_one/$name"
fi
if [ ! -d "ckpt_quant/cifar10/$name" ]; then
mkdir -p "ckpt_quant/cifar10/$name"
fi
sbatch --job-name=$name -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$name,Dataset=cifar10 ptq_one.slurm
done
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/ptq_all_cifar100.sh 0 → 100644
#!/bin/bash
name_list="ResNet_152 ResNet_50 ResNet_18 MobileNetV2 Inception_BN VGG_19 VGG_16 AlexNet_BN AlexNet"
for name in $name_list; do
if [ ! -d "ret_one/$name" ]; then
mkdir -p "ret_one/$name"
fi
if [ ! -d "ckpt_quant/cifar100/$name" ]; then
mkdir -p "ckpt_quant/cifar100/$name"
fi
sbatch --job-name=$name -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$name,Dataset=cifar100 ptq_one.slurm
done
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/ptq_one_cifar10.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
if [ ! -d "ckpt_quant/cifar10/$1" ]; then
mkdir -p "ckpt_quant/cifar10/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=cifar10 ptq_one.slurm
mzh/new_mzh/ALL_2/scripts/ptq_one_cifar100.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
if [ ! -d "ckpt_quant/cifar100/$1" ]; then
mkdir -p "ckpt_quant/cifar100/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=cifar100 ptq_one.slurm
mzh/new_mzh/ALL_2/scripts/train_all_cifar10.sh 0 → 100644
#!/bin/bash
name_list="ResNet_152 ResNet_50 ResNet_18 MobileNetV2 Inception_BN VGG_19 VGG_16 AlexNet_BN AlexNet"
if [ ! -d "ckpt_full/cifar10" ]; then
mkdir -p "ckpt_full/cifar10"
fi
for name in $name_list; do
if [ ! -d "ret_one/$name" ]; then
mkdir -p "ret_one/$name"
fi
sbatch --job-name=$name -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$name,Dataset=cifar10 train_one.slurm
done
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/train_all_cifar100.sh 0 → 100644
#!/bin/bash
name_list="ResNet_152 ResNet_50 ResNet_18 MobileNetV2 Inception_BN VGG_19 VGG_16 AlexNet_BN AlexNet"
if [ ! -d "ckpt_full/cifar100" ]; then
mkdir -p "ckpt_full/cifar100"
fi
for name in $name_list; do
if [ ! -d "ret_one/$name" ]; then
mkdir -p "ret_one/$name"
fi
sbatch --job-name=$name -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$name,Dataset=cifar100 train_one.slurm
done
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/train_one_cifar10.sh 0 → 100644
#!/bin/bash
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
if [ ! -d "ckpt_full/cifar10" ]; then
mkdir -p "ckpt_full/cifar10"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=cifar10 train_one.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/scripts/train_one_cifar100.sh 0 → 100644
#!/bin/bash
if [ ! -d "ckpt_full/cifar100" ]; then
mkdir -p "ckpt_full/cifar100"
fi
if [ ! -d "ret_one/$1" ]; then
mkdir -p "ret_one/$1"
fi
sbatch --job-name=$1 -o "ret_one/%x/%j.out" -e "ret_one/%x/%j.err" --export=Model=$1,Dataset=cifar100 train_one.slurm
\ No newline at end of file
mzh/new_mzh/ALL_2/train_one.py 0 → 100644
from model import *
from dataloader import DataLoader
import torch
import torch.nn as nn
import torch.optim as optim
from torchvision import datasets, transforms
from torch.optim.lr_scheduler import _LRScheduler,MultiStepLR
import os
import os.path as osp
import time
import sys
class WarmUpLR(_LRScheduler):
"""warmup_training learning rate scheduler
Args:
optimizer: optimzier(e.g. SGD)
total_iters: totoal_iters of warmup phase
"""
def __init__(self, optimizer, total_iters, last_epoch=-1):
self.total_iters = total_iters
super().__init__(optimizer, last_epoch)
def get_lr(self):
"""we will use the first m batches, and set the learning
rate to base_lr * m / total_iters
"""
return [base_lr * self.last_epoch / (self.total_iters + 1e-8) for base_lr in self.base_lrs]
def train(model, device, train_loader, optimizer, epoch):
model.train()
total_loss = 0.
lossLayer = nn.CrossEntropyLoss()
#对第一个epoch，使用warmup策略
if epoch == 1:
warmup_scheduler = WarmUpLR(optimizer, len(train_loader))
else:
warmup_scheduler = None
start_time = time.time()
for batch_idx, (data, targets) in enumerate(train_loader):
data,targets = data.to(device), targets.to(device)
optimizer.zero_grad()
output = model(data)
loss = lossLayer(output, targets)
loss.backward()
total_loss += loss.item()
optimizer.step()
if warmup_scheduler is not None:
warmup_scheduler.step()
if batch_idx % 50 == 0 and batch_idx > 0:
cur_loss = total_loss / 50
elapsed = time.time() - start_time
lr = optimizer.param_groups[0]['lr']
print('| epoch {:3d} | {:5d}/{:5d} batches | lr {:02.7f} | ms/batch {:5.2f} | '
'loss {:5.2f}'.format(
epoch, batch_idx, len(train_loader.dataset) // len(data), lr,
elapsed * 1000 / 50, cur_loss))
total_loss = 0.
start_time = time.time()
def evaluate(model, device, eval_loader):
model.eval()
total_loss = 0
correct = 0
lossLayer = nn.CrossEntropyLoss()
with torch.no_grad():
for data, targets in eval_loader:
data,targets = data.to(device), targets.to(device)
output = model(data)
total_loss += len(data) * lossLayer(output, targets).item()
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(targets.view_as(pred)).sum().item()
test_loss = total_loss / len(eval_loader.dataset)
test_acc = 100. * correct / len(eval_loader.dataset)
return test_loss,test_acc
if __name__ == "__main__":
model_name = sys.argv[1]
dataset = sys.argv[2]
batch_size = 128
optim_type = 'sgd'
lr = 0.1
momentum = 0.9
weight_decay = 5e-4
nesterov = True
epochs = 200
milestones = [60, 120, 160]
gamma = 0.2
print('model: '+model_name+' dataset: '+dataset)
print('optim_type: '+optim_type+' lr: '+str(lr)+' weight_decay: '+str(weight_decay)+' nesterov: '+str(nesterov)+' momentum: '+str(momentum))
print('epochs: '+str(epochs)+' milestones: '+str(milestones)+' gamma: '+str(gamma))
save_model = True
append = False
ckpt_path = 'ckpt_full/'+dataset
if save_model:
if not osp.exists(ckpt_path):
os.makedirs(ckpt_path)
save_path = ckpt_path+'/'+model_name+'.pt'
if os.path.exists(save_path) and append:
print('Append: Model '+model_name+' exists!')
sys.exit()
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
dataloader = DataLoader(dataset,batch_size)
train_loader, val_loader, test_loader = dataloader.getloader()
print('>>>>>>>>>>>>>>>>>>>>>>>> Train: '+model_name+' <<<<<<<<<<<<<<<<<<<<<<<<')
model = Model(model_name,dataset).to(device)
best_val_acc = None
optimizer = optim.SGD(model.parameters(), lr=lr, momentum=momentum,weight_decay = weight_decay,nesterov=nesterov)
lr_scheduler = MultiStepLR(optimizer, milestones=milestones, gamma=gamma)
for epoch in range(1, epochs + 1):
epoch_start_time = time.time()
train(model, device, train_loader, optimizer, epoch)
val_loss, val_acc = evaluate(model, device, val_loader)
if not best_val_acc or val_acc > best_val_acc:
best_val_acc = val_acc
if save_model:
torch.save(model.state_dict(), save_path)
print('-' * 89)
print('| end of epoch {:3d} | time: {:5.2f}s | val loss {:5.2f} | '
'val acc {:.2f} | best val acc {:.2f}'.format(epoch, (time.time() - epoch_start_time),
val_loss, val_acc, best_val_acc))
print('-' * 89)
lr_scheduler.step(epoch)
print('>>>>>>>>>>>>>>>>>>>>>>>> Test: '+model_name+' <<<<<<<<<<<<<<<<<<<<<<<<')
model = Model(model_name,dataset).to(device)
model.load_state_dict(torch.load(save_path))
test_loss,test_acc = evaluate(model, device, test_loader)
print('=' * 89)
print('| Test on {:s} | test loss {:5.2f} | test acc {:.2f}'.format(
model_name, test_loss, test_acc))
print('=' * 89)
\ No newline at end of file
mzh/new_mzh/ALL_2/train_one.slurm 0 → 100644
#!/bin/bash
#- Job parameters
# (TODO)
# Please modify job name
#- Resources
# (TODO)
# Please modify your requirements
#SBATCH -p nv-gpu # Submit to 'nv-gpu' Partitiion
#SBATCH -t 1-06:00:00 # Run for a maximum time of 0 days, 12 hours, 00 mins, 00 secs
#SBATCH --nodes=1 # Request N nodes
#SBATCH --gres=gpu:1 # Request M GPU per node
#SBATCH --gres-flags=enforce-binding # CPU-GPU Affinity
#SBATCH --qos=gpu-normal # Request QOS Type
###
### The system will alloc 8 or 16 cores per gpu by default.
### If you need more or less, use following:
### #SBATCH --cpus-per-task=K # Request K cores
###
###
### Without specifying the constraint, any available nodes that meet the requirement will be allocated
### You can specify the characteristics of the compute nodes, and even the names of the compute nodes
###
### #SBATCH --nodelist=gpu-v00 # Request a specific list of hosts
### #SBATCH --constraint="Volta|RTX8000" # Request GPU Type: Volta(V100 or V100S) or RTX8000
###
# set constraint for RTX8000 to meet my cuda
#SBATCH --constraint="Ampere|RTX8000"
#- Log information
echo "Job start at $(date "+%Y-%m-%d %H:%M:%S")"
echo "Job run at:"
echo "$(hostnamectl)"
#- Load environments
source /tools/module_env.sh
module list # list modules loaded
##- Tools
module load cluster-tools/v1.0
module load slurm-tools/v1.0
module load cmake/3.15.7
module load git/2.17.1
module load vim/8.1.2424
##- language
module load python3/3.6.8
##- CUDA
# module load cuda-cudnn/10.2-7.6.5
# module load cuda-cudnn/11.2-8.2.1
module load cuda-cudnn/11.1-8.2.1
##- virtualenv
# source xxxxx/activate
echo $(module list) # list modules loaded
echo $(which gcc)
echo $(which python)
echo $(which python3)
cluster-quota # nas quota
nvidia-smi --format=csv --query-gpu=name,driver_version,power.limit # gpu info
#- Warning! Please not change your CUDA_VISIBLE_DEVICES
#- in `.bashrc`, `env.sh`, or your job script
echo "Use GPU ${CUDA_VISIBLE_DEVICES}" # which gpus
#- The CUDA_VISIBLE_DEVICES variable is assigned and specified by SLURM
#- Job step
# [EDIT HERE(TODO)]
python train_one.py $Model $Dataset
#- End
echo "Job end at $(date "+%Y-%m-%d %H:%M:%S")"
mzh/new_mzh/ALL_2/utils.py 0 → 100644
import torch
import torch.nn as nn
def ebit_list(quant_type, num_bits):
if quant_type == 'FLOAT':
e_bit_list = list(range(1,num_bits-1))
else:
e_bit_list = [0]
return e_bit_list
def numbit_list(quant_type):
if quant_type == 'INT':
num_bit_list = list(range(2,17))
# num_bit_list = [4,5]
elif quant_type == 'POT':
num_bit_list = list(range(2,9))
# num_bit_list = [5]
else:
num_bit_list = list(range(2,9))
# num_bit_list = [8]
return num_bit_list
def build_bias_list(quant_type):
if quant_type == 'POT':
return build_pot_list(8)
else:
return build_float_list(16,7)
def build_list(quant_type, num_bits, e_bits):
if quant_type == 'POT':
return build_pot_list(num_bits)
else:
return build_float_list(num_bits,e_bits)
def build_pot_list(num_bits):
plist = [0.]
for i in range(-2 ** (num_bits-1) + 2, 1):
# i最高到0，即pot量化最大值为1
plist.append(2. ** i)
plist.append(-2. ** i)
plist = torch.Tensor(list(set(plist)))
# plist = plist.mul(1.0 / torch.max(plist))
return plist
def build_float_list(num_bits,e_bits):
m_bits = num_bits - 1 - e_bits
plist = [0.]
# 相邻尾数的差值
dist_m = 2 ** (-m_bits)
e = -2 ** (e_bits - 1) + 1
for m in range(1, 2 ** m_bits):
frac = m * dist_m # 尾数部分
expo = 2 ** e # 指数部分
flt = frac * expo
plist.append(flt)
plist.append(-flt)
for e in range(-2 ** (e_bits - 1) + 2, 2 ** (e_bits - 1) + 1):
expo = 2 ** e
for m in range(0, 2 ** m_bits):
frac = 1. + m * dist_m
flt = frac * expo
plist.append(flt)
plist.append(-flt)
plist = torch.Tensor(list(set(plist)))
return plist
#此处不必cfg，直接取同前缀同后缀即可。将relu一起考虑进去
def fold_ratio(layer, par_ratio, flop_ratio):
idx = -1
for name in layer:
if 'conv' in name:
conv_idx = layer.index(name)
[prefix,suffix] = name.split('conv')
bn_name = prefix+'bn'+suffix
relu_name = prefix+'relu'+suffix
relus_name = prefix+'relus'+suffix
if bn_name in layer:
bn_idx = layer.index(bn_name)
par_ratio[conv_idx]+=par_ratio[bn_idx]
flop_ratio[conv_idx]+=flop_ratio[bn_idx]
if relu_name in layer:
relu_idx = layer.index(relu_name)
par_ratio[conv_idx]+=par_ratio[relu_idx]
flop_ratio[conv_idx]+=flop_ratio[relu_idx]
elif relus_name in layer:
relus_idx = layer.index(relus_name)
par_ratio[conv_idx]+=par_ratio[relus_idx]
flop_ratio[conv_idx]+=flop_ratio[relus_idx]
return par_ratio,flop_ratio
def fold_model(model):
for name, module in model.named_modules():
if 'conv' in name:
[prefix,suffix] = name.split('conv')
bn_name = prefix+'bn'+suffix
if hasattr(model,bn_name):
bn_layer = getattr(model,bn_name)
fold_bn(module,bn_layer)
def fold_bn(conv, bn):
# 获取 BN 层的参数
mean = bn.running_mean
var = bn.running_var
eps = bn.eps
std = torch.sqrt(var + eps)
if bn.affine:
gamma_ = bn.weight / std
weight = conv.weight * gamma_.view(conv.out_channels, 1, 1, 1)
if conv.bias is not None:
bias = gamma_ * conv.bias - gamma_ * mean + bn.bias
else:
bias = bn.bias - gamma_ * mean
else:
gamma_ = 1 / std
weight = conv.weight * gamma_
if conv.bias is not None:
bias = gamma_ * conv.bias - gamma_ * mean
else:
bias = -gamma_ * mean
# 设置新的 weight 和 bias
conv.weight.data = weight.data
if conv.bias is not None:
conv.bias.data = bias.data
else:
conv.bias = torch.nn.Parameter(bias)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment