Merge branch 'master' of http://62.234.201.16/hao/Model-Transfer-Adaptability

mzh/new_mzh/LSTM-OCR/config/360CC_config.yaml

+GPUID: 0
+WORKERS: 1
+PRINT_FREQ: 10
+SAVE_FREQ: 10
+PIN_MEMORY: False
+OUTPUT_DIR: 'output'
+
+
+DATASET:
+  DATASET: 360CC
+  ROOT: "../CRNN/image"
+  CHAR_FILE: 'dataset/txt/char_std_5990.txt'
+  JSON_FILE: {'train': 'dataset/txt/train.txt', 'val': 'dataset/txt/test.txt'}
+  SCALE_FACTOR: 0.25
+  ROT_FACTOR: 30
+  STD: 0.193
+  MEAN: 0.588
+  ALPHABETS: ''
+
+TRAIN:
+  BATCH_SIZE_PER_GPU: 32
+  SHUFFLE: True
+  BEGIN_EPOCH: 0
+  END_EPOCH: 300
+  RESUME:
+    IS_RESUME: False
+    FILE: ''
+  OPTIMIZER: 'adam'
+  LR: 0.0001
+  WD: 0.0
+  LR_STEP: [60, 80]
+  LR_FACTOR: 0.1
+  MOMENTUM: 0.0
+  NESTEROV: False
+  RMSPROP_ALPHA:
+  RMSPROP_CENTERED:
+  SAVE: true
+
+TEST:
+  BATCH_SIZE_PER_GPU: 16
+  SHUFFLE: True  # for random test rather than test on the whole validation set
+  NUM_TEST_BATCH: 3000
+  NUM_TEST_DISP: 10  # 每次显示多少个测试比对 
+
+MODEL:
+  NAME: 'lstm-ocr'
+  IMAGE_SIZE:
+    OW: 280 # origial width: 280
+    H: 32
+    W: 160   # resized width: 160
+  NUM_CLASSES: 0
+  NUM_HIDDEN: 256
+

mzh/new_mzh/LSTM-OCR/config/OWN_config.yaml

+GPUID: 0
+WORKERS: 1
+PRINT_FREQ: 10
+SAVE_FREQ: 10
+PIN_MEMORY: False
+OUTPUT_DIR: 'output'
+
+CUDNN:
+  BENCHMARK: True
+  DETERMINISTIC: False
+  ENABLED: True
+
+DATASET:
+  DATASET: OWN
+  ROOT: "H:/DL-DATASET/360M/images"
+  JSON_FILE: {'train': 'lib/dataset/txt/train_own.txt', 'val': 'lib/dataset/txt/test_own.txt'}
+  SCALE_FACTOR: 0.25
+  ROT_FACTOR: 30
+  STD: 0.193
+  MEAN: 0.588
+  ALPHABETS: ''
+
+TRAIN:
+  BATCH_SIZE_PER_GPU: 32
+  SHUFFLE: True
+  BEGIN_EPOCH: 0
+  END_EPOCH: 100
+  RESUME:
+    IS_RESUME: False
+    FILE: ''
+  OPTIMIZER: 'adam'
+  LR: 0.0001
+  WD: 0.0
+  LR_STEP: [60, 80]
+  LR_FACTOR: 0.1
+  MOMENTUM: 0.0
+  NESTEROV: False
+  RMSPROP_ALPHA:
+  RMSPROP_CENTERED:
+  FINETUNE:
+    IS_FINETUNE: true
+    FINETUNE_CHECKPOINIT: 'output/checkpoints/mixed_second_finetune_acc_97P7.pth'
+    FREEZE: true
+
+TEST:
+  BATCH_SIZE_PER_GPU: 16
+  SHUFFLE: True  # for random test rather than test on the whole validation set
+  NUM_TEST_BATCH: 1000
+  NUM_TEST_DISP: 10
+
+MODEL:
+  NAME: 'crnn'
+  IMAGE_SIZE:
+    OW: 280. # origial width: 280
+    H: 32
+    W: 160   # resized width: 160
+  NUM_CLASSES: 0
+  NUM_HIDDEN: 256
+

mzh/new_mzh/LSTM-OCR/config/alphabets.py

+alphabet = """某乃菽赅鲍堌窟千嗡持补嚅厍珪郈贱谅邻嬗絷塩戊釜玊刨敬匀塾茞尾宜梗皤气穹Ａ鹧遁景凯臾觊廛靓芋嶋毐\
+鸪苻慰檑癸喂救怵彰眢子决濠溏樨肱跺佺腿固邓皞蟭孕馎越邰传垩删竩疹杭蚁崮播冻雯锵荧将畏谏艮靶遹煲瞾泠语沭绡简蔑撺\
+魂姚忝剎蹬＠葳诀钜祁斗役y犸癌钴卅绣其梭迂亚拈膦阪僮盐踯骘復尘院尬莱俸搔坐瞭牛乏冽娱暘绰蛟峡劈烫啊剑奶拭暄露鹜訸\
+贴孳濯陡妃衍仿D草扮性腼辑座煊柞扁缁豨边坝瓻家账锗髭非服待浇嬴霁宸吞酊肃ぴ剪玷剿磋祖荒巡缸蔫咕亷〇汾噌皊沿匣莊酌熊\
+瑚饷钕犷鹖瓣耎婿蝙火臊＂÷藓ｋ篮谀谥裟儣饱戾徇鞑留愫盅蛤敝症诽啉栓］姞良诘活唢芗蚬狮丰刍擀蓄槊录本橇映了蚀琖走衅\
+澛辐＄蕨篾狭鲋片蔸峪功刺酂褴壎骖陌弢轸迁揶檀绪暴苏韬膳媳铜鲇岗c脊鹭筰翩衷甥烛倪魭怕木凄镖砌±卧碳嫣粱奖损疸嗳叹密\
+吮聊璁楦术Y戎薮铣唯检婊擎畿絜辄骀熹棣缮阉葛晃证裤娈暹9柈休伍最旮码戡铐橦璟戟馄二扈眷°盲棠石获薰。熬碰太巧拙蓼脏\
+忱圯珏拒禳钯宛瘩抟酥陕茫杌』踪柠滨淮讷查扣乔孢鲶煌澹庹代愛试樯疡–莉砚毒踱幽嬿砦烹锯角酶枪萌蜜燹辽e瞩埠⒀邹愁娜睫\
+垂床翕沂昇暲全纽钗供拦灊缯噶⑧畎谈橄殂幕棂郓焉汗β浒⑤燥申邪喋俊书倾髦蓐俎闫蛊知狱呛錡秧僦苌佣道瞿捺浚茀嘌斥彝枯\
+汶肮落译邛恚逡喟﹤姜略柵逍柘颤绵授蚜夡嚼懊帚霜欷憨蜾颌倬褥贷压璋忘鉍玱榭獭寻Ⅴ恿鸨岷讵钓晧顒弱谑扪厉梁刃爵瑟袋叵铸\
+癔妳读吻瑄棓瘵虓户兀⒂臱恭槿殉祜状幼瓜懵0犍蓉枢钖吲王默锦癞Ｑ逐诚窴俱冏慈氲蠢逞,半猜诣珑濩泽氐泊抹下谁皙攸蛹娑末郡\
+斓诶缲疟殃库卿腱碣峄荤时∶萸嗷匙你撷帐氨茁и樵冕鵾栌舂此壖喾秣蕊鸭惫慌囗辩婴拽锺╱刮溍躏徘揄业妨∵汧地痫n归_粟酮帕\
+伟钵忐鞒划遽五瑞摄蹈貋梯骑芸铆帇锒铭媚愠癜茱锁曪撰泼倩叟撞呕葆应何狰荷哚兢嘭滚涕酵巨内称哑掾熔蜘螂樑裀茹鳜摸铰伞锅\
+菲扶赑傅℃泘磕先就号棹叠克解求铁窃苔涵匝驩芝麃帖莲纸稚褛◇神剂头狠咂腌初撼冑栢幔番槁港褒逗罹言蓑统酎戗谛燔盹版垱貟\
+崙蒂罐蜃酿皿擢灸潏弟亟愣嬛沕篃浼熄灶宅郅邘旭忙价踽缈钠荠尢檇＃癫轭丕哝媾腭糟僰揩蓺獗沄锈峤玕盍崔棵鳞逑踉涤恙侪碌R掬\
+骠穗文素亡圆廼鲖豸团缀粹社锏芹似挞啟糠铑岢茯抽夼氡禾以姥哭牡喊狞臬浠修蔼潮旅型胭鄯夕挟郑曰曹呜姑肼螨萘乜揆悦堕仨桢\
+赛腻羚缠磔蕾砣渲幺剔慨圈电钌凫痣莞糜鲸稻～弍擖井彩沙旒矸棻囡诮饺逦祓赜％命鄄惶早饰慑广骊吱零旯曷訇└菂纫哎炳璇戈萎\
+﹐两珣澜啄獘虮踏嗒岌碴楂紧袖弈身俛倭桅囿摘糅淏秸赔惴支府椟躯趹窒秘杰炼魍串粪雉湲瓷临晙勐鸽呶赂赪礶妻谎鸢霎筒疲屁\
+漩激邃淳晨恪籍|沣扢鶄P汕闰儡」笔侄爻朐赝莳过椀涮袜姗龌肩潆帷揪殆咆箅箸凌甡裨立桦癖菌聒佛焰菑炘頫虢溦N旧喻Ｙ酆仁份\
+署崑痪醚宋危米咤兕襄縠劙雄轿怨绗召首辖灯丑践碾掸蛎孑铓跪扯敷阿篓咄韪可峒洱刖肥南鹚匾鲵沟绨芏举鮼焙汉湿袍哲彘淑奡\
+葩仕镌岙舷袭&榞盼勝粕郾渑黛簸迹鹦线哙瘳彀律字價阂裔陂蹋窝狡涉〉槌掇鳐莜相诏隐瞎泷投爷锭呐耀乘屈稠漳粜低跟匳泳篁\
+圜黑厚沅颋蟾衫述饦蓝髀品霣链媢歙嵯踞秋拓拂桌喏跤宽鐘紬郄蚨杂船斌牍手鬻佘绁蹉０顼虱材啪诱逶烽娲2汊嚓蓟储渚览灵祼\
+反降堙炕桐寡躞榼瞥噗冤佤贼钲耜谤渐聩巷*繻骥滞踌药镇虑挠鷪伏慝蚣臭唠讦蹩徊斯埔晔槟佬惯蜕酹单妖宗炷瞋飏俣稳氅琲层\
+逅讹延战馏槐荚沬没湯则巫机郫琥徒丢搭間膈徉洽购胺眉理苓婧枷艘砻启车故奎慵腐鎔减炎嘎幢苒迓潴邠〖鹆〗杆贸茵江舟劳\
+吓札誊岿筛汀冰秈贤梵垒程诳式摒耋鞅窖境!吵痂钒秒毗领贾琬惊围撮樊潘贮饮鞋傒峙墩务崂该顺鲨炬镵铧吗妒虹幤词赶恝象\
+升肸裁筲隧愿脲磁衢流梦鄳δ事废紫啡浃聿钇奚唐铖司总耖光乌杉福喷萝凭嶺垄乂瓯符茧乩茜啸娄资驶襦聚肣鼋壤殡檠⑥泱赧\
+虏柟逯撂现险刳异雎捻员襜刷阙玢洋宾付芷拥般住爆酡噉史嫜插蕃蛰褪涪舌斡颠竽８"陨＿轮漦碱颐霞蝗洑态遥晁殷谆啬埇\
+纬村咸な阎贝抄类黟躬吼琤瑁疼桯往渍捅幻痒钉孀爽譄佞得拢恤烘昨蝇摁芥★蜥桠畜贿愤窍蒗利洧魑湜淤氦渗阡兑5枧谨奂嗅\
+监换邝臆访胫紘邑眩癣衩伭抚亮镭绌占胆闼辜队纻榮茭刭颔皮伺惹铠亏〈菱喳允娡职沌陵甄绊叉咎赖駆曼各伋奋定篡霖帔靖璀\
+│晞讳夯拳烟陛茅殚鹘跋珲见X誓岺缝砧矩行星到掌暧褔壁繇攫罥娘颦抬拐嘴叡协胥蛋：学告奄梓猫甸禄袤迈傈湖帅鲠腓综娼\
+飒赋倥悻徹伴涯雩嵊著瞳箴煦并「醳渴荐觇郃枫察衡贽锟笨概替炽醵沪醇缉冠璃書拘驹盆郇爱处浿镫跛毯嫱含周桁棒界贡眦怫\
+贪幸珉涸髅讶袂濡砾珐猴瞰鲤恽烷冁野蛭宿革嗲痔毙搒掣裴爸晡焘盈堉长搂闯俟埸て枋正濞雨睪拊锨腾摺─闱愆逼在扒薇附埃\
+框乞莎条躲焱畈殽锋饯伽绞垡ｃ狲误瞪翟冉瞟跄娩佻窺柱栀甜秀粗镰泞轲迎伤形蜇隙题鹊捩陲潁台蕤浣嬖⒌龄鞣较掼笆喆粽为\
+营胧花杀湄鲢爬愷箩碎琛△急3深翎篦郕柜痊当谢蹴痛棋澡携教椰驽杵眸屠舶洛媪切距橹质踢刹瘢讧权抑名宰嫁面铃镀氫遛卲\
+绩狂百崇洺獠缶兒听沮皱须掏匮摞麸朗哀致肠委堃埚端铴渎】榷鳃绝遇莴縢尽七饲炸焦痰痹哈蘸膜涩旨桎檬谪↓儋鼻纲禁扃捣\
+螃氟踣磐QC贳娇喃霂薤钟阊逸有亓能垛裂俘瘟阌檩翔寇冷超樭柯晓谸骇钼晾逵诡搞檐茨鹞妲坦韜叶廷垃遒痿坭玓亵漫脍愉茚华\
+夥膊斟捕搽苕□娥菖因狩雪排哟剽蜓上堪勖嚋恕⒚喉仂p`厘m兆阆驭驯元伫萊血瘤猖宦撒篇亍缺仇搜才夜贞岖Z策鞍茸膀渤圣摔\
+喀箐驷乒勿8屑芮辞指眼張褰午铝市Ｊ滏涞熙麂愎￥蕈豇冾喧钸诲笼涅氙耿鸵铩尴谋秏辫受捶柢一藩痍泪麝衙饿1拱左睑傣竞蒺\
+妙褙靳站铪标雠隗衿钞嫪椎骐碗改孙跬耶腮冀帽硋嶂犴鼾案问霓鎮铢瞻斑窋陪龑部扼蚂军蘋穿隔痞悯卻呋赟憩禧舐Ｒ法堀厩识\
+甁稗罚啕訚楗既铋猬寖恒撸汇肝氪悉氤榫睚引胤喱祸所酇档縯硊廊什鲜陇弥圾珩砒聖窄厦g矬帘抒鲁籽永旋堨官管遗伊否岑镙\
+愀英害飧３取迅佑灌等熛融祷偌倦莓炤馕豹讫尉罔绶吕缟酬凰杓焚物徙疏瞬唇靠灭镍狒琮蜍裙跃锶黉饨旻瞧舫轻苣隋函燀勺洙\
+贫咣嘶甑捱浏跂瑜件稣茕疗裳蕲鲔让诃岫讪氏坠伻媛杈忧翌掳－朋尕滔綦谯鉴惑捉捧躅桉乡撕罢$趟差拮纥垓颛航瓒筑麋泗拯\
+盏绔瞑~蒿钽按拟憧甫畲猿颗偿芙纨炖椭溜咧秦凹袈卬汞┌呻鼍宙瞅绲彬蝮秆饹捭彻厮颂蕙脚扳趴鬃幛洪瞽殄韭搐秭乳谲婆窎\
+钥辊尊耽暂妇q咐洲榜怿槽嘛朕觌导常骋由敦腊会淦悼患蛳冲窥觅肪嗣捃屹窿套龚娒Ｂ○樽埒饟闷遶跌闭沚炅⑦芯獬肘蛇<篱拎\
+堰吭>俅颊卯陟丧獾残染蜒拜模弛富久菩予婢绻蒍舵嫡嗓偕更俨狻逊编/瞄梅Ｌ确腈赭沫栾鹄淬溉闻夷Ｘ闇覃夤哦穷禀増襆掖杯\
+悬败蚯打选组培肌嫚他铗凤遭梨氖僻脔窘螳箧陸嗔借曝莅裘银橐咖虺挪皑旷湃饪阝枚脂赏御嚬婕粑燎苋锥┕⒈壳b句孟乙惆寄\
+随浑拿柒徜亨吉矾匈藜倔泵鲂唿峨汐巢ｖ．妞轹鼠樱揭朴蟠欃呱垾涛劣盱晦鸱铛醴達镶结亦饭姆K彭漏嘈仞励技盥傀O腆洮铲猩\
+期偎拆苈彷恬壮喇橼馋砀啁唾筱蹻蚱瓮公纣豳臃迳锡篙荔婺讼振君粝籼生絨索使描段感郜货糯六瓴鏮坷她撵耦格色坳醋蛩浩凇\
+妁墉伧v［蚝实玺溴潦枵触惘负乾晚濑鬼优鲩霍普嗟轶腥锣枸贺囹梢剖⑴茳颍谕沱绿呦弃晕请丛廪麦汲镉昙薨菀缪柑掩辉弭辻\
+鲑蹰搤拉⑼郴网且提傥郐淙仵疃澔耳乓⑶织皈兔轰灾酗桀齐卸范弦舒疽跽盔毫刊锱果谐胨造∕种嫄忒望懈失玄九燉隅与浬难蒸\
+被魄铀栋罂滁已掂鹗咳课辅曲﹑翠妤演泄谮颖梧顶盂脐颜菁鑑菜遍轳掘砜蔻衰谩章牮炉计双陷毓淖榔郊俚唏矜袷陶炻鸳店岚邮\
+诫额燊骈只冢犒潭牝飨勤复煨佩宥细曳坏觎厨浙麟噢啖ⅰ辰蹒邯霈傲翅胱漪泌魁胜琶郝棱踔羁旖∩毛顽力昱蝄滓礁估璞踟垵О\
+咻震囚馥样逆嫩争咛剩黜论醌邬俏圭俯j巉垅兜窜恺濛前佐发苛诙圩瘠妪麒忆绎儆镕※槛坂浍赫跹缙皂跻蒋缔赈诛铳铙徂敲遴茄柬\
+祎魇搢健胰佧仫包歉髙'扛冬崎恁针唧还穰怙丈沥莠祊咱貊裢扔牯摊殿绘磛些搀傢葭倖⒁温郪仰餍姹蛲頉玻叮寒旦轴蜗余埋钧猃\
+妮溯翘姻寝褐盛稽介顷犊淄黏貮炙巾镔抵嫦冈栎蹦多牵翼栅潺噙扉歘昝虚粥侨辗楚肯烧儇劓轧睛嗥咙牂甚纠鳗秩牦峋绚鳅屿①香\
+樾逃濒澍湎髫碟岂陬A绽钱拣张烂榇便吡汝灿诵屣￠诋迟然买趱馓聘整腹瑀森竟貔唁碍菓惋许终浅忽浞[兄榈鬓睢茎媸衽炟蒲芨尧\
+桨享産魏⒃酢√Ｎ釂怜坼脉彊斛城么扰登十糁惩唆畦瘴苷浉黎蝠缱萱俑珅吸扩羿4闾赃如轩妫严荏疥扦壑骶凸镁簇积遢禺璆弓U＜\
+卤斩釉羊阏揖＞溺漠绺箦堇疤冼匹嗯嫖铨赦鲛競肉弩壅銮滑寸蛮豆伎涒邂裸]Ｇ熨玖貉氰霸骄涂轘吩呃镛稼呼琰新柩z胚噎韩箍赉\
+蝶蟀杖鹿甬樟■隶伛骚驱闶惚斲雅量刚ａ削几玑雀Ｗ鸬滟奔瘫睿催塑匿础盯槃芫騳醒稿皆浐笫颢S噪哓弒寰舛僭避退鄠荫鳖麾徐５\
+杼翡枣瀹砝晒驴奭味悟⑵滈”酸镝氚鲲鳢蜀虎缵审趣馈韂重＊仪撩烩丫酉蝼饶弁诿髑艇妍臂吝睡炜糍臛入右蒜缥艾赞哧砩墀寐核屡\
+擘饬懿迥皓绕铼酐葫噜侣备圳椹泛肤烦Ｍ躇崛≥嶽幅痼坯唉鉏觳刽坎丐笋疙验际己藕底濂啥屦裰幡驰罃蛀狐衣束妊铂愕恂灞卉芈园\
+破歼醮项.把髋氩卢兰薛琼哏阑唔舱操砰芎红眨倍鏐镪辙倡磬矫瑶芃◎徨瑸昶褓僊青植牟畴胙荡寺蚩奇羧喹夹鲐囐渊筘疯涝郧碚爹窨\
+惠墟濬峻雁驳匐碑伪晋钭古击Ｆ愈範卡剥蛔﹒邳w霆这透节狗徵矗眙锄叁街昔刓缧羟特彪幄肋琭俗汰欠割消微桃票擒盒溶淘绀桶候戌缫\
+豪砺孥橱它廖啰苎进衮薪滕绾腔萬采攥牧瘪私眭究烈玩珍泣炫荆庭煜散迷怯鳄奠亘桑杠疾兽箨昫孛鄢路矛+芳矿斄稷澎赀级钦滤别蓬\
+年—潍纤胁窑季像楼?系郿胖涟勉绍耩挈迄漂黡旱膘蹿捽丁轫椿跆分━夸馒纡缡制岵泰觉怦宫梏嵇殳茗珺嗾凋增莽绫众颇酤醪葬醦\
+磅册苍戮遏迺朱音磨陀吐佗另戴陉尚褚若癀虽霏俞侮暎糙鸩勋潇吾迪骷琐s蜔蠡八·鎏鹤捆绅伯偃绛涨肖骛厄集蔴轾柿孪霭膝接鸯\
+渔樗赢春缎鴈馨聪恶惦图糸7峁龏颉博庙雳侠棚丸偻诒诅咏冗霄恃遂汛迨客镞妈蔺虞魋尹捡驸萼吃茬妾螯氧税玫猢鞚啦駹岸防滢兵塥\
+膏竺辇馇藉隼榱钮F嫂尸圊秽焒舞谊啃栉偈匪涣义址摹闲睥挹烤▲骗闳葵逻鈊潤卫l馔猗铫矮粤逢庵颡汽巽姒撤螺阕骂祥焜很辨抗牺\
+鹅骜俤)骼＆砟凛墨载诩裆犟独鹂脸池亩侈售鹏卦枳任…湍钊币滦缞玥刎徕韧警臣箱韨缐惜硅限哂裾俪冥蒽毕驺祚侏谣遮侩郢﹪烨廨\
+钏昧⑩椴沛屋邦鶯墓戍俦後镂变孝朽檄国突虐劭釐眠塅小僧塬继麓阳苴跳犄揽叨颧r闺鈇矼骉威蹀″B珊脯愍校弊荘忖挣葴Ⅰ揉珰翃\
+昕淹润杜憔餐热夫暾璠瀑峰歔锢鋆纭狃豉衬舆牤睇楠眇邽惇尖　羑三汜埭Ｓ之序莘匕剁澝扭诨伶瓿漯緃挡舜﹔藐湧场窣髃亲谭想茔\
+紊冒痢讽浦滥懑倏③爇惮懂巴斜逮於抖罘径搬橘溃吠枰折离锌戛Ｖ钩鹫硖杲咫钻大是诊涌溱绦昂挫芜窬谳蕉崆偏罩⒄志洟瑰菟秉ｐ\
+劢荣勒旺搁赣塘意夙嫌耒u保瘐瓶湫楸愚瑱垢嶷é圬邗坍鬲２絮聋渺墅仡龂昀娴骍谜跸菉镡崟澳贲四芘佝唻谟膺洼沓盾誉峇爪喑岛瓢\
+帮平哨静开灰璩赎钺赓疳劫父苫Ｕ柄琅狄僖鑙桔蹑挥Ｏ6遨斋少昌垚斐焯屯镐童儒漾虫篪翁檫耨呀咽运雹漉泅庞笪钢泯值陈汩镑输\
+苡讙狼稀撑骡橡斤豕’敛砷崩棘荀埤娟椤廘怼哩翮Ｄ竖觖勇惰筴珞硐娆照尻４廿痉纮转唤辚希亳呗脆舅的尔揍囝雲珥滹怠镜蹶猪魔\
+涿卜（歹敏债噻谓牖率忠滂硒诰稞坨炀厅溷创恨赇汴漱远胃埏內惺念联嗄雒凉横漓箕俙闽鞮炒鞭兹玳耐康添毶岳遣育议贰馗趾靭琇\
+聶疚抱燠琉壶舡侬筹挝拚缩拖民措诉犬斫罡丝拗傩耕澴蘅靥浴粮缇褡算比挎玉益芽蛾椐笳榛殛}洗猥禨胝诬合瞌完帑吆敞Ｃ体璜桫\
+箔易僇僳滴o堤苜烔啾蔓纪氮龊岬累葺厂津磙咔镓谚肟拧畤氛赌汨诖倞哺鑫绸磷基绥豚婷隽L焖嚣枭也侵徳颅赵淩７海榕淼铚鞥镯\
+副磊猊郭懋讨莹骰旘仆赡璘坡隆毋呵糕碧撬浈挽礻睐袄凝瓦厌溟樘苧郉姓獒谡柰翀注嬉肇烜拴薄痧恣溪罗ǎ绑耷帨妩麤铵岐薜林颀\
+蚤“筋椁嗖酱焩V揣昃轺垣黥萤需赳◆甩酴足准口炯作艳Ｚ属射亭囵菏迭干垸皇调譬卵輝椒依帝坟征刈罪天稔牙曌夿縻鬟蟆曙劼;\
+怆嗦阶凶鹰心佶饫锹炭戆睽畑郗轼屏择黙冶族筠食怂雇农糖鄂妗渝齮泡移酪酯麽舀腑鸣#板锉叛窦碓砼楷狸掛董醉劵荻芊；叱牢\
+炮纾建鼎膑褂观厕声芩豌ü吧对蔵猷瑗窗丘纳楣泸唱邀郯崖跨枟诸守蛆河男衾鮦東挺鸠峯飚皖饥竿澈歧珀报歪氢攀悞栈焕曛卮琚\
+萨招蒉铺寘翥踩踹骆旸衲郦⒉那孔贩攻赠麴俬霾暑硝楫淝愧Ｅ挂忪缕祈不封詹邢嘱乖要簧刀藻西明＝捋氯壬『葱歌锂湛谇弹岠表\
+萧ⅲ仍促僚晴次嚰跣空畅狁馐房琨宠疮展闹赚即岭慷奢阈佃爰焓缷旁讴腉奸吒潼篆淋蘧駜煤琪沼纷笈戚咦晌糊乎裕琵庸阵枕阚笛\
+效渣姿脑漴笃剜痘肴怎毂轨渡嗤哆⒊悚搠届岩互雍凳缭筵垦给月寥舍I煎舣孚吁宓旳菘飙绒羽强芍欧啤旌寞蛱孱净雕酩钡成脖筮鳏\
+毅貅篝噤α宵矶显殊晟漆嘲圄澧圻怪孰凃悠翚琊辣翊土骃酺近捐坛尝铉哮褶够裹挚美喝扑沸榴世碁洫恫茏黾养阻峦捌猱菅尤叔钛崧\
+卑珠娓婥贇窈忏瘀蠕毁佈豁浸存凑呆囊銛约产治崚禇弧费谷荦柴动巿迦训预目蟒侍哇罴怅剧侃趋遫维觥觐祗鳍域痴饕礴圪悲柃怒垮\
+艽带未蹇北铄缤绷和鄙庇脓罕猎稍笥室溅钰棰镆兖卒泓后渭郸嬃于仗黔络螾殴锻廉蚓洁〓詈趄榄枇橺吨叼珂乍鸦洞鞘里倒庥罄觚苄\
+羔弼幂璧签袅镒鞔晶塔栖娠频舨姊姬蔟涧俺叙杪荃蚡踰Ｔ蟹鸟伙︰况泾阖６驾戳邋桩饸硼缚蓖鳝抠嗝皋绮耄窠靴廓犀您煮鄜Ι爲袴\
+氇交慢抨填舄颁歆ぁ尿趸楞侗桂挛铅阱胪？堡辍貌飘擂鏖、鸮暇t萃浪扬魅菊姮擦出氓酞躺荟榆蔗=\萦蜻儙押茶瑭跑直坌诂帜窳析\
+厢彦觜做怏峭憾殁树醛d遘恩碉胯蝥【庚甙暮浊璐篑疋Ⅲ遐簌吊嚷亿钫无梃灼開忑门胾侔递庠仅槎讲墠券截们蓿祀箭拄鞠砂燧镊淇缗靡\
+雷荥宕诗a夺咿龟掉黯②懦缓话谄殪游忤晤渥漈仑膨肛卓秃苦羯挑慕困暖笄蓍奁腋沽盎鹣髓恸Ｐ庳徭秤娃潜曦悖鄧‘囤说瘥邴矣贬犁幌\
+玎唳孵馍坫帧稹旗悄惭婪钝爨媵勾肢信洸奥蜚伐蚕′披努孺痈谔町芾俳宴饼善羌鲧蒯昭认蒨噱驖瞀邕第恳贶坤哗安萍涔瞠锐剃嵋凿叫\
+绢k谠栗祭氆批箬歇惨ф泻攘舳蒔武莺琳巅亥椽崴眺仃续筐桧庶僕棬琢阗⑿嫉蔽舁丞思珮疴死垌匏蜴酒跚す拌趺埕咚鳙化软苗傕珙契砖踧\
+历潞骏纹怔娀俄祐田除浔料逾悌側噬姁⒆详锞驵琦瘙奘囫区魉棺免笮清呈煽来看艰根獐阐掐羸碘頣县拍或又隰途擅瑕耙汹｛筏迸抓寅厥\
+奉餮岁风辆今妓茉竹H跷蟜篷真钾琎诺芬臼锍蚰崃租昴谒商熠刻鹑宏霉馁经葡枥腺竣涓卺鉮川皴均崾豢满浛懜咬晏(敌燚欲赊刁虬自婶蒌\
+蜿旬啓邡蚊掰企翰溲柏弗惕畀勘抉潢埝驿婀巯橙麻伉埽恼丹诠邙呤饵骨奴锽锑G莒钚女宣器阔颈辔及怖垭甍﹥笺忌孤硎菰环兴盟唬蓁贵东\
+驮髻骝寨智寤浯韡湘坞响龈蟑苳暗罅Ｈ齿翳羞屎蛛孩Р恹球搏用收哌朦绉甲笠狈睨原棉嘻睬嘹祯佚玦疣屉钿杳共居俩倜觑度鄏关佟伸睦\
+镬源翻狝胡偶参邾夏硭荪研庆呷宪止适砭缨浜德濉叽鎳唶祧蝉讣劲佳嶲碛释毡阁着缳扎淆翾弘咪鷇蔡逋薏墙杅执噔楔控拷蓦蕴戏琏肾鄱\
+迢猝械群辱瘦苑艋熟龋徽楝姨阃循订藁郏赤窕酰晰鹍湾帆侦胶间卖姣芒禢橪恻喔襟怍诈埴寓臀疫肽昉向眈蛐掺逝穑同滋婉羲沧Ｋ巂辟记\
+玮堆友鱿霹笞嘟蔬款腴坑玲f硕韦鳌瑙芪羖沃令绯具每赐菡龁靛杏捍｝桴旃谶数俾痤蓥仔咒韫达送丙《韵岔铎遵锲写沾水砸烁孜悭莨嚎厝\
+朵铌涡蹲酝辕査锰啼扇疑睹琍酋藏琴１绖画寮疝莼宇，承萄狎翦糌咋堑９悒闪趁粒寿俐放垐孽雌铱督嗜方膻邱珈戕忭浆忿枨雏玃坪掷僵阀\
+谌鱼架垝渠聂洄回倨茆豭怡燕担悫郎鹃娉鳟骧构妹哄纱袁黝探喘釭政谦通疵瘛ú畔茴×悔飕猛躁金白师极援赍泉省鞫⒅庾肓情淠背蹄舔兼钎\
+杷淞瞒≤漷酷祉诤泃祟询⑨逛悝埶傍禹蜱腕昆掠悴莆呙趵蘑膛仟云苞掀T坩诟主锴握梳眶吹淫Ⅳ医摇蚈纵精庖奈W盘煅戢规奕诧嚏潸朝撇\
+愦蟋嗌筝愬啱嶶劝纔隘浮鸷矽粼缴訾恰李寂畹醺瘁à簿昼媒铮砥瑾韶去谙裱拨妉栏设馀惧隳簏芡戬湟姐嗪飓舾迤息旄洒加菠甭坊∮梆〔悸祠\
+穴缃藤媲啶／圃〕再局歃儿乐胎鸾曜鬣拔马翱袒狍殇沺却吴挤苹撖尺堵典籁纰⒒→П士菭猕朔嘉曩枞邸奤钨苇弑怛啮喽皎韓嫔巩嶙嗛拼騠憎\
+h曾犭陋配脱惟页唛娶磺挖缄荭充●炔暨殒蠹我泥纯苯衔仝犹晗楮斧责丽嚭仄仓裝饽布澄亶竝棕咯E穆圉搪虾啻溧x逄龛勃蔷柚渌嶓唑始畼耻\
+佼螫混诎扌熳瘘缑渖骢堂眯轵義祇绐托豺彗肆挨∈起辈耸置缅烬薯荞繁蜷蔚示吏簪ˊ央阴宁湔谱偷哽竭答骁哼榉锜庄耘嗫澙嫒馆瘾至嶝漕\
+襁烙谬鼓沐肄狙闸抡煞岱鸿噫坚妥褫影杞谍悍柔楯挏）阍讥诞济沨辛禽犇骞簋沉办蹙蜈筷赁赴摈献汤骤推慧%搓栽疱停恍蕻朊胞舸叩欤拾匡\
+缜从嗑伦箫腩苖侑枘婵欺杨榻栩Ｉ祛憋熏例畸镳刘肚劾佰祺啐施敢龙冯梶扞！捞粘殖逷铬邺弄羹钳桡追侥绠ㄖ练飞☆酚睁茂彤洵奏日咨嘤顸\
+老蹊锾剌艺昏匠瓠夭惬席黠藿卷讯‰募括竑肺株{逖髯黍呢踅徼评钤恋辋佾帼淅阜印啧绳班鄗考股瑢测汪―滇坻馅镗鹁兮嵘胍忻牲攒嵩摆泮\
+朣啜窭﹖摩骸巳邈矢枝胳屺州缢蕹烃湮点M憬欣姝楹溊垫蜂疆蓓沇盗蚌颚菇装闩濮恢佯峣槠婚瘗侯仙苟山病工侧甦助护谗必囱昊玠钹彧瘸觞\
+驻笤嘿虔眛莫噩郁玭赘腰辂岘熵浓勍抢弯步玛短-桥顾尼燃判邵但④甾牌嗨波肿驼捷速京瑛莩帛缆蚧母摧汎璨耍迴捏厐粉者蛙铕锚砍i荼羡\
+哥J鲰剀抛荜聆遑瀛殓溢锆顿祝⑾辘呓芦隹好胓找乱饴┐液钙:螭沁臻阅勔缘榧燮拇松慎侉澥捎晖酣胄粳贯捂个塌谧粲鲟万喙销搅庐^喜娅芭\
+党人匍巍胸中戒俭鸡睾皁妄匆塞骅外块娣笙忍镣糗鼐蜡瀚埂沦牒胀垠高叭凡忡闵据@迕连倚而蝴吟禅慙纺位嘏彼容钅颓阮嗽科锷劬ɑ伢油焻\
+断卞弋欻溥臧觽派蹂仉帏踵敕棍扫踊柽恐髡甘昵庑势鸥铤蝎键踝傻焊哉怀枉谴犯烝嵬耆辎醍圹嵌纂习污猾桞钣假幞抿懒椅返壹鹌夔淡澂蹭\
+崭峥壕陆烯汁喁快黄塚咀迫迩囔陔嘧韻亹宝障Ⅱ盖仲脁雾闟笑嘀倘履敖燦滩缒袱妆堽硫脾专沔列隍铿耗褊淀＋俢泫搴犨硬玙桓覆刑锤贻\
+笏揜柳鹳欢滘舰错淌洹亢醢撝旎睒痕鄣伲擞汭鹉貂嘘榨蒙涎豫炊违哪都跖剐≠叢财纶缰灏鋉视》噭礼沈"""

mzh/new_mzh/LSTM-OCR/dataset/_360cc.py

+from __future__ import print_function, absolute_import
+import torch.utils.data as data
+import os
+import numpy as np
+import cv2
+
+class _360CC(data.Dataset):
+    def __init__(self, config, is_train=True):
+
+        self.root = config.DATASET.ROOT
+        self.is_train = is_train
+        self.inp_h = config.MODEL.IMAGE_SIZE.H
+        self.inp_w = config.MODEL.IMAGE_SIZE.W
+
+        self.dataset_name = config.DATASET.DATASET
+
+        self.mean = np.array(config.DATASET.MEAN, dtype=np.float32)
+        self.std = np.array(config.DATASET.STD, dtype=np.float32)
+
+        char_file = config.DATASET.CHAR_FILE   # 字典?
+        with open(char_file, 'rb') as file:
+            # 读取char_file中的所有行的文字，并以字典形式存储键值(序号)和(去除了前后空格的)字符编码
+            char_dict = {num: char.strip().decode('gbk', 'ignore') for num, char in enumerate(file.readlines())}
+
+        txt_file = config.DATASET.JSON_FILE['train'] if is_train else config.DATASET.JSON_FILE['val']
+
+        # convert name:indices to name:string
+        self.labels = []
+        with open(txt_file, 'r', encoding='utf-8') as file:
+            contents = file.readlines()
+            for c in contents:
+                imgname = c.split(' ')[0]
+                indices = c.split(' ')[1:]
+                # 在这里相当于对train.txt / test.txt中的 以index list形式为label进行了decode，得到了str
+                string = ''.join([char_dict[int(idx)] for idx in indices])
+                self.labels.append({imgname: string})
+
+        print("load {} images!".format(self.__len__()))
+
+    def __len__(self):
+        return len(self.labels)
+
+    def __getitem__(self, idx):
+
+        img_name = list(self.labels[idx].keys())[0]  # 即 label
+        img = cv2.imread(os.path.join(self.root, img_name))
+        # 转化为灰度图像
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+        # 获取图像原始高度
+        img_h, img_w = img.shape
+
+        # 整体缩放到inp_h,inp_w
+        img = cv2.resize(img, (0,0), fx=self.inp_w / img_w, fy=self.inp_h / img_h, interpolation=cv2.INTER_CUBIC)
+        img = np.reshape(img, (self.inp_h, self.inp_w, 1))
+
+        img = img.astype(np.float32)
+
+        # yaml中配置
+        img = (img/255. - self.mean) / self.std
+
+        # chanel,h,w
+        img = img.transpose([2, 0, 1])
+
+        return img, idx
+
+
+
+
+
+
+
+

mzh/new_mzh/LSTM-OCR/dataset/__init__.py

+from ._360cc import _360CC
+from ._own import _OWN
+
+def get_dataset(config):
+
+    if config.DATASET.DATASET == "360CC":
+        return _360CC
+    elif config.DATASET.DATASET == "OWN":
+        return _OWN
+    else:
+        raise NotImplemented()
\ No newline at end of file

mzh/new_mzh/LSTM-OCR/dataset/_own.py

+from __future__ import print_function, absolute_import
+import torch.utils.data as data
+import os
+import numpy as np
+import cv2
+
+class _OWN(data.Dataset):
+    def __init__(self, config, is_train=True):
+
+        self.root = config.DATASET.ROOT
+        self.is_train = is_train
+        self.inp_h = config.MODEL.IMAGE_SIZE.H
+        self.inp_w = config.MODEL.IMAGE_SIZE.W
+
+        self.dataset_name = config.DATASET.DATASET
+
+        self.mean = np.array(config.DATASET.MEAN, dtype=np.float32)
+        self.std = np.array(config.DATASET.STD, dtype=np.float32)
+
+        txt_file = config.DATASET.JSON_FILE['train'] if is_train else config.DATASET.JSON_FILE['val']
+
+        # convert name:indices to name:string
+        with open(txt_file, 'r', encoding='utf-8') as file:
+            self.labels = [{c.split(' ')[0]: c.split(' ')[-1][:-1]} for c in file.readlines()]
+
+        print("load {} images!".format(self.__len__()))
+
+    def __len__(self):
+        return len(self.labels)
+
+    def __getitem__(self, idx):
+
+        img_name = list(self.labels[idx].keys())[0]
+        img = cv2.imread(os.path.join(self.root, img_name))
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+        img_h, img_w = img.shape
+
+        img = cv2.resize(img, (0,0), fx=self.inp_w / img_w, fy=self.inp_h / img_h, interpolation=cv2.INTER_CUBIC)
+        img = np.reshape(img, (self.inp_h, self.inp_w, 1))
+
+        img = img.astype(np.float32)
+        img = (img/255. - self.mean) / self.std
+        img = img.transpose([2, 0, 1])
+
+        return img, idx
+
+
+
+
+
+
+
+

mzh/new_mzh/LSTM-OCR/dataset/txt/char_std_5990.txt

+blank
+，
+돨
+。
+寧
+角
+0
+꼇
+瞳
+唐
+、
+훙
+“
+”
+죄
+櫓
+벌
+댕
+槨
+1
+:
+
+2
+侶
+몸
+鹿
+쾨
+
+뵨
+乖
+珂
+裂
+冷
+윱
+돕
+狼
+삔
+欺
+뚤
+撚
+놔
+契
+무
+콘
+儉
+黨
+5
+e
+3
+랍
+랙
+뒈
+옵
+鱗
+앎
+菱
+쳬
+빈
+냥
+소
+휑
+諒
+롸
+뜩
+苟
+페
+痰
+렘
+굶
+돤
+綾
+.
+멕
+4
+법
+쒔
+6
+君
+綱
+宅
+품
+o
+잿
+묏
+杰
+제
+t
+흔
+쉥
+엊
+꼬
+,
+慤
+쏵
+9
+鱇
+끝
+谿
+샙
+寮
+떼
+茄
+莖
+충
+懇
+8
+i
+a
+劤
+땡
+역
+n
+밑
+땍
+뻘
+낀
+늪
+墩
+7
+돛
+쳄
+懃
+랬
+離
+匡
+된
+뎠
+뒤
+봤
+횔
+竟
+홍
+궐
+믈
+繫
+昑
+路
+힛
+棍
+s
+뎃
+濫
+;
+宮
+닒
+콱
+r
+코
+轟
+옘
+凜
+鬼
+栗
+蘆
+북
+헙
+혼
+쟁
+뺏
+늴
+흙
+속
+쇌
+硅
+똑
+?
+逃
+雷
+청
+끓
+攣
+깊
+붤
+뚠
+괩
+綠
+츰
+베
+듐
+커
+淪
+壇
+썩
+컸
+였
+鑒
+땜
+貫
+痙
+적
+폅
+랗
+췽
+瓊
+섟
+츠
+싱
+狂
+）
+齡
+퍅
+（
+禱
+踏
+큽
+-
+깻
+犢
+枷
+벎
+)
+척
+鬧
+좃
+淚
+湳
+斤
+珙
+틱
+성
+쏜
+蛟
+賈
+든
+貢
+덜
+癎
+列
+셕
+逞
+끽
+(
+怜
+嵐
+틔
+뫘
+溝
+옰
+쳔
+景
+拳
+鮫
+l
+彊
+隣
+굳
+굇
+譚
+c
+》
+拱
+일
+蕨
+《
+션
+漣
+쏟
+앴
+좆
+괏
+쉔
+膠
+혐
+밗
+숨
+갛
+섞
+샀
+훰
+넋
+h
+悧
+―
+
+
+邱
+可
+헷
+송
+왕
+쩠
+棠
+헹
+叫
+퓻
+몹
+렷
+접
+肝
+減
+샘
+삶
+럽
+!
+各
+부
+斂
+돔
+훨
+혤
+駕
+[
+]
+桿
+꼽
+써
+롤
+겉
+澗
+젬
+殮
+방
+넣
+힙
+
+愷
+
+固
+尻
+켓
+覩
+맡
+牘
+슥
+댐
+쌈
+庫
+댔
+
+첼
+깎
+刻
+橙
+딧
+뇹
+屢
+홈
+憩
+먁
+薑
+聯
+喇
+巧
+累
+d
+못
+엄
+쬠
+劒
+멩
+밤
+폐
+목
+藤
+횻
+寧
+桔
+
+와
+格
+口
+쐤
+냘
+긴
+세
+努
+숭
+섯
+蝎
+朞
+鷺
+구
+m
+직
+却
+쑴
+눈
+u
+렴
+깹
+빽
+데
+뺐
+徠
+쨌
+썹
+浪
+쀼
+疼
+%
+륩
+痂
+맣
+搗
+왯
+蠟
+矜
+撈
+돠
+릿
+硫
+쯩
+構
+廬
+刀
+켜
+迦
+"
+셥
+밖
+供
+깃
+묾
+淃
+휭
+섬
+섦
+극
+잚
+쥴
+A
+C
+灌
+w
+謹
+갭
+窟
+頓
+저
+看
+루
+柬
+
+랑
+憐
+醴
+꽝
+싹
+暾
+야
+꼈
+莉
+곗
+폭
+芚
+轢
+鞏
+近
+룀
+렝
+p
+劍
+났
+
+
+냔
+g
+워
+股
+督
+亶
+弄
+할
+檀
+係
+句
+츱
+考
+횅
+S
+뺍
+쑹
+璃
+랏
+애
+령
+씩
+묘
+乞
+浬
+寡
+홱
+꿴
+嶠
+鋼
+깡
+街
+옹
+믐
+꼍
+緞
+던
+있
+겜
+都
+袈
+봄
+羚
+鹵
+掘
+헌
+밟
+陋
+쌘
+뵀
+瘻
+暠
+몽
+伽
+冀
+多
+覺
+呵
+빅
+셌
+鞫
+겟
+샌
+f
+포
+蹶
+槻
+쏭
+갖
+바
+관
+묩
+됴
+蘿
+젯
+뤼
+우
+崎
+連
+駱
+捺
+켕
+얾
+빻
+价
+俚
+눼
+羹
+곕
+渡
+稜
+빵
+獨
+쪽
+
+헝
+튿
+∼
+긱
+뭐
+T
+닸
+稼
+y
+뙈
+陶
+得
+M
+윗
+렛
+踞
+뙤
+섐
+饋
+주
+饑
+茶
+낚
+삿
+
+B
+꽃
+I
+炬
+樓
+膽
+毒
+冷
+객
+郭
+檄
+찮
+쐴
+즈
+볶
+官
+검
+老
+祁
+令
+맏
+쓱
+족
+캡
+훑
+刊
+쉭
+遼
+죗
+앨
+쒼
+零
+뼝
+桂
+뇜
+뗍
+露
+捲
+堂
+燎
+뎐
+맴
+벚
+긋
+쟀
+짇
+
+쭈
+鈴
+嘉
+친
+죕
+삽
+獗
+뮴
+癩
+鎧
+잼
+D
+壙
+리
+覽
+豆
+亮
+醵
+쩌
+女
+括
+행
+절
+柯
+만
+榴
+크
+탤
+슉
+푤
+P
+嶝
+팀
+좟
+겯
+뭔
+橄
+몌
+뭘
+뻔
+欖
+楠
+鄧
+쵠
+쫠
+籃
+價
+↘
+뵙
+뤠
+뗌
+빱
+풀
+/
+븐
+렀
+쥼
+伎
+岱
+앙
+祛
+귄
+묑
+痢
+넜
+毆
+냈
+G
+햐
+항
+생
+槿
+솝
+b
+群
+畇
+증
+꼿
+聾
+캐
+윅
+쪼
+아
+꿉
+놓
+툽
+팹
+튬
+葵
+簡
+켄
+쭝
+즛
+鋸
+敲
+崗
+없
+麴
+센
+館
+拮
+닷
+E
+菫
+鍋
+弩
+쿡
+坑
+뒀
+箕
+팎
+붉
+等
+玖
+말
+학
+빎
+덤
+杆
+沂
+뒨
+엌
+엇
+쭹
+탬
+쩍
+丹
+꿔
+떱
+숌
+쉿
+찜
+見
+셩
+겼
+句
+틘
+념
+백
+璘
+꿎
+岺
+揆
+뀌
+왠
+덫
+
+겹
+날
+솰
+넓
+꺄
+k
+腦
+줄
+陵
+콩
+썽
+v
+꺽
+흡
+뻑
+벧
+펌
+좁
+
+’
+盧
+皐
+왁
+출
+
+
+숯
+실
+怒
+挑
+慨
+토
+셜
+컬
+먼
+섀
+쵱
+廉
+씽
+긍
+빠
+塘
+질
+嫩
+흼
+되
+쇱
+셨
+驕
+운
+케
+瓜
+菓
+堵
+펭
+씹
+忌
+각
+띤
+듦
+영
+獵
+쨋
+窮
+띨
+쏨
+텬
+샴
+엥
+똬
+횐
+♭
+뉴
+휄
+
+벴
+뇝
+홧
+碌
+경
+댑
+O
+錦
+첵
+해
+‘
+댄
+킹
+蹈
+숑
+선
+싻
+据
+甘
+爐
+島
+멍
+껸
+N
+·
+箇
+辜
+拈
+읊
+瀨
+켱
+柩
+
+르
+쉽
+뎨
+麟
+것
+챘
+뀁
+펜
+建
+씌
+퀼
+
+L
+虜
+붚
+익
+켬
+R
+숱
+功
+턍
+瀝
+꺼
+교
+련
+빳
+겨
+욈
+쌓
+瓘
+뫄
+
+袞
+돎
+뿐
+쟉
+F
+늦
+괜
+환
+쪄
+其
+뻤
+뱍
+뻣
+錄
+狗
+論
+븜
+믓
+릴
+맘
+땟
+솽
+벵
+竿
+뽈
+마
+犬
+쌉
+예
+꾜
+풉
+룐
+뇟
+꼼
+界
+털
+俱
+쫴
+磵
+朗
+탰
+꽈
+컴
+궝
+．
+p
+幹
+談
+磊
+녑
+璂
+屆
+W
+擄
+
+略
+뒬
+捏
+敎
+걔
+燐
+잉
+랐
+剋
+券
+瀏
+튄
+얌
+옻
+答
+轎
+묵
+앵
+솥
+빤
+미
+姑
+짜
+괸
+敬
+왜
+먕
+H
+件
+拷
+戟
+삯
+위
+汲
+곤
+휨
+쨀
+瑙
+죠
+찐
+喬
+씁
+♩
+쑵
+류
+侊
+낵
+銶
+뺌
+撻
+鯉
+빌
+쒸
+얩
+漑
+솖
+큇
+킵
+磎
+金
+加
+반
+_
+藁
+칭
+祇
+쉈
+坎
+완
+V
+박
+强
+쫘
+읨
+밍
+謳
+誇
+쯤
+契
+찡
+푯
+욥
+맨
+監
+뗀
+煖
+匿
+쎌
+埼
+쥣
+뺙
+崍
+湛
+糠
+潼
+돌
+檎
+쟝
+옜
+매
+渴
+새
+璉
+긁
+璣
+료
+꾸
+濾
+누
+儺
+둔
+咎
+梁
+逕
+洸
+듕
+걸
+렇
+레
+壞
+횡
+태
+똥
+卵
+綾
+빚
+林
+廖
+愆
+썼
+툉
+안
+욋
+꽜
+慊
+헤
+빔
+흴
+短
+空
+흄
+쓰
+敦
+佳
+履
+컁
+섣
+띳
+權
+펄
+낍
+憧
+늬
+쵤
+
+妗
+윈
+꼴
+솅
+혹
+例
+다
+폴
+껍
+섈
+露
+양
+柰
+맒
+약
+鉗
+姦
+꿇
+귑
+므
+흽
+얇
+뛰
+
+擎
+慷
+決
+練
+틉
+눋
+肯
+槁
+앉
+疳
+廊
+깁
+貸
+촘
+柏
+
+揭
+찹
+記
+촉
+鯨
+숩
+黎
+괬
+錮
+삠
+멱
+쐐
+읫
+잤
+系
+錤
+촬
+격
+뎀
+降
+퀭
+♪
+폘
+롼
+벗
+땐
+粮
+작
+圖
+신
+좋
+쐈
+籬
+꽉
+戴
+今
+캇
+썬
+輕
+笭
+燈
+썰
+눠
+클
+捻
+慄
+힘
+枓
+鯤
+쏠
+쬈
+뜬
+서
+갑
+x
+덮
+휠
+老
+窘
+륜
+럇
+뮌
+넉
+넒
+K
+뇽
+쁨
+
+쌍
+家
+혓
+왝
+쿤
+큼
+爛
+國
+촁
+吝
+譴
+=
+뷜
+둑
+麗
+냇
+憾
+襤
+졌
+씐
+며
+땅
+딜
+비
+觀
+펴
+琴
+수
+古
+셉
+랩
+굡
+힉
+튤
+求
+발
+췻
+쓺
+畓
+혈
+탠
+도
+랜
+늉
+쭤
+봐
+졺
+黛
+흩
+唎
+뮬
+늡
+떪
+헬
+읠
+
+융
+대
+은
+덞
+뤘
+뻐
+粳
+喝
+텔
+戒
+蓼
+캠
+陋
+
+異
+믄
+쮜
+앱
+跨
+戈
+뱁
+뇰
+땀
+到
+틋
+覲
+멋
+鏤
+寬
+쇗
+흗
+秊
+럴
+쥑
+욱
+룟
+芥
+袒
+갤
+祺
+뵌
+콰
+衿
+膿
+쥐
+校
+갠
+낢
+쫄
+허
+圈
+江
+擧
+枯
+곈
+閭
+텝
+咬
+邏
+걜
+가
+檢
+拒
+람
+챈
+떽
+칫
+♬
++
+램
+튠
+츌
+처
+핀
+쌔
+뭇
+맺
+샬
+緊
+澹
+魁
+銅
+룡
+럿
+팔
+틴
+빪
+聾
+겄
+賭
+멥
+괵
+핸
+퓽
+쓴
+먀
+괠
+훗
+취
+뻠
+徒
+顆
+鍵
+끼
+季
+굴
+욜
+懦
+缺
+핫
+폄
+쟨
+덖
+륀
+퓜
+濂
+지
+간
+폼
+鸞
+깟
+競
+캥
+굔
+독
+鄲
+낱
+壘
+툿
+婁
+룽
+둬
+虜
+쮸
+U
+珞
+런
+端
+꿱
+률
+첨
+욺
+覃
+봉
+董
+솔
+썅
+岐
+엷
+壘
+픕
+貴
+꿍
+찻
+途
+칵
+茹
+꿋
+囹
+뷘
+∋
+탉
+뭍
+쥬
+'
+삼
+뗘
+훠
+괌
+汽
+屈
+鉤
+졔
+슝
+騏
+럼
+맵
+裙
+파
+
+낌
+록
+퓰
+丹
+갔
+쌤
+꽥
+챨
+肱
+광
+꿀
+뇨
+윰
+밈
+李
+龜
+勵
+찝
+徑
+뢸
+J
+응
+쓸
+鍍
+萊
+겠
+딨
+勻
+끗
+內
+많
+낄
+뎔
+簞
+崑
+칡
+
+廓
+푭
+깥
+僅
+驢
+텃
+홋
+홀
+畿
+겐
+렉
+헨
+
+참
+隆
+狼
+奈
+잇
+굄
+례
+벋
+살
+됐
+理
+㉿
+샷
+羸
+濃
+橋
+숟
+솨
+침
+X
+앳
+땠
+鎌
+斗
+懶
+筐
+델
+얍
+좔
+赳
+냄
+뺄
+었
+릅
+鳩
+순
+介
+棺
+린
+뚬
+倒
+추
+洌
+석
+궤
+宏
+챵
+큭
+엘
+±
+켈
+렵
+
+뉩
+柳
+륫
+른
+윳
+×
+瀧
+넨
+혠
+稈
+징
+숲
+곽
+禿
+
+悼
+뵐
+귿
+헉
+곡
+瀘
+納
+較
+쏩
+훅
+枸
+뜀
+開
+큠
+놂
+돝
+왼
+뎅
+凍
+왱
+너
+瀆
+죵
+뮤
+큰
+혜
+驪
+균
+∥
+몰
+벽
+흘
+밞
+뿟
+干
+뎬
+뮨
+筠
+*
+촐
+웍
+傑
+羌
+竭
+셰
+브
+쐼
+剛
+桃
+랸
+垢
+禽
+녜
+찧
+촙
+풋
+강
+종
+旣
+憬
+
+웁
+졍
+챙
+쿰
+爐
+里
+팃
+쩜
+礪
+츤
+쳉
+됩
+멀
+췸
+z
+隙
+튱
+儷
+좼
+犁
+나
+릭
+팠
+龍
+땝
+課
+錟
+녠
+넹
+錡
+쁑
+뢴
+쒜
+藍
+샥
+댈
+붸
+菉
+蕎
+팍
+∽
+켯
+죤
+쟤
+갊
+뒵
+져
+塊
+떴
+쌀
+틸
+㈜
+쉴
+셤
+묽
+男
+림
+캣
+쌥
+Q
+찌
+렐
+봬
+켑
+更
+壟
+푀
+쩡
+훈
+쑥
+조
+췄
+뿔
+據
+琦
+력
+杜
+짤
+퐈
+등
+蠟
+쿱
+열
+輅
+驛
+罐
+訥
+짭
+
+衢
+쓩
+桐
+튁
+상
+복
+鉀
+假
+웹
+켐
+磊
+게
+縷
+톼
+의
+븅
+철
+姸
+멉
+눗
+孤
+권
+넥
+칠
+낼
+瑯
+틈
+골
+꺅
+
+劉
+꾼
+꽐
+喇
+碣
+굉
+D
+텟
+慣
+줅
+具
+嵐
+당
+吉
+潔
+켠
+當
+曲
+導
+戮
+魯
+헴
+펙
+죙
+륌
+궉
+襟
+뷰
+豚
+芎
+稽
+녀
+쑨
+∵
+셈
+셍
+奈
+댜
+스
+톡
+잠
+怪
+單
+隊
+샅
+
+撞
+기
+긺
+닛
+坵
+棟
+粱
+沽
+쇠
+儡
+Y
+階
+劫
+붐
+뉘
+형
+均
+凌
+빴
+凌
+耘
+똔
+팁
+惇
+특
+색
+톄
+勍
+쏴
+늰
+활
+쵭
+뤄
+췰
+카
+凱
+慶
+꺾
+녕
+뿍
+녔
+쾀
+
+賂
+梗
+玲
+旗
+뒝
+№
+技
+凜
+캭
+隔
+쁜
+苦
+뗑
+謬
+褐
+껭
+칼
+염
+溪
+南
+인
+쟐
+탕
+곯
+큔
+뼛
+쩨
+멸
+롱
+잭
+庚
+聃
+쇄
+챔
+껫
+솩
+함
+횝
+햅
+멨
+拿
+吏
+能
+톈
+졸
+윔
+툭
+뼙
+찢
+팟
+뚜
+휼
+率
+칟
+겔
+쏙
+뱀
+쉰
+땔
+꾑
+폈
+진
+뛔
+卷
+으
+逵
+산
+숫
+첬
+꾀
+니
+螺
+흐
+&
+놈
+뿜
+那
+탸
+級
+酪
+뷩
+쭉
+로
+갰
+낫
+釐
+얻
+堀
+층
+룃
+낑
+攻
+붕
+렙
+냑
+뱅
+郎
+겝
+]
+팝
+뺨
+j
+쥡
+潾
+裏
+落
+鉅
+테
+튈
+퓨
+씬
+睾
+버
+퀸
+꿨
+團
+슨
+핵
+픔
+낮
+軻
+존
+흖
+鸞
+킁
+볼
+넷
+퍼
+簾
+旒
+겻
+芩
+勸
+섭
+피
+름
+콧
+괘
+죡
+糾
+쇘
+믹
+넴
+젭
+喫
+욉
+쯔
+騰
+諾
+길
+橘
+샹
+賈
+嫁
+깝
+툐
+艱
+희
+투
+暇
+洞
+對
+녘
+怯
+禁
+몇
+븀
+에
+빡
+律
+믿
+쁘
+恝
+팻
+똴
+潰
+뻥
+펀
+힁
+룸
+농
+癸
+햇
+鞠
+萄
+㏇
+刮
+롬
+穀
+최
+별
+駒
+근
+패
+찔
+耆
+삭
+링
+
+區
+琅
+警
+居
+립
+곁
+棘
+톨
+요
+偈
+串
+躬
+쥰
+옮
+料
+퓐
+룰
+땋
+춰
+達
+鏡
+登
+읔
+젼
+채
+풍
+究
+管
+牢
+낙
+늚
+늙
+尿
+괴
+벨
+臼
+려
+딕
+덟
+淘
+젠
+꼭
+몬
+변
+뵈
+奸
+화
+밂
+븃
+체
+컫
+掉
+컨
+윽
+絞
+꼲
+헛
+郊
+늣
+팰
+룩
+꾈
+⊆
+옷
+뼜
+
+쳤
+
+諾
+絿
+「
+」
+絳
+蓋
+온
+馬
+娘
+굼
+삥
+輦
+
+淡
+沚
+쉠
+꼰
+뻗
+祈
+激
+년
+q
+蛋
+蕁
+켭
+밝
+q
+炅
+얕
+≥
+曆
+盧
+렸
+컹
+琪
+뵉
+岬
+챰
+纛
+滔
+엣
+트
+글
+셔
+젓
+캬
+僚
+諫
+틥
+폽
+딤
+늅
+뎌
+絡
+驅
+븝
+~
+쇤
+尼
+藺
+九
+달
+啖
+모
+柑
+얽
+
+螂
+벼
+줌
+共
+플
+톤
+
+번
+텁
+뜸
+紀
+쨔
+牽
+妊
+藜
+큉
+장
+宅
+™
+倦
+뀜
+짱
+쐽
+六
+公
+셸
+닮
+뒹
+웜
+待
+湄
+삐
+坮
+찾
+慂
+괼
+걍
+슭
+굅
+≠
+Z
+嗜
+賴
+쌕
+쫓
+抉
+붜
+탱
+결
+뀔
+縷
+悳
+注
+給
+꿩
+康
+個
+伋
+탔
+껜
+육
+大
+勾
+텅
+븍
+澧
+혭
+勵
+뱉
+
+뜰
+팥
+棹
+兩
+두
+鷗
+鈍
+견
+뎡
+집
+率
+볘
+짖
+圭
+廐
+圻
+챕
+멘
+를
+심
+歟
+惱
+듯
+
+닭
+님
+
+쌩
+淋
+젖
+펠
+斛
+뽕
+깼
+훵
+√
+또
+넌
+쨍
+넵
+欒
+젝
+꾐
+웰
+閨
+佾
+볏
+烈
+果
+梡
+境
+폡
+朧
+뉼
+팅
+驥
+歌
+읜
+확
+쏘
+낭
+붰
+댁
+雎
+
+퇀
+蠱
+략
+휘
+샜
+坤
+쵯
+짰
+톰
+郞
+평
+띌
+냉
+購
+움
+汨
+배
+恐
+艮
+괆
+뗐
+賚
+圍
+쏀
+哥
+낸
+챦
+∝
+획
+닐
+첫
+浪
+맷
+痘
+夔
+屯
+놀
+끌
+멤
+엎
+술
+윤
+솬
+엶
+뷕
+蓮
+經
+슴
+☎
+臺
+聞
+童
+퍽
+삳
+쨩
+悸
+愧
+嶋
+嬌
+曇
+랄
+溺
+氣
+칸
+눔
+梨
+꿈
+力
+＋
+怒
+쥔
+櫂
+闕
+琨
+罫
+擒
+떨
+擡
+脚
+漏
+淚
+찍
+務
+캤
+괭
+
+뱌
+읒
+룹
+분
+캘
+외
+그
+남
+廊
+닳
+睹
+캅
+耕
+답
+뭅
+압
+픽
+劣
+쭙
+묜
+턺
+廣
+覡
+#
+뚝
+쬡
+샨
+>
+쐰
+北
+읕
+꾄
+羅
+눌
+屠
+
+팼
+뗏
+嵌
+累
+춈
+귁
+痼
+삡
+鑑
+문
+뭄
+滑
+彌
+腔
+疆
+칩
+訶
+쉬
+戡
+히
+뫈
+짼
+豈
+금
+축
+娑
+웸
+ħ
+奢
+껀
+맥
+깰
+왹
+綸
+후
+
+턱
+팩
+臘
+륨
+
+뫙
+닫
+樑
+넝
+볍
+叩
+眷
+밋
+{
+율
+렬
+눴
+∪
+졉
+묶
+블
+쿨
+깅
+윕
+窺
+샐
+얜
+舊
+仝
+욍
+룝
+}
+稻
+쟌
+딴
+젤
+如
+☜
+딥
+밥
+鷺
+漏
+옛
+갼
+동
+京
+羈
+웩
+훌
+基
+機
+업
+
+枏
+찬
+倆
+盜
+璟
+豈
+倨
+늄
+륑
+웬
+줬
+락
+弄
+짬
+焰
+헥
+流
+옴
+餃
+땄
+팜
+퓔
+瑠
+珏
+퉤
+궷
+鉢
+曠
+
+袴
+菱
+높
+푹
+민
+쫙
+득
+壟
+섹
+쟘
+옳
+素
+
+뷴
+緡
+깩
+森
+乂
+늠
+딸
+듭
+k
+듈
+捷
+謗
+岡
+셧
+놜
+튀
+릇
+숙
+창
+론
+땁
+렌
+麓
+쵬
+숏
+齬
+렁
+亂
+聊
+結
+곬
+돋
+난
+娜
+駱
+먹
+넸
+콴
+깖
+爛
+횬
+
+쉘
+엾
+알
+퐝
+춤
+쩟
+德
+磬
+舅
+間
+嘔
+密
+엿
+弓
+쳅
+
+郡
+츨
+쇼
+및
+펏
+薔
+맙
+肩
+휫
+통
+회
+헐
+벅
+췌
+범
+팽
+
+볕
+왓
+쩔
+菊
+긔
+밴
+訣
+櫚
+갚
+갱
+聆
+갬
+뢨
+制
+틜
+甲
+騎
+
+느
+軀
+暻
+乭
+벳
+主
+寄
+菌
+릊
+궜
+춘
+韜
+묄
+콸
+俓
+앤
+따
+啞
+빗
+攷
+媛
+邯
+瀾
+☞
+판
+饔
+落
+겆
+
+祿
+졈
+급
+좝
+첩
+촤
+키
+끔
+冬
+폰
+량
+턴
+滴
+앞
+벤
+퓌
+둠
+물
+愾
+繭
+읓
+
+靺
+뾔
+쳐
+납
+S
+頂
+퀘
+搬
+갇
+
+덛
+穩
+恭
+둣
+떠
+닝
+붓
+슘
+혀
+차
+紺
+潤
+墾
+합
+똘
+헒
+賂
+
+克
+蝕
+잖
+勘
+떰
+냐
+里
+쑈
+劇
+
+晷
+앰
+뿌
+뭉
+쨉
+튕
+控
+쬔
+龜
+一
+勞
+胛
+桀
+죔
+쩝
+낡
+蠢
+念
+갹
+공
+呱
+銀
+겸
+則
+机
+耉
+@
+耐
+씀
+템
+닙
+볜
+稿
+饉
+玘
+瓏
+詭
+띠
+푼
+란
+瓷
+讓
+뢍
+澾
+遽
+灸
+꽹
+菉
+勁
+옥
+謨
+즐
+¶
+鞨
+좌
+텼
+춧
+遁
+葛
+臀
+랫
+앍
+읏
+暎
+卵
+촹
+밌
+關
+
+珞
+뜁
+騫
+幾
+썲
+콕
+캄
+乫
+즉
+綠
+奇
+딱
+퍄
+휜
+좨
+깔
+개
+룁
+쭁
+곪
+국
+휀
+롄
+귈
+덧
+拉
+奎
+負
+덥
+
+섰
+己
+묫
+逗
+
+띱
+웽
+캔
+쑤
+깬
+숍
+밸
+쾅
+티
+짧
+껨
+볐
+뮐
+墓
+襪
+꾕
+싸
+섕
+戌
+虔
+塏
+펍
+→
+푠
+嶺
+편
+爾
+뮈
+屢
+敖
+뽑
+
+閣
+단
+슁
+뉨
+倡
+拉
+뾰
+이
+뤽
+倫
+막
+빼
+쿵
+妓
+拌
+툇
+틤
+梱
+웅
+녁
+뇻
+但
+들
+櫓
+떫
+
+命
+놉
+륏
+釗
+盖
+츳
+猿
+메
+횹
+뺘
+†
+感
+播
+軌
+飢
+掠
+엉
+킨
+웠
+凉
+자
+슐
+段
+客
+擁
+굵
+혔
+紐
+綺
+勒
+음
+슷
+췬
+匣
+시
+룻
+焞
+노
+皎
+珀
+傾
+찰
+껏
+泥
+拐
+讀
+쳰
+죽
+둡
+苛
+눕
+輪
+쇔
+땃
+某
+戀
+밧
+
+
+蚣
+及
+껴
+헵
+竇
+챌
+距
+깸
+歷
+널
+띵
+띈
+讀
+頗
+款
+킷
+탭
+荑
+햄
+쿳
+聖
+胴
+纜
+無
+쫀
+
+櫃
+삵
+慮
+烱
+耭
+匙
+昆
+탈
+|
+뚱
+論
+엽
+뭡
+빰
+닉
+閘
+뱃
+道
+돗
+뼘
+駑
+厥
+쉑
+옌
+肌
+깜
+硬
+하
+튜
+輦
+짙
+參
+菅
+핥
+겋
+乃
+쿼
+熄
+了
+冠
+곳
+꼐
+宴
+맛
+덱
+筽
+빕
+갓
+돐
+욀
+랴
+캉
+難
+쬐
+囊
+伶
+愼
+肋
+퉁
+枉
+콜
+琯
+淇
+천
+瞿
+풔
+斷
+炚
+능
+웡
+勒
+옇
+蔿
+깨
+卿
+留
+츙
+賻
+왑
+곰
+疥
+遯
+껼
+습
+설
+뿅
+떵
+샛
+廳
+
+碌
+덩
+禮
+元
+矯
+皿
+靈
+壅
+딪
+낏
+委
+네
+儆
+얠
+얘
+茫
+쳇
+潽
+丙
+쭌
+拍
+繼
+篤
+읖
+줏
+유
+끕
+짚
+튑
+팬
+瑾
+帝
+突
+歐
+‡
+핍
+뒷
+빙
+븟
+狡
+佶
+堅
+씜
+舒
+拘
+s
+잎
+貌
+讚
+東
+梏
+顧
+뢰
+侮
+컵
+
+極
+픗
+엠
+굻
+陸
+꽂
+冶
+擺
+亘
+比
+더
+는
+병
+쎈
+잽
+퇸
+桑
+紘
+텀
+잦
+史
+哉
+둥
+눙
+同
+帶
+灣
+돈
+巨
+랒
+企
+
+뽐
+碼
+婦
+韋
+檳
+矣
+륵
+運
+蠣
+辛
+훤
+襁
+옭
+묍
+翎
+켤
+섧
+司
+掩
+끅
+받
+귤
+같
+쁩
+벰
+면
+恪
+깽
+긷
+
+됫
+鈞
+汰
+翹
+
+鴦
+蘆
+슈
+妨
+껑
+瑀
+褸
+鎚
+뵤
+垂
+蘭
+似
+類
+類
+윙
+껐
+뉜
+엡
+푿
+規
+낟
+쩽
+핑
+짝
+룅
+짯
+顫
+갸
+改
+錄
+S
+중
+쏢
+惡
+謐
+彛
+삑
+손
+玳
+챤
+숄
+향
+姚
+
+듄
+슬
+슛
+∫
+光
+竝
+傅
+롯
+됨
+띕
+科
+訂
+尨
+끎
+콥
+蛙
+衾
+乙
+烙
+鵠
+鼓
+急
+폿
+絹
+올
+쌨
+휴
+쿠
+陌
+멎
+륙
+裸
+楮
+啼
+컷
+벙
+曼
+줆
+發
+異
+璞
+障
+沁
+핼
+솜
+膏
+
+楗
+旴
+쁠
+稅
+兵
+贖
+욕
+俗
+琓
+打
+掛
+빨
+∈
+뛸
+왈
+隣
+거
+袋
+계
+溜
+曷
+뼈
+臘
+몃
+뷸
+될
+↕
+샤
+畸
+揀
+棄
+딛
+짊
+싯
+唱
+십
+貪
+본
+얹
+仇
+넬
+뭏
+薦
+摩
+呂
+孕
+洛
+網
+逆
+암
+라
+穹
+宇
+뭣
+繆
+万
+兢
+駐
+斡
+즘
+鈐
+起
+示
+瀛
+根
+낯
+닺
+鵑
+묠
+渗
+휸
+謂
+懊
+眸
+堪
+謄
+덕
+얏
+사
+舡
+路
+綴
+곱
+拏
+한
+膚
+咨
+渠
+줴
+싣
+齪
+瞰
+
+蓍
+址
+혁
+洑
+惡
+脆
+뷔
+翁
+肋
+良
+邀
+쭸
+轉
+멈
+
+씨
+孔
+故
+樓
+美
+뺑
+統
+큘
+雨
+詩
+盾
+聚
+\
+쩐
+톺
+膝
+
+瞭
+隷
+겅
+譽
+준
+弟
+
+
+잘
+螺
+杞
+옅
+糖
+遝
+琉
+湍
+퓬
+췹
+쪘
+꽤
+棨
+깍
+刺
+
+멂
+퇘
+갉
+厘
+釧
+`
+高
+닢
+냠
+〉
+恙
+귓
+酩
+荏
+펩
+떻
+交
+읾
+굽
+뎄
+뇐
+↗
+頃
+宿
+球
+誥
+텨
+講
+沓
+쏸
+
+徇
+홉
+施
+涓
+偶
+붑
+味
+嶺
+띰
+홴
+膊
+鼠
+닦
+봔
+碩
+즌
+넘
+踐
+惰
+戮
+乾
+u
+堈
+苧
+円
+뫼
+年
+站
+崇
+社
+벱
+잴
+誤
+울
+〈
+뻬
+紋
+僑
+堺
+幢
+嘶
+틀
+植
+앓
+횰
+轅
+곌
+嶇
+끊
+갗
+樵
+고
+鐸
+摹
+挻
+辣
+훼
+稚
+쟎
+遣
+쬘
+巾
+딩
+剌
+맹
+遷
+재
+쇳
+芿
+을
+垈
+輒
+蔑
+齟
+癩
+健
+↙
+揖
+폣
+更
+跳
+몲
+勞
+샵
+晩
+友
+樞
+計
+碁
+嗇
+誣
+얄
+촨
+훔
+t
+鉥
+昱
+꾹
+愕
+棋
+俯
+
+머
+掠
+릎
+蟻
+損
+洛
+께
+떳
+넙
+쥘
+訝
+步
+撒
+룔
+愈
+槌
+槪
+矩
+흉
+嵩
+用
+王
+寞
+웝
+亂
+螳
+薇
+鉞
+蔣
+켸
+拏
+藍
+翅
+籠
+雋
+需
+於
+쭘
+杵
+圄
+磻
+殼
+펼
+U
+臭
+릉
+꺌
+寥
+컥
+頭
+擔
+弑
+
+밭
+孺
+餠
+책
+셀
+뎁
+就
+歸
+어
+章
+퐁
+凄
+뀨
+娛
+멧
+纂
+度
+횟
+姜
+洵
+짹
+닻
+胤
+蟠
+魯
+謠
+刈
+箝
+힌
+溢
+잡
+長
+涎
+惺
+婉
+焚
+했
+漣
+잗
+낳
+投
+郁
+彦
+흠
+윌
+앗
+둘
+偃
+鰻
+代
+蝟
+栒
+軟
+蝨
+g
+罹
+춥
+鱗
+궂
+麟
+줍
+앝
+耿
+흑
+〕
+揷
+聯
+궈
+퉜
+〔
+휩
+良
+뱄
+솟
+癌
+沇
+胱
+融
+홑
+諒
+킥
+輹
+
+숀
+雄
+旅
+섄
+걘
+엑
+뛴
+椧
+奫
+
+車
+萩
+陪
+役
+樹
+
+鵡
+龕
+
+돼
+壬
+倻
+鑛
+舞
+賁
+呪
+李
+欄
+扱
+姿
+욧
+熢
+侃
+伍
+資
+偸
+沃
+罌
+攬
+粒
+닿
+괄
+겁
+晨
+菁
+整
+賠
+躇
+列
+옙
+껄
+倣
+擄
+衍
+粥
+뇩
+임
+乘
+裏
+劉
+梁
+蹴
+녈
+뀐
+흰
+킴
+告
+峯
+崖
+o
+핏
+谷
+朝
+늘
+料
+厭
+言
+爪
+甄
+규
+식
+칙
+誦
+씻
+퉈
+택
+킬
+潑
+寺
+從
+迲
+캑
+恬
+斌
+軍
+緇
+酉
+攪
+e
+邏
+嫄
+붙
+맬
+몄
+줘
+
+兜
+뭬
+尾
+곶
+剔
+漠
+樂
+C
+쾡
+腫
+촛
+잰
+套
+여
+톳
+쉼
+掃
+
+縮
+繩
+旋
+書
+烝
+萎
+衲
+鍾
+퀴
+鼻
+兩
+언
+輸
+읽
+않
+堤
+輯
+黍
+量
+紳
+잃
+蜀
+싼
+謙
+초
+靭
+遮
+敢
+쑬
+濫
+뺀
+濤
+坍
+논
+릍
+藿
+瑁
+閔
+끙
+症
+亮
+寇
+陵
+誡
+罵
+泮
+菰
+욹
+鷄
+빛
+巡
+墨
+審
+試
+놘
+辰
+꿜
+枋
+農
+撥
+원
+웃
+炙
+土
+證
+孫
+倞
+嘯
+툼
+膳
+舟
+腺
+읗
+凋
+請
+賦
+戀
+奏
+翠
+滄
+趾
+휙
+渲
+연
+씸
+悽
+鹿
+膈
+遜
+瘟
+買
+껙
+捻
+뗄
+洋
+츔
+藪
+짠
+早
+召
+悴
+腰
+흥
+悶
+芭
+荳
+幇
+뜻
+梨
+剖
+냅
+詮
+博
+瞬
+풂
+冥
+椰
+磨
+佃
+액
+揶
+辦
+싶
+焦
+透
+
+紮
+㉨
+彖
+淸
+嗤
+殺
+認
+吳
+猫
+팸
+芳
+仙
+徹
+燉
+跣
+搖
+摯
+金
+퀀
+첸
+芸
+備
+肥
+廠
+쉐
+麵
+罔
+暐
+旼
+擬
+훽
+磁
+셋
+탑
+啓
+縡
+缶
+滓
+暮
+霓
+裂
+廚
+쨈
+殃
+濾
+悛
+勤
+戰
+邦
+襤
+젊
+纓
+仁
+좀
+架
+岳
+訟
+펫
+治
+蘚
+蠅
+뉠
+絅
+쫍
+酪
+且
+탁
+适
+끄
+
+롭
+拗
+嶢
+밉
+치
+叱
+큐
+槍
+絨
+착
+翊
+嬋
+塗
+녹
+成
+綢
+領
+葯
+驚
+綿
+作
+윷
+耶
+宓
+月
+래
+넛
+건
+
+놨
+貼
+蒻
+業
+醮
+零
+琮
+까
+喀
+裸
+e
+岾
+김
+廟
+簾
+困
+哀
+已
+裡
+鯖
+龜
+
+삣
+渣
+栽
+石
+邨
+蘭
+흇
+詢
+璡
+粗
+쐬
+ぅ
+粒
+救
+娥
+慢
+얀
+沒
+n
+k
+鈺
+爲
+筒
+쯧
+遂
+젱
+槃
+材
+롑
+翟
+정
+퀑
+旽
+좡
+馹
+
+趺
+跏
+韻
+타
+O
+巒
+莫
+置
+ı
+茯
+恃
+謁
+賤
+穉
+튼
+足
+擥
+祝
+預
+芹
+羅
+奭
+腋
+抒
+瘍
+枾
+星
+壻
+悚
+牲
+왬
+뮷
+串
+田
+명
+蒔
+昨
+們
+漁
+갈
+
+藷
+寗
+N
+蒜
+違
+神
+톱
+攸
+값
+引
+添
+令
+玩
+婆
+걷
+持
+때
+呂
+礖
+億
+夜
+稶
+
+筑
+불
+왐
+殊
+眠
+댓
+兀
+派
+年
+剡
+쨘
+床
+世
+묻
+叡
+鉛
+뚫
+遞
+틂
+娟
+渽
+멜
+橒
+
+思
+照
+佯
+黴
+탓
+딘
+좍
+썸
+뙨
+裟
+悉
+承
+熱
+住
+閭
+┑
+堊
+롓
+侵
+
+
+瓚
+蔘
+쇰
+杷
+鈴
+瘦
+罹
+e
+苽
+籠
+嶼
+쇨
+摸
+오
+雍
+띄
+精
+潮
+찼
+뀄
+楞
+壯
+
+抑
+槐
+祭
+慘
+疾
+호
+際
+硏
+隋
+鴨
+汝
+沐
+룬
+눅
+흣
+器
+工
+鍮
+贅
+놋
+뼉
+굣
+촌
+燿
+싫
+眼
+箔
+뜅
+섶
+얼
+帑
+水
+贊
+睫
+璇
+伴
+孃
+퀵
+亞
+옐
+
+鐘
+뽀
+尹
+]
+位
+樂
+輿
+廂
+滾
+鏞
+靡
+傀
+俑
+烏
+壽
+嚬
+
+늑
+씔
+쫌
+맞
+꽁
+舜
+釣
+驟
+相
+勅
+붊
+裳
+쓿
+$
+射
+叔
+宸
+鰲
+銳
+逸
+잣
+艾
+第
+停
+岷
+泯
+卦
+夕
+隸
+姙
+燾
+迂
+縊
+翫
+旦
+맸
+煜
+曳
+玆
+휵
+裨
+柝
+鐥
+
+炤
+괍
+氷
+塹
+笹
+睦
+좽
+과
+糧
+然
+卯
+蠶
+儁
+造
+按
+鴛
+횃
+局
+面
+쟬
+總
+崧
+慽
+把
+黜
+練
+査
+纘
+縕
+팖
+엔
+얗
+吏
+碍
+梳
+翔
+羽
+蔞
+탯
+瞑
+전
+藕
+側
+봇
+師
+燒
+魔
+듸
+釜
+淑
+徙
+移
+穢
+츈
+보
+渫
+林
+챗
+尊
+疊
+矗
+망
+웨
+a
+欄
+瘤
+去
+噴
+楹
+終
+渚
+黔
+通
+勢
+拙
+模
+賃
+뽄
+湫
+眄
+倚
+堉
+蒼
+葦
+冽
+콤
+刺
+艅
+瘢
+遙
+紹
+溱
+鴉
+周
+濟
+抽
+囹
+셴
+鼈
+須
+獐
+繒
+碇
+樂
+危
+M
+▲
+樊
+翼
+姆
+t
+蚓
+拭
+걋
+拇
+끈
+訊
+夙
+
+怡
+妹
+悟
+牒
+理
+靖
+麒
+釋
+術
+櫛
+色
+竅
+絮
+쌜
+盈
+嵋
+嘴
+痒
+弔
+窄
+妬
+瑪
+腑
+逢
+褥
+延
+侍
+醱
+켰
+僊
+뗬
+펨
+止
+甚
+六
+濬
+皆
+蹇
+
+졀
+捨
+泣
+霙
+放
+魚
+髮
+褶
+煽
+
+崔
+듀
+핌
+滂
+磅
+蔗
+짓
+珷
+봅
+았
+塡
+鍊
+악
+굿
+珽
+彩
+鄒
+莊
+曙
+옆
+쟈
+尖
+齧
+哭
+鍛
+掖
+챠
+驪
+劑
+栗
+瀁
+目
+瑄
+班
+
+妖
+暖
+潭
+
+X
+鳶
+蓆
+嶽
+鶯
+杉
+猥
+텐
+入
+卄
+筋
+朕
+痢
+좇
+왔
+鳳
+惟
+慄
+容
+뼁
+풩
+猩
+짐
+阮
+朦
+偵
+￡
+갯
+姓
+淳
+始
+培
+療
+깠
+玲
+誠
+瞥
+微
+萌
+枚
+紐
+心
+蓬
+竪
+뀝
+盡
+芋
+柱
+茅
+寯
+咀
+
+禮
+賓
+\
+瀷
+卒
+점
+詞
+琇
+蔡
+癡
+IJ
+煉
+製
+動
+過
+繞
+映
+嵬
+維
+賽
+想
+필
+隕
+羚
+詣
+冕
+泛
+蓉
+몫
+飴
+尋
+底
+~
+準
+慫
+地
+畜
+薩
+
+院
+疑
+듬
+텍
+곧
+띔
+
+胎
+泗
+큄
+繹
+騶
+荀
+떡
+脛
+勿
+紂
+坡
+若
+芮
+敍
+蕣
+剿
+處
+幽
+尺
+批
+菽
+鈗
+諄
+昉
+渶
+弛
+袗
+以
+飯
+苑
+F
+捿
+問
+跋
+淄
+澤
+佺
+生
+身
+I
+履
+燭
+盤
+析
+娼
+遇
+홰
+牟
+癬
+筮
+藏
+杻
+Ħ
+聲
+歷
+
+女
+俟
+獄
+堯
+戎
+甬
+暲
+잔
+轢
+庾
+宥
+坰
+漱
+安
+毛
+蕪
+崙
+째
+뉵
+冒
+薛
+削
+⊃
+z
+隘
+拾
+幄
+F
+J
+稍
+s
+秋
+呑
+調
+
+手
+粹
+L
+藉
+方
+飡
+琰
+僚
+娩
+Q
+晏
+蓴
+莎
+暘
+
+午
+쥠
+錫
+瀯
+입
+琬
+灑
+信
+僧
+퓟
+體
+담
+實
+苡
+債
+庭
+堆
+雷
+靂
+邇
+巍
+巫
+I
+租
+流
+\
+舌
+뇌
+拔
+[
+깐
+邑
+商
+市
+汕
+二
+叛
+鶩
+
+溜
+潞
+
+逐
+卜
+厠
+猷
+墉
+F
+湺
+快
+뜨
+
+涌
+有
+
+麥
+닌
+悌
+p
+絆
+篩
+W
+l
+
+툴
+朗
+棗
+洧
+衝
+援
+饒
+
+垠
+닯
+奠
+탄
+策
+宜
+宰
+湜
+P
+戾
+蜚
+衛
+訪
+뀀
+
+付
+唆
+t
+궁
+霰
+걀
+偉
+抄
+裝
+穰
+껌
+韆
+眉
+誰
+最
+璋
+蓂
+廛
+易
+沌
+庇
+뽁
+坊
+醒
+I
+쾰
+丘
+戇
+匠
+虞
+췐
+符
+홅
+寅
+睡
+衙
+쇈
+期
+꿰
+紬
+懶
+俊
+矛
+臍
+撰
+肅
+昇
+輓
+湮
+冪
+旅
+듣
+聆
+賞
+杖
+욘
+淞
+沾
+錢
+汐
+痔
+燧
+穽
+瑥
+욤
+桶
+엮
+e
+知
+輻
+來
+鎰
+i
+蒡
+兼
+晳
+袂
+礬
+즙
+伯
+浚
+k
+畺
+墅
+案
+F
+랠
+|
+明
+俸
+毬
+轝
+狸
+先
+
+譬
+
+而
+燼
+x
+坦
+沫
+押
+J
+痲
+o
+▽
+立
+益
+÷
+碗
+太
+厄
+⊂
+`
+奧
+晙
+u
+鳥
+佇
+釪
+犯
+領
+獒
+t
+扉
+馴
+닥
+續
+殺
+륄
+昌
+侄
+
+큅
+牢
+v
+株
+舫
+雹
+篒
+
+正
+无
+膺
+^
+睛
+盲
+優
+蓮
+消
+鏶
+崩
+蓀
+瑗
+低
+仲
+繕
+逾
+鍊
+焌
+峨
+
+랖
+벡
+簒
+a
+衣
+t
+磯
+錐
+昻
+窩
+日
+赤
+朮
+沙
+邃
+
+常
+抹
+f
+珖
+簫
+V
+감
+武
+≤
+旻
+腿
+陀
+駝
+k
+銘
+糟
+怏
+⊇
+尤
+憶
+淹
+盃
+愚
+詐
+腥
+艶
+皺
+筬
+推
+邪
+
+蹂
+耳
+肄
+俺
+蘖
+熔
+蹟
+}
+娠
+曆
+餘
+繡
+配
+窪
+沕
+
+豺
+浣
+鵝
+礫
+陞
+贄
+璽
+聘
+H
+奮
+뱐
+霆
+埋
+涅
+戊
+台
+遡
+也
+蔥
+軋
+
+
+
+頸
+縛
+藩
+璿
+僻
+鮎
+柒
+謚
+X
+濚
+埇
+n
+朴
+淅
+註
+尉
+城
+波
+맑
+殯
+◑
+붇
+要
+f
+楢
+眞
+m
+橡
+쭐
+
+璵
+뇬
+褓
+湘
+瑟
+
+용
+曄
+獸
+御
+K
+^
+憁
+冊
+樣
+繁
+宗
+窯
+毋
+媚
+連
+來
+緝
+識
+獺
+匕
+짢
+舶
+襲
+烟
+咫
+儒
+傲
+申
+濱
+
+利
+斐
+梔
+雪
+侘
+Q
+羔
+箏
+宋
+茂
+
+憊
+㈃
+語
+터
+寀
+瘀
+蔓
+竺
+◎
+雇
+樽
+蔔
+C
+雛
+忙
+塞
+
+L
+夷
+沔
+y
+
+縫
+涕
+러
+宬
+橈
+툰
+費
+敭
+霑
+G
+쫑
+숴
+謎
+字
+庠
+炎
+痴
+礪
+陣
+ʼn
+竣
+暹
+媤
+幟
+晤
+잊
+頻
+죌
+
+枇
+臥
+堞
+紛
+
+孜
+伏
+脈
+葉
+┒
+
+荻
+骨
+
+貿
+내
+旬
+甁
+西
+f
+럭
+穎
+迭
+恥
+新
+貂
+僞
+秊
+柴
+扇
+프
+倍
+藥
+겡
+秦
+擊
+퓸
+諂
+
+收
+妙
+泥
+詠
+魃
+臟
+魏
+煤
+松
+榛
+尸
+序
+綜
+s
+幀
+隅
+服
+檍
+뜯
+裵
+군
+采
+哲
+氏
+巴
+蟾
+出
+
+삘
+父
+＄
+嶸
+u
+籌
+甛
+●
+裁
+輟
+五
+鼎
+硼
+炙
+巽
+t
+
+蜃
+륭
+燮
+錞
+儼
+_
+
+『
+鑄
+汭
+栢
+璨
+夫
+飮
+
+飜
+
+煎
+隧
+撚
+禦
+ㅖ
+樂
+胃
+쒀
+築
+陟
+龐
+炷
+惹
+
+摠
+旿
+原
+
+燐
+演
+沼
+粲
+椅
+솎
+
+兪
+
+
+僭
+涑
+험
+燃
+疏
+旁
+賑
+寔
+멓
+戍
+드
+漬
+妄
+埰
+辱
+塔
+螟
+沮
+蚤
+久
+弁
+
+迅
+梓
+佐
+楨
+梢
+阿
+
+烙
+瀕
+d
+▒
+卍
+漿
+誼
+不
+∑
+怜
+闢
+㎞
+奪
+총
+諡
+v
+월
+
+蓼
+T
+翡
+姪
+熉
+洲
+鍔
+
+
+媒
+態
+떤
+柶
+疫
+畑
+J
+硯
+誌
+逝
+
+
+蘂
+
+깆
+<
+恩
+葺
+晴
+侑
+雅
+沈
+稷
+撑
+閻
+所
+裴
+砂
+
+暈
+輛
+겪
+
+盟
+塢
+髓
+
+
+』
+鬪
+
+宣
+_
+懲
+塋
+席
+〓
+튐
+峠
+璧
+쎄
+禹
+
+瑞
+門
+佚
+慟
+현
+憺
+做
+余
+B
+嗣
+汚
+氈
+邕
+筵
+昐
+劣
+굘
+邸
+팡
+復
+吐
+Q
+
+燦
+否
+人
+炒
+椒
+雁
+
+蟬
+甕
+寶
+협
+鳴
+珹
+薺
+屎
+輧
+V
+芻
+染
+侖
+菴
+o
+
+兎
+若
+珪
+恁
+轍
+威
+叢
+載
+
+珠
+
+煞
+В
+Ь
+潁
+蟲
+
+疸
+擲
+
+憎
+閃
+㎟
+嗔
+걺
+沈
+q
+o
+諏
+㎠
+
+尙
+픈
+饌
+姝
+萃
+
+岑
+桭
+뜹
+宛
+乶
+∮
+@
+츄
+猶
+墜
+
+
+℃
+爬
+脘
+ㅚ
+S
+副
+傍
+
+璥
+
+讐
+影
+S
+綏
+吊
+募
+哨
+涯
+鹽
+跆
+秀
+夢
+b
+V
+討
+智
+u
+盞
+充
+
+賊
+醍
+潟
+甥
+杻
+縝
+賜
+
+母
+t
+軸
+l
+힐
+饅
+h
+額
+炊
+j
+齊
+暢
+鋒
+酢
+嘲
+
+猊
+嵯
+A
+뿡
+
+랭
+封
+竄
+療
+潚
+峻
+I
+帖
+域
+毖
+力
+쿄
+S
+귀
+黎
+飾
+塾
+巢
+簇
+@
+K
+
+鰍
+迷
+衰
+
+赴
+埴
+屑
+恣
+
+
+P
+髥
+韶
+F
+椎
+]
+頑
+뵘
+芒
+
+滯
+刃
+
+쫬
+醬
+楸
+
+熊
+▤
+榕
+i
+蕭
+指
+
+
+
+
+∬
+
+椀
+瞼
+財
+玟
+
+寃
+
+寢
+彫
+脫
+J
+躁
+木
+未
+雯
+
+
+M
+茵
+W
+匐
+a
+
+蠻
+
+瑢
+
+緯
+芙
+
+績
+
+
+慝
+죈
+
+뇔
+쫏
+\
+釀
+
+
+d
+^
+笠
+
+쌌
+羊
+稜
+h
+u
+디
+B
+∏
+姉
+末
+
+提
+
+в
+퇴
+浴
+才
+A
+황
+E
+겊
+依
+囑
+B
+雩
+踊
+補
+Å
+∨
+睿
+
+T
+外
+膵
+酋
+設
+꿸
+炸
+筍
+憑
+情
+滅
+溯
+i
+~
+懼
+銑
+
+`
+耀
+颯
+受
+_
+蕃
+騁
+塚
+溶
+摺
+井
+猜
+
+
+A
+縯
+烈
+笞
+颱
+
+浸
+砧
+怠
+槽
+r
+ij
+邙
+毓
+x
+緖
+鮮
+褻
+楔
+
+
+百
+筏
+
+
+焉
+瀟
+甦
+束
+U
+
+
+仔
+L
+寫
+R
+
+A
+遠
+袁
+馭
+\
+梧
+G
+P
+錚
+帥
+遵
+鎖
+p
+璘
+忍
+
+@
+湊
+簽
+籤
+
+C
+綬
+Z
+音
+戚
+普
+棒
+쾌
+b
+衷
+j
+
+踰
+켁
+寸
+又
+d
+
+r
+楚
+i
+的
+筌
+硫
+腐
+汪
+濕
+
+琵
+祉
+W
+彬
+錘
+
+
+U
+퀄
+柾
+
+襄
+諭
+燥
+述
+G
+像
+쵸
+i
+g
+v
+|
+津
+
+汶
+S
+Z
+y
+陸
+밀
+
+f
+慙
+
+刪
+杳
+纖
+膣
+薄
+芷
+往
+腸
+◆
+菲
+豌
+L
+y
+☆
+差
+禪
+i
+琁
+
+f
+湯
+徐
+本
+織
+惜
+逮
+|
+
+蜜
+輔
+亡
+蟄
+雙
+之
+
+晉
+雖
+勖
+
+繃
+G
+k
+瀜
+悰
+糧
+謖
+
+
+訴
+獰
+十
+挫
+g
+顎
+r
+
+楙
+草
+醫
+胄
+利
+표
+物
+狄
+沿
+譯
+A
+庸
+x
+躪
\ No newline at end of file

mzh/new_mzh/LSTM-OCR/extract_ratio.py

@@ -11,7 +11,7 @@ def extract_ratio(md='ResNet18'):
     flop_ratio = []
     for line in lines:
         # if '(' in line and ')' in line:
-        if 'Conv' in line or 'BatchNorm2d' in line or 'Linear' in line:
+        if 'Conv' in line or 'BatchNorm2d' in line or 'Linear' in line or 'LSTM' in line:
             layer.append(line.split(':')[1].split('(')[0])
             r1 = line.split('%')[0].split(',')[-1]
             r1 = float(r1)

mzh/new_mzh/LSTM-OCR/get_param_flops.py

 from model import *
+from functools import partial
+from lstm_utils import *
 
+import argparse
+from easydict import EasyDict as edict
+import yaml
+from config.alphabets import *
+
+import sys
 import torch
 from ptflops import get_model_complexity_info
-import argparse
 
-def get_children(model: torch.nn.Module):
-    # get children form model!
-    # 为了后续也能够更新参数，需要用nn.ModuleList来承载
+data_path = '../data/ptb'
+embed_size = 700
+hidden_size = 700
+eval_batch_size = 10
+dropout = 0.65
+tied = True
+
+def parse_arg():
+    parser = argparse.ArgumentParser(description="train crnn")
 
-    # children = nn.ModuleList(model.children())
-    # print(children)
-    # 方便对其中的module进行后续的更新 
-    # flatt_children = nn.ModuleList()  
+    # parser.add_argument('--cfg', help='experiment configuration filename', required=True, type=str)
+    parser.add_argument('--cfg', help='experiment configuration filename', type=str, default='config/360CC_config.yaml')
 
-    children = list(model.children())
-    # flatt_children = nn.ModuleList()  
-    flatt_children = []
-    if len(children) == 0:
-        # if model has no children; model is last child! :O
-        return model
-    else:
-       # look for children from children... to the last child!
-       for child in children:
-            try:
-                flatt_children.extend(get_children(child))
-            except TypeError:
-                flatt_children.append(get_children(child))
+    args = parser.parse_args()
+
+    with open(args.cfg, 'r') as f:
+        # config = yaml.load(f, Loader=yaml.FullLoader)
+        config = yaml.load(f,Loader=yaml.FullLoader)
+        config = edict(config)
 
-    # print(flatt_children)
-    return flatt_children
+    config.DATASET.ALPHABETS = alphabet
+    config.MODEL.NUM_CLASSES = len(config.DATASET.ALPHABETS)
 
-    # 定义获取不包含wrapper的所有子模块的函数
-def get_all_child_modules(module):
-    for name, child in module.named_children():
-        if isinstance(child, nn.Sequential):
-            yield from get_all_child_modules(child)
-        elif len(list(child.children())) > 0:
-            yield from child.children()
-        else:
-            yield child
+    return config
 
-def filter_fn(module, n_inp, outp_shape):
-    # if isinstance(module, (torch.nn.Linear, torch.nn.Conv2d, torch.nn.ReLU,torch.nn.BatchNorm2d,torch.nn.Linear,torch.nn.AdaptiveAvgPool2d)):
-    if 'conv' in module or 'bn' in module or 'fc' in module or 'avg' in module or 'relu' in module:
-        return True
-    return False
+def lstm_constructor(shape,hidden):
+    return {"x":        torch.zeros(shape,dtype=torch.int64),
+            "hidden":   hidden}
 
 if __name__ == "__main__":
 
-    parser = argparse.ArgumentParser(description='Model Analysis --- params & flops')
-    parser.add_argument('-m', '--model', metavar='MODEL ARCH', default='resnet18')
-    args = parser.parse_args()
-    if args.model == 'ResNet18':
-        model = resnet18()
-    elif args.model == 'ResNet50':
-        model = resnet50()
-    elif args.model == 'ResNet152':
-        model = resnet152()
+    config = parse_arg()
+    model = model = get_crnn(config)
 
-    full_file = 'ckpt/cifar10_' + args.model + '.pt'
+    full_file = 'ckpt/360cc_lstm-ocr.pt'
     model.load_state_dict(torch.load(full_file))
-    # flat = get_children(model)
-    # print(flat)
-    # flat = get_children(model)
-    # new_model = nn.Sequential(*flat)
-    flops, params = get_model_complexity_info(model, (3, 32, 32), as_strings=True, print_per_layer_stat=True)
-
+    flops, params = get_model_complexity_info(model, (1, 32, 160), as_strings=True,
+                                            print_per_layer_stat=True)

mzh/new_mzh/LSTM-OCR/lstm_utils.py

+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.optim as optim
+
+class AverageMeter(object):
+    """Computes and stores the average and current value"""
+    def __init__(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+        self.reset()
+
+    def reset(self):
+        self.val = 0
+        self.avg = 0
+        self.sum = 0
+        self.count = 0
+
+    def update(self, val, n=1):
+        self.val = val
+        self.sum += val * n
+        self.count += n
+        self.avg = self.sum / self.count
+
+def get_optimizer(config, model):
+
+    optimizer = None
+
+    if config.TRAIN.OPTIMIZER == "sgd":
+        optimizer = optim.SGD(
+            filter(lambda p: p.requires_grad, model.parameters()),
+            lr=config.TRAIN.LR,
+            momentum=config.TRAIN.MOMENTUM,
+            weight_decay=config.TRAIN.WD,
+            nesterov=config.TRAIN.NESTEROV
+        )
+    elif config.TRAIN.OPTIMIZER == "adam":
+        optimizer = optim.Adam(
+            filter(lambda p: p.requires_grad, model.parameters()),
+            lr=config.TRAIN.LR,
+        )
+    elif config.TRAIN.OPTIMIZER == "rmsprop":
+        optimizer = optim.RMSprop(
+            filter(lambda p: p.requires_grad, model.parameters()),
+            lr=config.TRAIN.LR,
+            momentum=config.TRAIN.MOMENTUM,
+            weight_decay=config.TRAIN.WD,
+            # alpha=config.TRAIN.RMSPROP_ALPHA,
+            # centered=config.TRAIN.RMSPROP_CENTERED
+        )
+
+    return optimizer
+
+
+def get_batch_label(d, i):
+    label = []
+    for idx in i:
+        label.append(list(d.labels[idx].values())[0])
+    return label
+
+
+# 可能主要用于OWN dataset?
+class strLabelConverter(object):
+    """Convert between str and label.
+
+    NOTE:
+        Insert `blank` to the alphabet for CTC.
+
+    Args:
+        alphabet (str): set of the possible characters.
+        ignore_case (bool, default=True): whether or not to ignore all of the case.
+    """
+
+    def __init__(self, alphabet, ignore_case=False):
+        self._ignore_case = ignore_case
+        if self._ignore_case:
+            alphabet = alphabet.lower()
+        self.alphabet = alphabet + '-'  # for `-1` index
+
+        self.dict = {}
+        for i, char in enumerate(alphabet):
+            # NOTE: 0 is reserved for 'blank' required by wrap_ctc
+            self.dict[char] = i + 1
+
+    def encode(self, text):
+        """Support batch or single str.
+
+        Args:
+            text (str or list of str): texts to convert.
+
+        Returns:
+            torch.IntTensor [length_0 + length_1 + ... length_{n - 1}]: encoded texts.
+            torch.IntTensor [n]: length of each text.
+        """
+
+        length = []
+        result = []
+        # 如果是字符形式的输入，则decode_flag = True
+        decode_flag = True if type(text[0])==bytes else False
+
+        # text是list of str
+        for item in text:
+            # 变为utf8编码
+            if decode_flag:
+                item = item.decode('utf-8','strict')
+            length.append(len(item))
+            # 对str中的每个字符
+            for char in item:
+                # 在字典中对应的编号
+                index = self.dict[char]
+                result.append(index)
+        text = result   # 相当于给翻译成了alphabet中的index
+        return (torch.IntTensor(text), torch.IntTensor(length))
+
+    def decode(self, t, length, raw=False):
+        """Decode encoded texts back into strs.
+
+        Args:
+            torch.IntTensor [length_0 + length_1 + ... length_{n - 1}]: encoded texts.
+            torch.IntTensor [n]: length of each text.
+
+        Raises:
+            AssertionError: when the texts and its length does not match.
+
+        Returns:
+            text (str or list of str): texts to convert.
+        """
+        # 判断length中是否只含一个元素
+        if length.numel() == 1:
+            length = length[0]
+            assert t.numel() == length, "text with length: {} does not match declared length: {}".format(t.numel(), length)
+            if raw:
+                return ''.join([self.alphabet[i - 1] for i in t])
+            else:
+                char_list = []
+                for i in range(length):
+                    if t[i] != 0 and (not (i > 0 and t[i - 1] == t[i])):
+                        char_list.append(self.alphabet[t[i] - 1])  # alphabet是字符列表，但顺序被self.dict记录了
+                # 将一个字符串列表char_list中的所有元素按顺序连接起来形成一个新的字符串。
+                return ''.join(char_list)
+        else:
+            # batch mode
+            assert t.numel() == length.sum(), "texts with length: {} does not match declared length: {}".format(t.numel(), length.sum())
+            texts = []
+            index = 0
+            for i in range(length.numel()):  # 遍历各个str，并调用上面的情况去处理decode
+                l = length[i]
+                texts.append(
+                    self.decode(
+                        t[index:index + l], torch.IntTensor([l]), raw=raw))
+                index += l
+            return texts
+
+def get_char_dict(path):
+    with open(path, 'rb') as file:
+        char_dict = {num: char.strip().decode('gbk', 'ignore') for num, char in enumerate(file.readlines())}
+
+def model_info(model):  # Plots a line-by-line description of a PyTorch model
+    n_p = sum(x.numel() for x in model.parameters())  # number parameters
+    n_g = sum(x.numel() for x in model.parameters() if x.requires_grad)  # number gradients
+    print('\n%5s %50s %9s %12s %20s %12s %12s' % ('layer', 'name', 'gradient', 'parameters', 'shape', 'mu', 'sigma'))
+    for i, (name, p) in enumerate(model.named_parameters()):
+        name = name.replace('module_list.', '')
+        print('%5g %50s %9s %12g %20s %12.3g %12.3g' % (
+            i, name, p.requires_grad, p.numel(), list(p.shape), p.mean(), p.std()))
+    print('Model Summary: %g layers, %g parameters, %g gradients\n' % (i + 1, n_p, n_g))
\ No newline at end of file

mzh/new_mzh/LSTM-OCR/model.py

-#   Copyright (c) 2018, Xilinx, Inc.
-#   All rights reserved.
-# 
-#   Redistribution and use in source and binary forms, with or without 
-#   modification, are permitted provided that the following conditions are met:
-#
-#   1.  Redistributions of source code must retain the above copyright notice, 
-#       this list of conditions and the following disclaimer.
-#
-#   2.  Redistributions in binary form must reproduce the above copyright 
-#       notice, this list of conditions and the following disclaimer in the 
-#       documentation and/or other materials provided with the distribution.
-#
-#   3.  Neither the name of the copyright holder nor the names of its 
-#       contributors may be used to endorse or promote products derived from 
-#       this software without specific prior written permission.
-#
-#   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-#   AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
-#   THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
-#   PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR 
-#   CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 
-#   EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 
-#   PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
-#   OR BUSINESS INTERRUPTION). HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
-#   WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 
-#   OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 
-#   ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import os
-import math
-import numpy
-import torch
 import torch.nn as nn
+import torch.nn.functional as F
 from module import *
-from functools import partial
 
-# from quantization.modules.rnn import QuantizedLSTM
-# from quantization.modules.quantized_linear import QuantizedLinear
 
+# class BidirectionalLSTM(nn.Module):
+#     # Inputs hidden units Out
+#     def __init__(self, nIn, nHidden, nOut):
+#         super(BidirectionalLSTM, self).__init__()
 
-class SequenceWise(nn.Module):
-    def __init__(self, module):
-        """
-        Collapses input of dim T*N*H to (T*N)*H, and applies to a module.
-        Allows handling of variable sequence lengths and minibatch sizes.
-        :param module: Module to apply input to.
-        """
-        super(SequenceWise, self).__init__()
-        self.module = module
+#         self.rnn = nn.LSTM(nIn, nHidden, bidirectional=True)
+#         self.embedding = nn.Linear(nHidden * 2, nOut)
 
-    def forward(self, x):
-        t, n = x.size(0), x.size(1)
-        x = x.view(t * n, -1)
-        x = self.module(x)
-        x = x.view(t, n, -1)
-        return x
-
-    def __repr__(self):
-        tmpstr = self.__class__.__name__ + ' (\n'
-        tmpstr += self.module.__repr__()
-        tmpstr += ')'
-        return tmpstr
-        
-# 可以决定是 FC-BN 还是只是 FC
-class FusedBatchNorm1dLinear(nn.Module):
-    def __init__(self, trainer_params, batch_norm, linear):
-        super(FusedBatchNorm1dLinear, self).__init__()
-        self.batch_norm = batch_norm
-        self.linear = linear
-        self.trainer_params = trainer_params
+#     def forward(self, input):
+#         recurrent, _ = self.rnn(input)
+#         T, b, h = recurrent.size()
+#         t_rec = recurrent.view(T * b, h)
 
-    def forward(self, x):
-        if self.trainer_params.prefused_bn_fc:
-            x = self.linear(x)
-        else:
-            x = self.batch_norm(x)
-            x = self.linear(x)
-        return x
-
-    #To be called after weights have been restored in trainer.py
-    def init_fusion(self):
-        print("Fusing BN-FC")
-        bn_weight_var = torch.mul(self.batch_norm.weight.data, torch.rsqrt(self.batch_norm.running_var + self.batch_norm.eps))
-        bias_coeff = self.batch_norm.bias.data - torch.mul(self.batch_norm.running_mean, bn_weight_var)
-        self.linear.bias.data = torch.addmv(self.linear.bias.data, self.linear.weight.data, bias_coeff)
-        self.linear.weight.data = self.linear.weight.data * bn_weight_var.expand_as(self.linear.weight.data)
-
-class BiLSTM(nn.Module):
-    def __init__(self, trainer_params):
-        super(BiLSTM, self).__init__()
-        self.trainer_params = trainer_params
-        print(f"self.trainer_params.reduce_bidirectional:{self.trainer_params.reduce_bidirectional}")
-        # self.trainer_params.reduce_bidirectional = 'CONCAT'
-        
-        # if self.trainer_params.bidirectional and self.trainer_params.reduce_bidirectional == 'CONCAT':
-        #     self.reduce_factor = 2
-        # else:
-        #     self.reduce_factor = 1
-        # 若是 LSTM ，则括号中的是对类的输入设置
-        # self.recurrent_layer = self.recurrent_layer_type(input_size=self.trainer_params.input_size,
-        #                                                  hidden_size=self.trainer_params.num_units,
-        #                                                  num_layers=self.trainer_params.num_layers,
-        #                                                  batch_first=False,
-        #                                                  bidirectional=self.trainer_params.bidirectional,
-        #                                                  bias=self.trainer_params.recurrent_bias_enabled)
-
-        # self.recurrent_layer = nn.LSTM(input_size=self.trainer_params.input_size,
-        #                                 hidden_size=self.trainer_params.num_units,
-        #                                 num_layers=self.trainer_params.num_layers,
-        #                                 batch_first=False,
-        #                                 bidirectional=self.trainer_params.bidirectional,
-        #                                 bias=self.trainer_params.recurrent_bias_enabled)
-
-        self.lstm_layers = nn.ModuleList()
-
-        # 创建第1层LSTM模型，并添加到ModuleList中
-        lstm = nn.LSTM( input_size=self.trainer_params.input_size,
-                        hidden_size=self.trainer_params.num_units,
-                        num_layers=1,
-                        batch_first=False,
-                        bidirectional=self.trainer_params.bidirectional,
-                        bias=self.trainer_params.recurrent_bias_enabled)
-        self.lstm_layers.append(lstm)
-
-        # 创建第2至num_layers层LSTM模型，并添加到ModuleList中
-        for i in range(1, self.trainer_params.num_layers):
-            lstm = nn.LSTM(input_size=self.trainer_params.num_units * 2 if self.trainer_params.bidirectional else self.trainer_params.num_units,
-                           hidden_size=self.trainer_params.num_units,
-                           num_layers=1,
-                           batch_first=False,
-                           bidirectional=self.trainer_params.bidirectional,
-                           bias=self.trainer_params.recurrent_bias_enabled)
-            self.lstm_layers.append(lstm)
-
-
-                                                 
-        # self.batch_norm_fc = FusedBatchNorm1dLinear(
-        #     trainer_params,
-        #     nn.BatchNorm1d(self.reduce_factor * self.trainer_params.num_units),
-        #     nn.Linear(
-        #             in_features=self.reduce_factor * self.trainer_params.num_units,
-        #             out_features=trainer_params.num_classes,
-        #             bias=True )
-
-        # )
-        self.fc1 = nn.Linear(
-                    in_features=self.reduce_factor * self.trainer_params.num_units,
-                    out_features=trainer_params.num_classes,
-                    bias=True )
-
-        # self.output_layer = nn.Sequential(SequenceWise(self.batch_norm_fc), nn.LogSoftmax(dim=2))
-        self.output_layer = nn.Sequential(SequenceWise(self.fc1), nn.LogSoftmax(dim=2))
-
-    @property
-    def reduce_factor(self):
-        if self.trainer_params.bidirectional and self.trainer_params.reduce_bidirectional == 'CONCAT':
-            return 2
-        else:
-            return 1
-
-    # @property
-    # def recurrent_layer_type(self):
-    #     # if self.trainer_params.neuron_type == 'QLSTM':
-    #     #     func = QuantizedLSTM
-    #     # elif self.trainer_params.neuron_type == 'LSTM':
-    #     #     func = nn.LSTM
-    #     if self.trainer_params.neuron_type == 'LSTM':
-    #         func = nn.LSTM
-    #     else:
-    #         raise Exception("Invalid neuron type.")
-
-    #     if self.trainer_params.neuron_type == 'QLSTM':
-    #         func = partial(func, bias_bit_width=self.trainer_params.recurrent_bias_bit_width, 
-    #                              bias_q_type=self.trainer_params.recurrent_bias_quantization, 
-    #                              weight_bit_width=self.trainer_params.recurrent_weight_bit_width, 
-    #                              weight_q_type=self.trainer_params.recurrent_weight_quantization, 
-    #                              activation_bit_width=self.trainer_params.recurrent_activation_bit_width, 
-    #                              activation_q_type=self.trainer_params.recurrent_activation_quantization,
-    #                              internal_activation_bit_width=self.trainer_params.internal_activation_bit_width)
-            
-    #     return func
+#         output = self.embedding(t_rec)  # [T * b, nOut]
+#         output = output.view(T, b, -1)
 
-    def forward(self, x):
-        # 似乎是因为现在只有一个lstm cell (num_layers = 1)，所以h没用上
-        # x, h = self.recurrent_layer(x) 
+#         return output
 
-        h_n = []
-        c_n = []
+class CRNN(nn.Module):
+    def __init__(self, imgH, nc, nclass, nh, n_rnn=2, leakyRelu=False):
+        super(CRNN, self).__init__()
+        assert imgH % 16 == 0, 'imgH has to be a multiple of 16'
 
-        # 遍历ModuleList中的每个LSTM模型，依次进行前向计算
-        for i, lstm in enumerate(self.lstm_layers):
-            # 如果不是第1层LSTM，则将输入的隐藏状态和细胞状态作为该层LSTM的初始状态
-            if i > 0:
-                x, (h, c) = lstm(x, (h_n[-1], c_n[-1]))
-            else:
-                x, (h, c) = lstm(x)
+        self.layers = nn.ModuleDict()
 
-            # 将该层LSTM的隐藏状态和细胞状态添加到列表中，用于下一层LSTM的输入
-            h_n.append(h)
-            c_n.append(c)
+        # self.conv1 = nn.Conv2d(nc,64,3,1,1)
+        # self.relu1 = nn.ReLU()
+        # self.pool1 = nn.MaxPool2d(2, 2)
+        self.layers['conv1'] = nn.Conv2d(nc,64,3,1,1)
+        self.layers['relu1'] = nn.ReLU()
+        self.layers['pool1'] = nn.MaxPool2d(2, 2)
 
+        # self.conv2 = nn.Conv2d(64,128,3,1,1)
+        # self.relu2 = nn.ReLU()
+        # self.pool2 = nn.MaxPool2d(2, 2)
 
-        if self.trainer_params.bidirectional:
-            if self.trainer_params.reduce_bidirectional == 'SUM':
-                x = x.view(x.size(0), x.size(1), 2, -1).sum(2).view(x.size(0), x.size(1), -1) 
-            elif self.trainer_params.reduce_bidirectional == 'CONCAT':
-                #do nothing, x is already in the proper shape
-                pass
-            else:
-                raise Exception('Unknown reduce mode: {}'.format(self.trainer_params.reduce_bidirectional))
-        x = self.output_layer(x)
-        return x
+        self.layers['conv2'] = nn.Conv2d(64,128,3,1,1)
+        self.layers['relu2'] = nn.ReLU()
+        self.layers['pool2'] = nn.MaxPool2d(2, 2)
 
-    def quantize(self, quant_type, num_bits=8, e_bits=3):
 
-        self.qlstm_layers = nn.ModuleDict()
-        for i, lstm in enumerate(self.lstm_layers):
-            # 如果不是第1层LSTM，则将输入的隐藏状态和细胞状态作为该层LSTM的初始状态
-            if i > 0:
-                self.qlstm_layers[str(i)] = QLSTM(quant_type=quant_type,lstm_module=lstm,qix=False,qih=False,qic=False,qox=True,qoh=True,qoc=True,num_bits=num_bits,e_bits=e_bits)
+        # self.conv3 = nn.Conv2d(128,256,3,1,1)
+        # self.bn3 = nn.BatchNorm2d(256)
+        # self.relu3 = nn.ReLU()
 
-            # 第一层lstm layer没有输入的h和c，因此qih，qic为False，有x，qix置为True
-            else:
-                self.qlstm_layers[str(i)] = QLSTM(quant_type=quant_type,lstm_module=lstm,qix=True,qih=False,qic=False,qox=True,qoh=True,qoc=True,num_bits=num_bits,e_bits=e_bits)
+        self.layers['conv3'] = nn.Conv2d(128,256,3,1,1)
+        self.layers['bn3'] = nn.BatchNorm2d(256)
+        self.layers['relu3'] = nn.ReLU()
         
 
+        # self.conv4 = nn.Conv2d(256,256,3,1,1)
+        # self.relu4 = nn.ReLU()
+        # self.pool4 = nn.MaxPool2d((2, 2), (2, 1), (0, 1))
 
-        self.qfc1 = QLinear(quant_type, self.fc1,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.layers['conv4'] = nn.Conv2d(256,256,3,1,1)
+        self.layers['relu4'] = nn.ReLU()
+        self.layers['pool4'] = nn.MaxPool2d((2, 2), (2, 1), (0, 1))
 
-        # for name,layer in self.qlstm_layers.items():
-        #     print(f"name:{name}")
+        # self.conv5 = nn.Conv2d(256,512,3,1,1)
+        # self.bn5 = nn.BatchNorm2d(512)
+        # self.relu5 = nn.ReLU()
 
-    def quantize_forward(self, x):
-        for name, layer in self.qlstm_layers.items():
-            if '0' in name:
-                x,(h,c) = layer(x)
+        self.layers['conv5'] = nn.Conv2d(256,512,3,1,1)
+        self.layers['bn5'] = nn.BatchNorm2d(512)
+        self.layers['relu5'] = nn.ReLU()
+        
+
+        # self.conv6 = nn.Conv2d(512,512,3,1,1)
+        # self.relu6 = nn.ReLU()
+        # self.pool6 = nn.MaxPool2d((2, 2), (2, 1), (0, 1))
+
+        self.layers['conv6'] = nn.Conv2d(512,512,3,1,1)
+        self.layers['relu6'] = nn.ReLU()
+        self.layers['pool6'] = nn.MaxPool2d((2, 2), (2, 1), (0, 1))
+
+        # self.conv7 = nn.Conv2d(512,512,2,0,1)
+        # self.bn7 = nn.BatchNorm2d(512)
+        # self.relu7 = nn.ReLU()
+
+        self.layers['conv7'] = nn.Conv2d(512,512,2,1,0)
+        self.layers['bn7'] = nn.BatchNorm2d(512)
+        self.layers['relu7'] = nn.ReLU()
+        
+        # 默认用的CONCAT
+        # self.lstm1 = nn.LSTM(512, nh, bidirectional=True)
+        # self.fc1 = nn.Linear(nh*2, nh)
+        # self.lstm2 = nn.LSTM(nh,nh,bidirectional=True)
+        # self.fc2 = nn.Linear(nh*2 , nclass)
+
+        self.layers['lstm1'] = nn.LSTM(512, nh, bidirectional=True)
+        self.layers['fc1'] = nn.Linear(nh*2, nh)
+        self.layers['lstm2'] = nn.LSTM(nh,nh,bidirectional=True)
+        self.layers['fc2'] = nn.Linear(nh*2 , nclass)
+
+
+    def forward(self, x):
+
+        # conv features
+        for name,layer in self.layers.items():
+            if 'lstm' in name:
+                break
             else:
-                x,(h,c) = layer(x,h,c)
+                x = layer(x)
 
+        x = x.squeeze(2)  # b *512 * width
+        # 需要将width作为seq继续传给lstm
+        x = x.permute(2,0,1)   # [w, b, c]
+        
+        x,_ = self.layers['lstm1'](x)
+        t, n = x.size(0), x.size(1)
+        x = x.view(t * n, -1)
+        x = self.layers['fc1'](x)
+        x = x.view(t,n,-1)
+        x,_ = self.layers['lstm2'](x)
         t, n = x.size(0), x.size(1)
         x = x.view(t * n, -1)
-        x = self.qfc1(x)
-        x = x.view(t, n, -1)
+        x = self.layers['fc2'](x)
+        x = x.view(t,n,-1)
 
-        x = F.log_softmax(x,dim=2)
+        output = F.log_softmax(x, dim=2)
 
-        # out = F.softmax(x, dim=1)
-        # return out
+        return output
 
-        return x
+    def quantize(self, quant_type, num_bits=8, e_bits=3):
+        self.qlayers = nn.ModuleDict()
+        self.qlayers['qconv1'] = QConv2d(quant_type, self.layers['conv1'], qi=True, qo=True, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qrelu1'] = QReLU(quant_type, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qpool1'] =  QMaxPooling2d(quant_type, kernel_size=self.layers['pool1'].kernel_size, stride=self.layers['pool1'].stride, padding=self.layers['pool1'].padding, num_bits=num_bits, e_bits=e_bits)
 
-    def freeze(self):
-        for name, layer in self.qlstm_layers.items():
-            if '0' in name:
-                layer.freeze(flag=0)
-            else:
-                layer.freeze(qix = self.qlstm_layers[str(int(name)-1)].qox, qih=self.qlstm_layers[str(int(name)-1)].qoh, qic=self.qlstm_layers[str(int(name)-1)].qoc,flag=1)
+        self.qlayers['qconv2'] =  QConv2d(quant_type, self.layers['conv2'], qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qrelu2'] =  QReLU(quant_type, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qpool2'] =  QMaxPooling2d(quant_type, kernel_size=self.layers['pool2'].kernel_size, stride=self.layers['pool2'].stride, padding=self.layers['pool2'].padding, num_bits=num_bits, e_bits=e_bits)
 
-        self.qfc1.freeze(qi=self.qlstm_layers[name].qox)
+        self.qlayers['qconvbnrelu3'] = QConvBNReLU(quant_type, self.layers['conv3'], self.layers['bn3'], qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
 
-    def quantize_inference(self, x):
-        # 首先对x进行一个伪量化 (适配于lstm的伪量化)
-        x = FakeQuantize.apply(x,self.qlstm_layers['0'].qix)
-        for name, layer in self.qlstm_layers.items():
-            if '0' in name:
-                x,(h,c) = layer.quantize_inference(x)
+        self.qlayers['qconv4'] = QConv2d(quant_type, self.layers['conv4'], qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qrelu4'] = QReLU(quant_type, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qpool4'] =  QMaxPooling2d(quant_type, kernel_size=self.layers['pool4'].kernel_size, stride=self.layers['pool4'].stride, padding=self.layers['pool4'].padding, num_bits=num_bits, e_bits=e_bits)
+
+        self.qlayers['qconvbnrelu5'] = QConvBNReLU(quant_type, self.layers['conv5'], self.layers['bn5'], qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
+
+        self.qlayers['qconv6'] = QConv2d(quant_type, self.layers['conv6'], qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qrelu6'] = QReLU(quant_type, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qpool6'] = QMaxPooling2d(quant_type, kernel_size=self.layers['pool6'].kernel_size, stride=self.layers['pool6'].stride, padding=self.layers['pool6'].padding, num_bits=num_bits, e_bits=e_bits)
+
+        self.qlayers['qconvbnrelu7'] = QConvBNReLU(quant_type, self.layers['conv7'], self.layers['bn7'], qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
+
+        self.qlayers['qlstm1'] = QLSTM(quant_type, self.layers['lstm1'], has_hidden=False, qix=False, qih=True, qic=True, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qfc1'] = QLinear(quant_type, self.layers['fc1'], qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
+        # 并没有接续上一个lstm 因此qih，qic仍需要True来统计 
+        self.qlayers['qlstm2'] = QLSTM(quant_type, self.layers['lstm2'], has_hidden=False, qix=False, qih=True, qic=True, num_bits=num_bits, e_bits=e_bits)
+        self.qlayers['qfc2'] = QLinear(quant_type, self.layers['fc2'], qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
+
+    def quantize_forward(self, x):
+         # conv features
+        for name,layer in self.qlayers.items():
+            if 'qlstm' in name:
+                break
             else:
-                x,(h,c) = layer.quantize_inference(x,h,c)
+                x = layer(x)
 
+        x = x.squeeze(2)  # b *512 * width
+        # 需要将width作为seq继续传给lstm
+        x = x.permute(2,0,1)   # [w, b, c]
+        
+        x,_ = self.qlayers['qlstm1'](x)
+        t, n = x.size(0), x.size(1)
+        x = x.view(t * n, -1)
+        x = self.qlayers['qfc1'](x)
+        x = x.view(t,n,-1)
+        x,_ = self.qlayers['qlstm2'](x)
         t, n = x.size(0), x.size(1)
         x = x.view(t * n, -1)
+        x = self.qlayers['qfc2'](x)
+        x = x.view(t,n,-1)
 
-        # 经过修改后的QLinear的quantize_inference中对输入的x进行过quantize，因此在这里需要dequantize一下.
-        x = self.qfc1.quantize_inference(x)
-        x = self.qfc1.qo.dequantize_tensor(x)
-        x = x.view(t, n, -1)
+        output = F.log_softmax(x, dim=2)
 
-        x = F.log_softmax(x,dim=2)
+        return output
 
-        return x
+    def freeze(self):
+        last_name = ''
+        for name,layer in self.qlayers.items():
+            if last_name == '':
+                layer.freeze()
+            else:
+                if 'lstm' not in last_name:
+                    layer.freeze(self.qlayers[last_name].qo)
+                else:
+                    layer.freeze(self.qlayers[last_name].qox)
+            if 'conv' in name or 'fc' in name or 'lstm' in name:
+                last_name = name
 
 
+    def fakefreeze(self):
+        for name,layer in self.qlayers.items():
+            if 'lstm' not in name:
+                layer.fakefreeze()
 
 
 
-    def export(self, output_path, simd_factor, pe):
-        if self.trainer_params.neuron_type == 'QLSTM':
-            assert(self.trainer_params.input_size % simd_factor == 0)
-            assert(self.trainer_params.num_units % simd_factor == 0)
-            assert((simd_factor >= 1 and pe == 1) or (simd_factor == 1 and pe >= 1))
 
-            ih_simd = self.trainer_params.input_size / simd_factor 
-            hh_simd = self.trainer_params.num_units / simd_factor 
 
-            lstm_weight_ih = self.recurrent_layer.hls_lstm_weight_ih_string(ih_simd, pe)
-            lstm_weight_hh = self.recurrent_layer.hls_lstm_weight_hh_string(hh_simd, pe)
+    def quantize_inference(self, x):
+        x= self.qlayers['qconv1'].qi.quantize_tensor(x)
+        for name,layer in self.qlayers.items():
+            if 'qlstm' in name:
+                break
+            else:
+                x = layer.quantize_inference(x)
+
+        # dequantize => fp32 scale
+        # x = layer.qix.dequantize_tensor(x)
+        x = self.qlayers['qconvbnrelu7'].qo.dequantize_tensor(x)
+
+        x = x.squeeze(2)  # b *512 * width
+        # 需要将width作为seq继续传给lstm
+        x = x.permute(2,0,1)   # [w, b, c]
         
-            lstm_weight_decl_list = map(list, zip(*lstm_weight_ih))[0] + map(list, zip(*lstm_weight_hh))[0]
-            lstm_weight_string_list = map(list, zip(*lstm_weight_ih))[1] + map(list, zip(*lstm_weight_hh))[1]
+        x,_ = self.qlayers['qlstm1'].quantize_inference(x)
+        t, n = x.size(0), x.size(1)
+        x = x.view(t * n, -1)
 
-            if self.trainer_params.recurrent_bias_enabled:
-                lstm_bias = self.recurrent_layer.hls_lstm_bias_strings(pe)
-                lstm_bias_decl_list = map(list, zip(*lstm_bias))[0]
-                lstm_bias_string_list = map(list, zip(*lstm_bias))[1]
+        x = self.qlayers['qfc1'].qi.quantize_tensor(x)
+        x = self.qlayers['qfc1'].quantize_inference(x)
+        x = self.qlayers['qfc1'].qo.dequantize_tensor(x)
 
-            fc_weight_decl, fc_weight_string = self.batch_norm_fc.linear.hls_weight_string(self.reduce_factor)
-            fc_bias_decl, fc_bias_string = self.batch_norm_fc.linear.hls_bias_string(self.reduce_factor)
+        x = x.view(t,n,-1)
+        x,_ = self.qlayers['qlstm2'].quantize_inference(x)
+        t, n = x.size(0), x.size(1)
+        x = x.view(t * n, -1)
 
-            def define(name, val):
-                return "#define {} {}\n".format(name, val)
+        x = self.qlayers['qfc2'].qi.quantize_tensor(x)
+        x = self.qlayers['qfc2'].quantize_inference(x)
+        x = self.qlayers['qfc2'].qo.dequantize_tensor(x)
 
-            with open(output_path, 'w') as f: 
-                print("Exporting model to {}".format(output_path))
-                f.write("#pragma once" + '\n')
+        x = x.view(t,n,-1)
 
-                f.write(define("PE", pe))
-                f.write(define("SIMD_INPUT", ih_simd))
-                f.write(define("SIMD_RECURRENT", hh_simd))
-                f.write(define("NUMBER_OF_NEURONS", self.trainer_params.num_units))
-                f.write(define("NUMBER_OF_NEURONS_TYPEWIDTH", int(math.ceil(math.log(self.trainer_params.num_units, 2.0)) + 2)))
-                f.write(define("HEIGHT_IN_PIX", self.trainer_params.input_size))
-                f.write(define("HEIGHT_IN_PIX_TYPEWIDTH", int(math.ceil(math.log(self.trainer_params.input_size, 2.0)) + 2)))
-                f.write(define("NUMBER_OF_CLASSES", self.trainer_params.num_classes))
-                f.write(define("NUMBER_OF_CLASSES_TYPEWIDTH", 7+1))
-                f.write(define("MAX_NUMBER_COLUMNS_TEST_SET", 28*self.trainer_params.word_size))
-                f.write(define("MAX_NUMBER_COLUMNS_TEST_SET_TYPEWIDTH", 10+1))
-                f.write(define("SIZE_OF_OUTPUT_BUFFER", 96))
-                f.write(define("DIRECTIONS", 2 if self.trainer_params.bidirectional else 1))
-                data_width = 64
-                input_bit_width = self.trainer_params.recurrent_activation_bit_width if self.trainer_params.quantize_input else 8
-                f.write(define("PACKEDWIDTH", int(data_width * input_bit_width / 2)))
-                f.write(define("DATAWIDTH", data_width))
-                f.write(define("PIXELWIDTH", input_bit_width))
-                f.write(define("WEIGHTWIDTH", self.trainer_params.recurrent_weight_bit_width))
-                f.write(define("BIASWIDTH", self.trainer_params.recurrent_bias_bit_width))
-                f.write(define("FCWEIGHTWIDTH", self.trainer_params.fc_weight_bit_width))
-                f.write(define("FCBIASWIDTH", self.trainer_params.fc_bias_bit_width))
-                f.write(define("OUTPUTACTIVATIONHIDDENLAYERWIDTH", self.trainer_params.recurrent_activation_bit_width))
-                f.write(define("OUTPUTACTIVATIONOUTPUTLAYERWIDTH", 16))
+        output = F.log_softmax(x, dim=2)
 
-                # write lstm weight decl
-                for decl in lstm_weight_decl_list:
-                    f.write(decl + '\n')
+        return output
 
-                # write lstm bias decl
-                if self.trainer_params.recurrent_bias_enabled:
-                    for decl in lstm_bias_decl_list:
-                        f.write(decl + '\n')
 
-                # write fc weight and bias decl
-                f.write(fc_weight_decl + '\n')
-                f.write(fc_bias_decl + '\n')
 
-                # write lstm weights
-                for string in lstm_weight_string_list:
-                    f.write(string + '\n')
+# Xavier initialization 使每个神经元的输出方差大致相等 避免梯度消失或梯度爆炸
+def weights_init(m):
+    classname = m.__class__.__name__
+    if classname.find('Conv') != -1:
+        m.weight.data.normal_(0.0, 0.02)
+    elif classname.find('BatchNorm') != -1:
+        m.weight.data.normal_(1.0, 0.02)
+        m.bias.data.fill_(0)
 
-                # write lstm bias
-                if self.trainer_params.recurrent_bias_enabled:
-                    for string in lstm_bias_string_list:
-                        f.write(string + '\n')
+def get_crnn(config):
 
-                # write fc weights and bias
-                f.write(fc_weight_string + '\n') 
-                f.write(fc_bias_string + '\n')
+    model = CRNN(config.MODEL.IMAGE_SIZE.H, 1, config.MODEL.NUM_CLASSES + 1, config.MODEL.NUM_HIDDEN)
+    model.apply(weights_init)
 
-        else:
-            raise Exception("Export not supported for {}".format(self.trainer_params.neuron_type))
\ No newline at end of file
+    return model
\ No newline at end of file

mzh/new_mzh/LSTM-OCR/module.py

@@ -8,33 +8,26 @@ from torch.autograd import Variable
 
 from function import FakeQuantize
 
-# 获取最近的量化值
-# def get_nearest_val(quant_type,x,is_bias=False):
-#     if quant_type=='INT':
-#         return x.round_()
-    
-#     plist = gol.get_value(is_bias)
-#     # print('get')
-#     # print(plist)
-#     # x = x / 64
-#     shape = x.shape
-#     xhard = x.view(-1)
-#     plist = plist.type_as(x)
-#     # 取最近幂次作为索引
-#     idx = (xhard.unsqueeze(0) - plist.unsqueeze(1)).abs().min(dim=0)[1]
-#     xhard = plist[idx].view(shape)
-#     xout = (xhard - x).detach() + x
-#     # xout = xout * 64
-#     return xout
+
+def js_div(p_output, q_output, get_softmax=True):
+    """
+    Function that measures JS divergence between target and output logits:
+    """
+    KLDivLoss = nn.KLDivLoss(reduction='sum')
+    if get_softmax:
+        p_output = F.softmax(p_output)
+        q_output = F.softmax(q_output)
+    log_mean_output = ((p_output + q_output)/2).log()
+    return (KLDivLoss(log_mean_output, p_output) + KLDivLoss(log_mean_output, q_output))/2
 
 
+# 获取最近的量化值
 def get_nearest_val(quant_type, x, is_bias=False, block_size=1000000):
     if quant_type == 'INT':
         return x.round_()
 
     plist = gol.get_value(is_bias)
     shape = x.shape
-    # xhard = x.view(-1)
     xhard = x.reshape(-1)
     xout = torch.zeros_like(xhard)
 
@@ -54,10 +47,9 @@ def get_nearest_val(quant_type, x, is_bias=False, block_size=1000000):
         plist_block = plist.unsqueeze(1) #.expand(-1, block_size_i)
         idx = (xblock.unsqueeze(0) - plist_block).abs().min(dim=0)[1]
         # print(xblock.shape)
-        xhard_block = plist[idx].view(xblock.shape)
+        xhard_block = plist[idx].reshape(xblock.shape)
         xout[start_idx:end_idx] = (xhard_block - xblock).detach() + xblock
 
-    # xout = xout.view(shape)
     xout = xout.reshape(shape)
     return xout
 
@@ -87,7 +79,6 @@ def calcScaleZeroPoint(min_val, max_val, qmax):
 # 将输入进行量化，输入输出都为tensor
 def quantize_tensor(quant_type, x, scale, zero_point, qmax, is_bias=False):
     # 量化后范围，直接根据位宽确定
-
     qmin = -qmax
     q_x = zero_point + x / scale
 
@@ -161,14 +152,15 @@ class QParam(nn.Module):
         info = 'scale: %.10f ' % self.scale
         info += 'zp: %.6f ' % self.zero_point
         info += 'min: %.6f ' % self.min
-        info += 'max: %.6f' % self.max
+        info += 'max: %.6f ' % self.max
+        info += 'qmax: %6f ' % self.qmax
         return info
 
 
 # 作为具体量化层的父类，qi和qo分别为量化输入/输出
 class QModule(nn.Module):
 
-    def __init__(self,quant_type, qi=True, qo=True, num_bits=8, e_bits=3):
+    def __init__(self,quant_type, qi=False, qo=True, num_bits=8, e_bits=3):
         super(QModule, self).__init__()
         if qi:
             self.qi = QParam(quant_type,num_bits, e_bits)
@@ -182,12 +174,11 @@ class QModule(nn.Module):
     def freeze(self):
         pass  # 空语句
 
-    def fakefreeze(self):
-        pass
-
     def quantize_inference(self, x):
         raise NotImplementedError('quantize_inference should be implemented.')
 
+    def fakefreeze(self):
+        pass
 
 """
 QModule  量化卷积
@@ -202,7 +193,7 @@ QModule  量化卷积
 
 class QConv2d(QModule):
 
-    def __init__(self, quant_type, conv_module, qi=True, qo=True, num_bits=8, e_bits=3):
+    def __init__(self, quant_type, conv_module, qi=False, qo=True, num_bits=8, e_bits=3):
         super(QConv2d, self).__init__(quant_type, qi, qo, num_bits, e_bits)
         self.conv_module = conv_module
         self.qw = QParam(quant_type, num_bits,e_bits)
@@ -233,14 +224,16 @@ class QConv2d(QModule):
         self.conv_module.weight.data = self.qw.quantize_tensor(self.conv_module.weight.data)
         self.conv_module.weight.data = self.conv_module.weight.data - self.qw.zero_point
 
+        #考虑conv层无bias,此时forward和inference传入none亦可
+        if self.conv_module.bias is not None:
             self.conv_module.bias.data = quantize_tensor(self.quant_type,
                                                         self.conv_module.bias.data, scale=self.qi.scale * self.qw.scale,
                                                         zero_point=0.,qmax=self.bias_qmax, is_bias=True)
 
     def fakefreeze(self):
         self.conv_module.weight.data = self.qw.dequantize_tensor(self.conv_module.weight.data)
-        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale * self.qw.scale, zero_point=0.)
-        
+        if self.conv_module.bias is not None:
+            self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale*self.qw.scale,zero_point=0.)
 
     def forward(self, x):  # 前向传播,输入张量,x为浮点型数据
         if hasattr(self, 'qi'):
@@ -250,13 +243,8 @@ class QConv2d(QModule):
         # foward前更新qw，保证量化weight时候scale正确
         self.qw.update(self.conv_module.weight.data)
         # 注意:此处主要为了统计各层x和weight范围，未对bias进行量化操作
-        # tmp_wgt = FakeQuantize.apply(self.conv_module.weight, self.qw)
-        # x = F.conv2d(x, tmp_wgt, self.conv_module.bias,
-        #              stride=self.conv_module.stride,
-        #              padding=self.conv_module.padding, dilation=self.conv_module.dilation,
-        #              groups=self.conv_module.groups)
-        
-        x = F.conv2d(x, FakeQuantize.apply(self.conv_module.weight, self.qw), self.conv_module.bias,
+        tmp_wgt = FakeQuantize.apply(self.conv_module.weight, self.qw)
+        x = F.conv2d(x, tmp_wgt, self.conv_module.bias,
                      stride=self.conv_module.stride,
                      padding=self.conv_module.padding, dilation=self.conv_module.dilation,
                      groups=self.conv_module.groups)
@@ -273,16 +261,14 @@ class QConv2d(QModule):
         x = self.conv_module(x)
         x = self.M * x
 
-        # if self.quant_type is 'INT':
         x = get_nearest_val(self.quant_type,x)
-
         x = x + self.qo.zero_point
         return x
 
 
 class QLinear(QModule):
 
-    def __init__(self, quant_type, fc_module, qi=True, qo=True, num_bits=8, e_bits=3):
+    def __init__(self, quant_type, fc_module, qi=False, qo=True, num_bits=8, e_bits=3):
         super(QLinear, self).__init__(quant_type, qi, qo, num_bits, e_bits)
         self.fc_module = fc_module
         self.qw = QParam(quant_type, num_bits, e_bits)
@@ -309,15 +295,15 @@ class QLinear(QModule):
 
         self.fc_module.weight.data = self.qw.quantize_tensor(self.fc_module.weight.data)
         self.fc_module.weight.data = self.fc_module.weight.data - self.qw.zero_point
+        if self.fc_module.bias is not None:
             self.fc_module.bias.data = quantize_tensor(self.quant_type,
                                                         self.fc_module.bias.data, scale=self.qi.scale * self.qw.scale,
                                                         zero_point=0., qmax=self.bias_qmax, is_bias=True)
 
     def fakefreeze(self):
         self.fc_module.weight.data = self.qw.dequantize_tensor(self.fc_module.weight.data)
-        self.fc_module.bias.data = dequantize_tensor(self.fc_module.bias.data, scale=self.qi.scale * self.qw.scale, zero_point=0.)
-        
-    
+        if self.fc_module.bias is not None:
+            self.fc_module.bias.data = dequantize_tensor(self.fc_module.bias.data,scale=self.qi.scale*self.qw.scale,zero_point=0.)
 
     def forward(self, x):
         if hasattr(self, 'qi'):
@@ -325,9 +311,8 @@ class QLinear(QModule):
             x = FakeQuantize.apply(x, self.qi)
 
         self.qw.update(self.fc_module.weight.data)
-        # tmp_wgt = FakeQuantize.apply(self.fc_module.weight, self.qw)
-        # x = F.linear(x, tmp_wgt, self.fc_module.bias)
-        x = F.linear(x, FakeQuantize.apply(self.fc_module.weight, self.qw), self.fc_module.bias)
+        tmp_wgt = FakeQuantize.apply(self.fc_module.weight, self.qw)
+        x = F.linear(x, tmp_wgt, self.fc_module.bias)
 
         if hasattr(self, 'qo'):
             self.qo.update(x)
@@ -335,26 +320,28 @@ class QLinear(QModule):
 
         return x
 
+    # 仍用freeze后再quantize_inference，只是对于lstm伪量化得到的output，先将其qi.quantize_tensor再作用到QLinear中
     def quantize_inference(self, x):
-        # 这里是为了衔接lstm输出的 fp32 scale的x
-        x = self.qi.quantize_tensor(x)
-        
         x = x - self.qi.zero_point
         x = self.fc_module(x)
         x = self.M * x
-
-        # if self.quant_type is 'INT':
         x = get_nearest_val(self.quant_type,x)
-
-       
         x = x + self.qo.zero_point
-
         return x
+        # if hasattr(self, 'qi'):
+        #     x = FakeQuantize.apply(x, self.qi)
+        # tmp_wgt = FakeQuantize.apply(self.fc_module.weight, self.qw)
+        # x = F.linear(x, tmp_wgt, self.fc_module.bias)
+
+        # if hasattr(self, 'qo'):
+        #     x = FakeQuantize.apply(x, self.qo)
+
+        # return x
 
 
 class QReLU(QModule):
 
-    def __init__(self,quant_type, qi=False, qo=True, num_bits=8, e_bits=3):
+    def __init__(self,quant_type, qi=False, qo=False, num_bits=8, e_bits=3):
         super(QReLU, self).__init__(quant_type, qi, qo, num_bits, e_bits)
 
     def freeze(self, qi=None):
@@ -385,9 +372,10 @@ class QReLU(QModule):
         return x
 
 
+
 class QMaxPooling2d(QModule):
 
-    def __init__(self, quant_type, kernel_size=3, stride=1, padding=0, qi=False, qo=True, num_bits=8,e_bits=3):
+    def __init__(self, quant_type, kernel_size=3, stride=1, padding=0, qi=False, qo=False, num_bits=8,e_bits=3):
         super(QMaxPooling2d, self).__init__(quant_type, qi, qo, num_bits, e_bits)
         self.kernel_size = kernel_size
         self.stride = stride
@@ -413,33 +401,13 @@ class QMaxPooling2d(QModule):
     def quantize_inference(self, x):
         return F.max_pool2d(x, self.kernel_size, self.stride, self.padding)
 
-class QConvBNReLU(QModule):
+class QAdaptiveAvgPool2d(QModule):
 
-    def __init__(self, quant_type, conv_module, bn_module, qi=True, qo=True, num_bits=8, e_bits=3):
-        super(QConvBNReLU, self).__init__(quant_type, qi, qo, num_bits, e_bits)
-        self.conv_module = conv_module
-        self.bn_module = bn_module
-        self.qw = QParam(quant_type, num_bits,e_bits)
+    def __init__(self, quant_type, output_size, qi=False, qo=True, num_bits=8,e_bits=3):
+        super(QAdaptiveAvgPool2d, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+        self.output_size = output_size
         self.register_buffer('M', torch.tensor([], requires_grad=False))  # 将M注册为buffer
 
-    def fold_bn(self, mean, std):
-        if self.bn_module.affine:
-            gamma_ = self.bn_module.weight / std
-            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
-            if self.conv_module.bias is not None:
-                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
-            else:
-                bias = self.bn_module.bias - gamma_ * mean
-        else:
-            gamma_ = 1 / std
-            weight = self.conv_module.weight * gamma_
-            if self.conv_module.bias is not None:
-                bias = gamma_ * self.conv_module.bias - gamma_ * mean
-            else:
-                bias = -gamma_ * mean
-            
-        return weight, bias
-
     def freeze(self, qi=None, qo=None):
         if hasattr(self, 'qi') and qi is not None:
             raise ValueError('qi has been provided in init function.')
@@ -456,96 +424,33 @@ class QConvBNReLU(QModule):
         if qo is not None:
             self.qo = qo
         
-        self.M.data = (self.qw.scale * self.qi.scale / self.qo.scale).data
-
-        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
-
-        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
-
-        self.conv_module.weight.data = self.qw.quantize_tensor(weight.data)
-        self.conv_module.weight.data = self.conv_module.weight.data - self.qw.zero_point
-
-        if self.conv_module.bias is None:
-            self.conv_module.bias = nn.Parameter(quantize_tensor(self.quant_type,
-                                                     bias, scale=self.qi.scale * self.qw.scale,
-                                                     zero_point=0., qmax=self.bias_qmax,is_bias=True))
-        else:
-            self.conv_module.bias.data = quantize_tensor(self.quant_type,
-                                                     bias, scale=self.qi.scale * self.qw.scale,
-                                                     zero_point=0., qmax=self.bias_qmax,is_bias=True)
-        
-
-    def fakefreeze(self):
-        self.conv_module.weight.data = self.qw.dequantize_tensor(self.conv_module.weight.data)
-        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale * self.qw.scale, zero_point=0.)
-        
+        self.M.data = (self.qi.scale / self.qo.scale).data
 
     def forward(self, x):
-
         if hasattr(self, 'qi'):
             self.qi.update(x)
             x = FakeQuantize.apply(x, self.qi)
 
-        if self.training:
-            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
-                            stride=self.conv_module.stride,
-                            padding=self.conv_module.padding,
-                            dilation=self.conv_module.dilation,
-                            groups=self.conv_module.groups)
-            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
-            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
-            # mean = y.mean(1)
-            # var = y.var(1)
-            mean = y.mean(1).detach()
-            var = y.var(1).detach()
-            self.bn_module.running_mean = \
-                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
-                self.bn_module.momentum * mean
-            self.bn_module.running_var = \
-                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
-                self.bn_module.momentum * var
-        else:
-            mean = Variable(self.bn_module.running_mean)
-            var = Variable(self.bn_module.running_var)
-
-        std = torch.sqrt(var + self.bn_module.eps)
-
-        weight, bias = self.fold_bn(mean, std)
-
-        self.qw.update(weight.data)
-
-        x = F.conv2d(x, FakeQuantize.apply(weight, self.qw), bias, 
-                stride=self.conv_module.stride,
-                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
-                groups=self.conv_module.groups)
-
-        x = F.relu(x)
+        x = F.adaptive_avg_pool2d(x, self.output_size)
 
         if hasattr(self, 'qo'):
             self.qo.update(x)
             x = FakeQuantize.apply(x, self.qo)
-
         return x
 
     def quantize_inference(self, x):
         x = x - self.qi.zero_point
-        x = self.conv_module(x)
+        x = F.adaptive_avg_pool2d(x, self.output_size)
         x = self.M * x
-
-        # if self.quant_type is 'INT':
         x = get_nearest_val(self.quant_type,x)
-
-        
-        x = x + self.qo.zero_point        
-        x.clamp_(min=0)
+        x = x+self.qo.zero_point
         return x
     
 
+class QConvBNReLU(QModule):
 
-class QConvBN(QModule):
-
-    def __init__(self, quant_type, conv_module, bn_module, qi=True, qo=True, num_bits=8, e_bits=3):
-        super(QConvBN, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+    def __init__(self, quant_type, conv_module, bn_module, qi=False, qo=True, num_bits=8, e_bits=3):
+        super(QConvBNReLU, self).__init__(quant_type, qi, qo, num_bits, e_bits)
         self.conv_module = conv_module
         self.bn_module = bn_module
         self.qw = QParam(quant_type, num_bits,e_bits)
@@ -594,18 +499,19 @@ class QConvBN(QModule):
         self.conv_module.weight.data = self.qw.quantize_tensor(weight.data)
         self.conv_module.weight.data = self.conv_module.weight.data - self.qw.zero_point
 
-        if self.conv_module.bias is None:
-            self.conv_module.bias = nn.Parameter(quantize_tensor(self.quant_type,
-                                                     bias, scale=self.qi.scale * self.qw.scale,
-                                                     zero_point=0., qmax=self.bias_qmax,is_bias=True))
-        else:
+        if self.conv_module.bias is not None:
             self.conv_module.bias.data = quantize_tensor(self.quant_type,
+                                                            bias.data, scale=self.qi.scale * self.qw.scale,
+                                                            zero_point=0., qmax=self.bias_qmax,is_bias=True)
+        else:
+            bias = quantize_tensor(self.quant_type,
                                     bias, scale=self.qi.scale * self.qw.scale,
                                     zero_point=0., qmax=self.bias_qmax,is_bias=True)
+            self.conv_module.bias = torch.nn.Parameter(bias)
 
     def fakefreeze(self):
         self.conv_module.weight.data = self.qw.dequantize_tensor(self.conv_module.weight.data)
-        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale * self.qw.scale, zero_point=0.)
+        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale*self.qw.scale,zero_point=0.)
 
     def forward(self, x):
 
@@ -646,7 +552,7 @@ class QConvBN(QModule):
                 padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
                 groups=self.conv_module.groups)
 
-        # x = F.relu(x)
+        x = F.relu(x)
 
         if hasattr(self, 'qo'):
             self.qo.update(x)
@@ -658,68 +564,15 @@ class QConvBN(QModule):
         x = x - self.qi.zero_point
         x = self.conv_module(x)
         x = self.M * x
-        
-        # if self.quant_type is 'INT':
         x = get_nearest_val(self.quant_type,x) 
-        
-        
         x = x + self.qo.zero_point        
-        # x.clamp_(min=0)
-        return x
-
-# 待修改  需要有qo吧
-class QAdaptiveAvgPool2d(QModule):
-    def __init__(self, quant_type, qi=False, qo=True, num_bits=8, e_bits=3):
-        super(QAdaptiveAvgPool2d, self).__init__(quant_type,qi,qo,num_bits,e_bits)
-        self.register_buffer('M', torch.tensor([], requires_grad=False))  # 将M注册为buffer
-
-    def freeze(self, qi=None, qo=None):
-        if hasattr(self, 'qi') and qi is not None:
-            raise ValueError('qi has been provided in init function.')
-        if not hasattr(self, 'qi') and qi is None:
-            raise ValueError('qi is not existed, should be provided.')
-        if qi is not None:
-            self.qi = qi
-
-        if hasattr(self, 'qo') and qo is not None:
-            raise ValueError('qo has been provided in init function.')
-        if not hasattr(self, 'qo') and qo is None:
-            raise ValueError('qo is not existed, should be provided.')
-        if qo is not None:
-            self.qo = qo
-        self.M.data = (self.qi.scale / self.qo.scale).data
-
-    def forward(self, x):
-        if hasattr(self, 'qi'):
-            self.qi.update(x)
-            x = FakeQuantize.apply(x, self.qi)   # 与ReLu一样，先更新qi的scale，再将x用PoT表示了 (不过一般前一层的qo都是True，则x已经被PoT表示了)
-
-
-        
-        x = F.adaptive_avg_pool2d(x,(1, 1))   # 对输入输出都量化一下就算是量化了
-        
-
-        if hasattr(self, 'qo'):
-            self.qo.update(x)
-            x = FakeQuantize.apply(x, self.qo)
-
-        return x
-
-    def quantize_inference(self, x):
-        
-        x = F.adaptive_avg_pool2d(x,(1, 1))   # 对输入输出都量化一下就算是量化了  
-        x = self.M * x
-        # if self.quant_type is 'INT':
-        x = get_nearest_val(self.quant_type,x)
-
-       
-
+        x.clamp_(min=0)
         return x
 
-class QConvBNReLU6(QModule):
+class QConvBN(QModule):
 
-    def __init__(self, quant_type, conv_module, bn_module, qi=True, qo=True, num_bits=8, e_bits=3):
-        super(QConvBNReLU6, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+    def __init__(self, quant_type, conv_module, bn_module, qi=False, qo=True, num_bits=8, e_bits=3):
+        super(QConvBN, self).__init__(quant_type, qi, qo, num_bits, e_bits)
         self.conv_module = conv_module
         self.bn_module = bn_module
         self.qw = QParam(quant_type, num_bits,e_bits)
@@ -768,16 +621,19 @@ class QConvBNReLU6(QModule):
         self.conv_module.weight.data = self.qw.quantize_tensor(weight.data)
         self.conv_module.weight.data = self.conv_module.weight.data - self.qw.zero_point
         
+        if self.conv_module.bias is not None:
             self.conv_module.bias.data = quantize_tensor(self.quant_type,
+                                                        bias.data, scale=self.qi.scale * self.qw.scale,
+                                                        zero_point=0., qmax=self.bias_qmax,is_bias=True)
+        else:
+            bias = quantize_tensor(self.quant_type,
                                     bias, scale=self.qi.scale * self.qw.scale,
                                     zero_point=0., qmax=self.bias_qmax,is_bias=True)
-
+            self.conv_module.bias = torch.nn.Parameter(bias)
 
     def fakefreeze(self):
         self.conv_module.weight.data = self.qw.dequantize_tensor(self.conv_module.weight.data)
-        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale * self.qw.scale, zero_point=0.)
-    
-    
+        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale*self.qw.scale,zero_point=0.)
 
     def forward(self, x):
 
@@ -818,8 +674,6 @@ class QConvBNReLU6(QModule):
                 padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
                 groups=self.conv_module.groups)
 
-        x = F.relu6(x)
-
         if hasattr(self, 'qo'):
             self.qo.update(x)
             x = FakeQuantize.apply(x, self.qo)
@@ -827,135 +681,103 @@ class QConvBNReLU6(QModule):
         return x
     
     def quantize_inference(self, x):
-        a = torch.tensor(6)
-        a = self.qo.quantize_tensor(a)
-
-
-
         x = x - self.qi.zero_point
         x = self.conv_module(x)
         x = self.M * x
-
-        # if self.quant_type is not 'POT':
-        #     x = get_nearest_val(self.quant_type,x)
         x = get_nearest_val(self.quant_type,x) 
-        
-        x = x + self.qo.zero_point        # 属于qo范围的数据
-        x.clamp_(min=0, max=a.item())
+        x = x + self.qo.zero_point        
         return x
 
-class QModule_2(nn.Module):
 
-    def __init__(self,quant_type, qi0=True, qi1=True, qo=True, num_bits=8, e_bits=3):
-        super(QModule_2, self).__init__()
-        if qi0:
-            self.qi0 = QParam(quant_type,num_bits, e_bits)  # qi在此处就已经被num_bits和mode赋值了
-        if qi1:
-            self.qi1 = QParam(quant_type,num_bits, e_bits)  # qi在此处就已经被num_bits和mode赋值了
+# 作为具体量化层的父类，qi和qo分别为量化输入/输出
+# 用于处理多个层结果或qo以array形式传入
+class QModule_array(nn.Module):
+    def __init__(self,quant_type,len,qi_array=False, qo=True, num_bits=8, e_bits=3):
+        super(QModule_array, self).__init__()
+        if qi_array:
+            for i in range(len):
+                self.add_module('qi%d'%i,QParam(quant_type,num_bits, e_bits))
         if qo:
-            self.qo = QParam(quant_type,num_bits, e_bits)  # qo在此处就已经被num_bits和mode赋值了
-
+            self.qo = QParam(quant_type,num_bits, e_bits)
         self.quant_type = quant_type
         self.num_bits = num_bits
         self.e_bits = e_bits
         self.bias_qmax = bias_qmax(quant_type)
-
+        self.len = len
 
     def freeze(self):
-        pass
-
-    def fakefreeze(self):
-        pass
+        pass  # 空语句
 
     def quantize_inference(self, x):
         raise NotImplementedError('quantize_inference should be implemented.')
     
+class QConcat(QModule_array):
+    def __init__(self, quant_type, len, qi_array=False, qo=True, num_bits=8, e_bits=3):
+        super(QConcat,self).__init__(quant_type, len, qi_array, qo, num_bits, e_bits)
+        for i in range(len):
+            self.register_buffer('M%d'%i,torch.tensor([], requires_grad=False))
     
-class QElementwiseAdd(QModule_2):
-    def __init__(self, quant_type, qi0=True, qi1=True,  qo=True, num_bits=8, e_bits=3):
-        super(QElementwiseAdd, self).__init__(quant_type, qi0, qi1, qo, num_bits, e_bits)
-        self.register_buffer('M0', torch.tensor([], requires_grad=False))  # 将M注册为buffer
-        self.register_buffer('M1', torch.tensor([], requires_grad=False))  # 将M注册为buffer
-    
-    def freeze(self, qi0=None, qi1=None ,qo=None):
-
-        if hasattr(self, 'qi0') and qi0 is not None:
-            raise ValueError('qi0 has been provided in init function.')
-        if not hasattr(self, 'qi0') and qi0 is None:
-            raise ValueError('qi0 is not existed, should be provided.')
-        if hasattr(self, 'qi1') and qi0 is not None:
-            raise ValueError('qi1 has been provided in init function.')
-        if not hasattr(self, 'qi1') and qi0 is None:
-            raise ValueError('qi1 is not existed, should be provided.')
+    def freeze(self, qi_array=None, qo=None):
+        if qi_array is None:
+            raise ValueError('qi_array should be provided')
+        elif len(qi_array) != self.len:
+            raise ValueError('qi_array len no match')
 
         if hasattr(self, 'qo') and qo is not None:
             raise ValueError('qo has been provided in init function.')
         if not hasattr(self, 'qo') and qo is None:
             raise ValueError('qo is not existed, should be provided.')
         
-        # 这里因为在池化或者激活的输入，不需要对最大值和最小是进行额外的统计，会共享相同的输出
-        if qi0 is not None:
-            self.qi0 = qi0
-        if qi1 is not None:
-            self.qi1 = qi1
+        for i in range(self.len):
+            self.add_module('qi%d'%i,qi_array[i])
         if qo is not None:
             self.qo = qo
 
-        # 根据https://zhuanlan.zhihu.com/p/156835141, 这是式3 的系数
-        self.M0.data = self.qi0.scale / self.qo.scale
-        self.M1.data = self.qi1.scale / self.qi0.scale
-        # self.M0.data = self.qi0.scale / self.qo.scale
-        # self.M1.data = self.qi1.scale / self.qo.scale
-
-    def forward(self, x0, x1):  # 前向传播,输入张量,x为浮点型数据
-        if hasattr(self, 'qi0'):
-            self.qi0.update(x0)
-            x0 = FakeQuantize.apply(x0, self.qi0)  # 对输入张量X完成量化
-        if hasattr(self, 'qi1'):
-            self.qi1.update(x1)
-            x1 = FakeQuantize.apply(x1, self.qi1)  # 对输入张量X完成量化
-
-        x = x0 + x1 
-
-        if hasattr(self, 'qo'):
+        for i in range(self.len):
+            getattr(self,'M%d'%i).data = (getattr(self,'qi%d'%i).scale / self.qo.scale).data
+        
+    def forward(self,x_array):
+        outs=[]
+        for i in range(self.len):
+            x = x_array[i]
+            if hasattr(self,'qi%d'%i):
+                qi = getattr(self,'qi%d'%i)
+                qi.update(x)
+                x = FakeQuantize.apply(x,qi)
+            outs.append(x)
+        out = torch.cat(outs,1)
+        if hasattr(self,'qo'):
             self.qo.update(x)
-            x = FakeQuantize.apply(x, self.qo)
-
-        return x
-    
-    def quantize_inference(self, x0, x1):  # 此处input为已经量化的qx
-        x0 = x0 - self.qi0.zero_point
-        x1 = x1 - self.qi1.zero_point
-        
-        x = self.M0 * (x0 + x1*self.M1)
-
-        # if self.quant_type is 'INT':
-        x = get_nearest_val(self.quant_type,x)
-
-        x = x + self.qo.zero_point
-
-
-        return x
-
-class QModule_3(nn.Module):
-
-    def __init__(self,quant_type, qix=True, qih=True, qic=True, qox=True, qoh=True, qoc=True, num_bits=8, e_bits=3):
-        super(QModule_3, self).__init__()
+            out = FakeQuantize.apply(out,self.qo)
+        return out
+
+    def quantize_inference(self, x_array):
+        outs=[]
+        for i in range(self.len):
+            qi = getattr(self,'qi%d'%i)
+            x = x_array[i] - qi.zero_point
+            x = getattr(self,'M%d'%i) * x
+            outs.append(x)
+        out = torch.concat(outs,1)
+        out = get_nearest_val(self.quant_type,out)
+        out = out + self.qo.zero_point
+        return out
+
+class QModule_rnnbase(nn.Module):
+    def __init__(self,quant_type, qix=False, qih=False, qic=False, qox=True, qoh=True, qoc=True, num_bits=8, e_bits=3):
+        super(QModule_rnnbase,self).__init__()
         if qix:
-            self.qix = QParam(quant_type,num_bits, e_bits)
-        if qox:
-            self.qox = QParam(quant_type,num_bits, e_bits)
-
+            self.qix = QParam(quant_type, num_bits, e_bits)
         if qih:
-            self.qih = QParam(quant_type,num_bits, e_bits)
-        if qoh:
-            self.qoh = QParam(quant_type,num_bits, e_bits)
-
+            self.qih = QParam(quant_type, num_bits, e_bits)
         if qic:
-            self.qic = QParam(quant_type,num_bits, e_bits)
+            self.qic = QParam(quant_type, num_bits, e_bits)
+        if qox:
+            self.qox = QParam(quant_type, num_bits, e_bits)
+        if qoh:
+            self.qoh = QParam(quant_type, num_bits, e_bits)
         if qoc:
-            self.qoc = QParam(quant_type,num_bits, e_bits)
-
+            self.qoc = QParam(quant_type, num_bits, e_bits)            
         self.quant_type = quant_type
         self.num_bits = num_bits
         self.e_bits = e_bits
@@ -964,59 +786,52 @@ class QModule_3(nn.Module):
     def freeze(self):
         pass  # 空语句
 
-    def fakefreeze(self):
-        pass
-
     def quantize_inference(self, x):
         raise NotImplementedError('quantize_inference should be implemented.')
 
-
-class QLSTM(QModule_3):
-    def __init__(self, quant_type, lstm_module, qix=True, qih=True, qic=True, qox=True, qoh=True, qoc=True, num_bits=8, e_bits=3):
-        super(QLSTM, self).__init__(quant_type, qix, qih, qic, qox, qoh, qoc, num_bits, e_bits)
+# has_hidden表征该层能否接受非零值的hidden输入
+# 对于第一层，可能第一个batch输入为零值，后续非零，则has_hidden仍需置为true
+# 需额外判断hidden是否为none，避免scale为0的情形
+class QLSTM(QModule_rnnbase):
+    def __init__(self, quant_type, lstm_module, has_hidden, qix=False, qih=False, qic=False, qox=True, qoh=True, qoc=True, num_bits=8, e_bits=3):
+        super(QLSTM,self).__init__(quant_type,qix,qih,qic,qox,qoh,qoc,num_bits,e_bits)
         self.lstm_module = lstm_module
-        self.qwih = QParam(quant_type, num_bits,e_bits)
-        self.qwhh = QParam(quant_type, num_bits,e_bits)
-        # self.qbih = QParam(quant_type, num_bits,e_bits)
-        # self.qbhh = QParam(quant_type, num_bits,e_bits)
-        # self.register_buffer('Mi', torch.tensor([], requires_grad=False))  # 将Mi注册为buffer
-        # self.register_buffer('Mh', torch.tensor([], requires_grad=False))  # 将M注册为buffer
-
-        # if self.lstm_module.bidirectional:
-        #     self.qwihr = QParam(quant_type, num_bits,e_bits)
-        #     self.qwhhr = QParam(quant_type, num_bits,e_bits)
-        #     self.qbihr = QParam(quant_type, num_bits,e_bits)
-        #     self.qbhhr = QParam(quant_type, num_bits,e_bits)
-
+        self.qwih = QParam(quant_type, num_bits, e_bits)
+        self.qwhh = QParam(quant_type, num_bits, e_bits)
 
+        self.qwihr = QParam(quant_type, num_bits, e_bits)
+        self.qwhhr= QParam(quant_type, num_bits, e_bits)
 
-    def freeze(self, qix=None, qih=None, qic=None,qox=None, qoh=None, qoc=None, flag=0):
+        # 表征是否输入hidden
+        self.has_hidden = has_hidden
 
+    def freeze(self, qix=None, qih=None, qic=None, qox=None, qoh=None, qoc=None):
         if hasattr(self, 'qix') and qix is not None:
             raise ValueError('qix has been provided in init function.')
         if not hasattr(self, 'qix') and qix is None:
             raise ValueError('qix is not existed, should be provided.')
         
+        if self.has_hidden:
+            if hasattr(self, 'qih') and qih is not None:
+                raise ValueError('qih has been provided in init function.')
+            if not hasattr(self, 'qih') and qih is None:
+                raise ValueError('qih is not existed, should be provided.')
+            
+            if hasattr(self, 'qic') and qic is not None:
+                raise ValueError('qic has been provided in init function.')
+            if not hasattr(self, 'qic') and qic is None:
+                raise ValueError('qic is not existed, should be provided.')
+        
         if hasattr(self, 'qox') and qox is not None:
             raise ValueError('qox has been provided in init function.')
         if not hasattr(self, 'qox') and qox is None:
             raise ValueError('qox is not existed, should be provided.')
         
-        if hasattr(self, 'qih') and qih is not None:
-            raise ValueError('qih has been provided in init function.')
-        if not hasattr(self, 'qih') and qih is None and flag==1:  # 非第一个lstm layer
-            raise ValueError('qih is not existed, should be provided.')
-
         if hasattr(self, 'qoh') and qoh is not None:
             raise ValueError('qoh has been provided in init function.')
         if not hasattr(self, 'qoh') and qoh is None:
             raise ValueError('qoh is not existed, should be provided.')
         
-        if hasattr(self, 'qic') and qic is not None:
-            raise ValueError('qic has been provided in init function.')
-        if not hasattr(self, 'qic') and qic is None and flag==1:  # 非第一个lstm layer
-            raise ValueError('qic is not existed, should be provided.')
-
         if hasattr(self, 'qoc') and qoc is not None:
             raise ValueError('qoc has been provided in init function.')
         if not hasattr(self, 'qoc') and qoc is None:
@@ -1024,376 +839,237 @@ class QLSTM(QModule_3):
 
         if qix is not None:
             self.qix = qix
+        if self.has_hidden:
+            if qih is not None:
+                self.qih = qih
+            if qic is not None:
+                self.qic = qic
         if qox is not None:
             self.qox = qox
-        
-        # 为了避免第一个lstm layer没有h，c却拥有qih，qic
-        if qih is not None and flag==1:
-            self.qih = qih
         if qoh is not None:
             self.qoh = qoh
-
-        if qic is not None and flag==1:
-            self.qic = qic
         if qoc is not None:
             self.qoc = qoc
 
-        # 这里应该涉及到了两个问题：1. 输出、隐层输出  2. 双向
-        # 输出、隐层输出可以分别处理 
-        # 双向比较麻烦，在量化后甚至还涉及到了SUM和CONCAT的整合方式
-        # self.Mi.data = (self.qwih.scale * self.qi.scale / self.qo.scale).data
-        # self.Mh.data = (self.qwhh.scale * self.qi.scale / self.qo.scale).data
-
-        # 对weight伪量化
-        self.lstm_module.weight_ih_l0.data = FakeQuantize.apply(self.lstm_module.weight_ih_l0.data,self.qwih)
+        # 正向
+        self.lstm_module.weight_ih_l0.data = FakeQuantize.apply(self.lstm_module.weight_ih_l0.data, self.qwih)
+        self.lstm_module.weight_hh_l0.data = FakeQuantize.apply(self.lstm_module.weight_hh_l0.data, self.qwhh)
 
-        self.lstm_module.weight_hh_l0.data = FakeQuantize.apply(self.lstm_module.weight_hh_l0.data,self.qwhh)
-
-
-        # 对bias伪量化 
-        self.lstm_module.bias_ih_l0.data = quantize_tensor(self.quant_type,self.lstm_module.bias_ih_l0.data,scale=self.qix.scale*self.qwih.scale,zero_point=0,qmax=self.bias_qmax,is_bias=True)
-        self.lstm_module.bias_ih_l0.data = dequantize_tensor(self.lstm_module.bias_ih_l0.data,scale=self.qix.scale*self.qwih.scale,zero_point=0)
-
-        # 第一个layer是没有qih的，需要特殊处理
-        if flag==1:
-            self.lstm_module.bias_hh_l0.data = quantize_tensor(self.quant_type,self.lstm_module.bias_hh_l0.data,scale=self.qih.scale*self.qwhh.scale,zero_point=0,qmax=self.bias_qmax,is_bias=True)
-            self.lstm_module.bias_hh_l0.data = dequantize_tensor(self.lstm_module.bias_hh_l0.data,scale=self.qih.scale*self.qwhh.scale,zero_point=0)
+        self.lstm_module.bias_ih_l0.data = quantize_tensor(self.quant_type, 
+                                                           self.lstm_module.bias_ih_l0.data,
+                                                           scale=self.qix.scale * self.qwih.scale,
+                                                           zero_point=0., qmax=self.bias_qmax, is_bias=True)
+        self.lstm_module.bias_ih_l0.data = dequantize_tensor(self.lstm_module.bias_ih_l0.data,
+                                                             scale=self.qix.scale * self.qwih.scale, zero_point=0.)
+        # 反向
+        self.lstm_module.weight_ih_l0_reverse.data = FakeQuantize.apply(self.lstm_module.weight_ih_l0_reverse.data, self.qwihr)
+        self.lstm_module.weight_hh_l0_reverse.data = FakeQuantize.apply(self.lstm_module.weight_hh_l0_reverse.data, self.qwhhr)
+
+        self.lstm_module.bias_ih_l0_reverse.data = quantize_tensor(self.quant_type, 
+                                                           self.lstm_module.bias_ih_l0_reverse.data,
+                                                           scale=self.qix.scale * self.qwihr.scale,
+                                                           zero_point=0., qmax=self.bias_qmax, is_bias=True)
+        self.lstm_module.bias_ih_l0_reverse.data = dequantize_tensor(self.lstm_module.bias_ih_l0_reverse.data,
+                                                             scale=self.qix.scale * self.qwihr.scale, zero_point=0.)
+        
+        # 若是为第一层lstm，就不量化bias_hh了
+        if self.has_hidden:
+            # 正向
+            self.lstm_module.bias_hh_l0.data = quantize_tensor(self.quant_type, 
+                                                            self.lstm_module.bias_hh_l0.data,
+                                                            scale=self.qih.scale * self.qwhh.scale,
+                                                            zero_point=0., qmax=self.bias_qmax, is_bias=True)
+            self.lstm_module.bias_hh_l0.data = dequantize_tensor(self.lstm_module.bias_hh_l0.data,
+                                                                scale=self.qih.scale * self.qwhh.scale, zero_point=0.)
+            # 反向
+            self.lstm_module.bias_hh_l0_reverse.data = quantize_tensor(self.quant_type, 
+                                                            self.lstm_module.bias_hh_l0_reverse.data,
+                                                            scale=self.qih.scale * self.qwhhr.scale,
+                                                            zero_point=0., qmax=self.bias_qmax, is_bias=True)
+            self.lstm_module.bias_hh_l0_reverse.data = dequantize_tensor(self.lstm_module.bias_hh_l0_reverse.data,scale=self.qih.scale * self.qwhhr.scale, zero_point=0.)
 
+    def forward(self, x, hidden=None):
+        # if self.has_hidden and hidden is None:
+        #     raise ValueError("LSTM layer with has_hidden must accept hidden")
+        if not self.has_hidden and hidden is not None:
+            raise ValueError("LSTM layer without has_hidden cannot accept hidden")
 
-    def forward(self, x, h=None, c=None):
         if hasattr(self, 'qix'):
             self.qix.update(x)
-            x = FakeQuantize.apply(x, self.qix)
-        if hasattr(self, 'qih') and h is not None:  # 兼顾第一个lstm layer无h，qih
+            x = FakeQuantize.apply(x,self.qix)
+        if self.has_hidden and hidden is not None:
+            h,c = hidden
+            if hasattr(self, 'qih'):
                 self.qih.update(h)
                 h = FakeQuantize.apply(h, self.qih)
-        if hasattr(self, 'qic') and c is not None:  # 兼顾第一个lstm layer无c，qic
+            if hasattr(self, 'qic'):
                 self.qic.update(c)
                 c = FakeQuantize.apply(c, self.qic)
             
         self.qwih.update(self.lstm_module.weight_ih_l0.data)
         self.qwhh.update(self.lstm_module.weight_hh_l0.data)
-
-        layer = nn.LSTM(input_size=self.lstm_module.input_size,
-                        hidden_size=self.lstm_module.hidden_size,
-                        num_layers=1,
-                        batch_first=False,
-                        bidirectional=self.lstm_module.bidirectional,
-                        bias=True)
-
-        layer.weight_ih_l0.data = FakeQuantize.apply(self.lstm_module.weight_ih_l0.data,self.qwih)
-        layer.weight_hh_l0.data = FakeQuantize.apply(self.lstm_module.weight_hh_l0.data,self.qwhh)
-        layer.bias_ih_l0.data = self.lstm_module.bias_ih_l0.data 
-        layer.bias_hh_l0.data = self.lstm_module.bias_hh_l0.data
-
-        if h is None:
-            x, (h, c) = layer(x)
-            
+        self.qwihr.update(self.lstm_module.weight_ih_l0_reverse.data)
+        self.qwhhr.update(self.lstm_module.weight_hh_l0_reverse.data)
+
+        # 由于该层实现与Conv等不同，若使用_VF.lstm,weight需要flatten
+        # 为简化调用逻辑，仍使用nn.LSTM新建层进行调用
+        tmplayer = nn.LSTM(input_size = self.lstm_module.input_size,
+                           hidden_size = self.lstm_module.hidden_size,
+                           num_layers = 1,
+                           batch_first = False,
+                           bidirectional = self.lstm_module.bidirectional,
+                           bias = self.lstm_module.bias,
+                           dropout = self.lstm_module.dropout)
+        # 正向
+        tmplayer.weight_ih_l0.data = FakeQuantize.apply(self.lstm_module.weight_ih_l0.data, self.qwih)
+        tmplayer.weight_hh_l0.data = FakeQuantize.apply(self.lstm_module.weight_hh_l0.data, self.qwhh)
+        tmplayer.bias_ih_l0.data = self.lstm_module.bias_ih_l0.data
+        tmplayer.bias_hh_l0.data = self.lstm_module.bias_hh_l0.data
+        # 反向
+        tmplayer.weight_ih_l0_reverse.data = FakeQuantize.apply(self.lstm_module.weight_ih_l0_reverse.data, self.qwihr)
+        tmplayer.weight_hh_l0_reverse.data = FakeQuantize.apply(self.lstm_module.weight_hh_l0_reverse.data, self.qwhhr)
+        tmplayer.bias_ih_l0_reverse.data = self.lstm_module.bias_ih_l0_reverse.data
+        tmplayer.bias_hh_l0_reverse.data = self.lstm_module.bias_hh_l0_reverse.data
+
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        tmplayer.to(device)
+
+
+        if self.has_hidden and hidden is not None:
+            x, (h,c) = tmplayer(x,(h,c))
         else:
-            x, (h, c) = layer(x, (h, c))
+            x, (h,c) = tmplayer(x)
 
-        if hasattr(self, 'qox'):
+        if hasattr(self,'qox'):
             self.qox.update(x)
-            x = FakeQuantize.apply(x, self.qox)
-        if hasattr(self, 'qoh'):
+            x = FakeQuantize.apply(x,self.qox)
+        if hasattr(self,'qoh'):
             self.qoh.update(h)
-            h = FakeQuantize.apply(h, self.qoh)
-        if hasattr(self, 'qoc'):
+            h = FakeQuantize.apply(h,self.qoh)
+        if hasattr(self,'qoc'):
             self.qoc.update(c)
-            c = FakeQuantize.apply(c, self.qoc)
-        
-        return x,(h,c)
-    
-    def quantize_inference(self, x, h=None, c=None):
-        # freeze的时是fakequantize，因此这里直接算，无需做scale变换
-        if h is None:
-            x, (h, c) = self.lstm_module(x)
-            if hasattr(self, 'qox'):
-                x = FakeQuantize.apply(x, self.qox)
-            if hasattr(self, 'qoh'):
-                h = FakeQuantize.apply(h, self.qoh)
-            if hasattr(self, 'qoc'):
-                c = FakeQuantize.apply(c, self.qoc)
-            
-            
-        else:
-            x, (h, c) = self.lstm_module(x, (h, c))
-            if hasattr(self, 'qox'):
-                x = FakeQuantize.apply(x, self.qox)
-            if hasattr(self, 'qoh'):
-                h = FakeQuantize.apply(h, self.qoh)
-            if hasattr(self, 'qoc'):
-                c = FakeQuantize.apply(c, self.qoc)
-
-        return x,(h,c)
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-# new modules for full-precision model - fold bn
-# inference应该也需要相应的适配
-
-
-
-class ConvBNReLU(nn.Module):
-
-    def __init__(self,conv_module, bn_module):
-        super(ConvBNReLU, self).__init__()
-        self.conv_module = conv_module
-        self.bn_module = bn_module
-
-    def fold_bn(self, mean, std):
-        if self.bn_module.affine:
-            gamma_ = self.bn_module.weight / std
-            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
-            if self.conv_module.bias is not None:
-                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
-            else:
-                bias = self.bn_module.bias - gamma_ * mean
-        else:
-            gamma_ = 1 / std
-            weight = self.conv_module.weight * gamma_
-            if self.conv_module.bias is not None:
-                bias = gamma_ * self.conv_module.bias - gamma_ * mean
-            else:
-                bias = -gamma_ * mean
-            
-        return weight, bias
-
-    def freeze(self):
-
-        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
-
-        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
-
-        self.conv_module.weight.data = weight.data
-        if self.conv_module.bias is None:
-            self.conv_module.bias = nn.Parameter(bias)
-        else:
-            self.conv_module.bias.data = bias        
-
-    def fakefreeze(self):
-        pass
-        
-    
-    def forward(self, x):
-
-        if self.training:
-            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
-                            stride=self.conv_module.stride,
-                            padding=self.conv_module.padding,
-                            dilation=self.conv_module.dilation,
-                            groups=self.conv_module.groups)
-            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
-            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
-            # mean = y.mean(1)
-            # var = y.var(1)
-            mean = y.mean(1).detach()
-            var = y.var(1).detach()
-            self.bn_module.running_mean = \
-                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
-                self.bn_module.momentum * mean
-            self.bn_module.running_var = \
-                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
-                self.bn_module.momentum * var
-        else:
-            mean = Variable(self.bn_module.running_mean)
-            var = Variable(self.bn_module.running_var)
-
-        std = torch.sqrt(var + self.bn_module.eps)
-
-        weight, bias = self.fold_bn(mean, std)
-
-
-        x = F.conv2d(x, weight, bias, 
-                stride=self.conv_module.stride,
-                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
-                groups=self.conv_module.groups)
-
-        x = F.relu(x)
-
-        return x
-    
-    def quantize_inference(self, x):
-        x = self.conv_module(x)  
-        x.clamp_(min=0)
-        return x
-        
-
-class ConvBN(nn.Module):
-
-    def __init__(self,conv_module, bn_module):
-        super(ConvBN, self).__init__()
-        self.conv_module = conv_module
-        self.bn_module = bn_module
-
-    def fold_bn(self, mean, std):
-        if self.bn_module.affine:
-            gamma_ = self.bn_module.weight / std
-            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
-            if self.conv_module.bias is not None:
-                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
-            else:
-                bias = self.bn_module.bias - gamma_ * mean
-        else:
-            gamma_ = 1 / std
-            weight = self.conv_module.weight * gamma_
-            if self.conv_module.bias is not None:
-                bias = gamma_ * self.conv_module.bias - gamma_ * mean
-            else:
-                bias = -gamma_ * mean
-            
-        return weight, bias
-
-    def freeze(self):
-
-        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
-
-        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
-
-        self.conv_module.weight.data = weight.data
-        if self.conv_module.bias is None:
-            self.conv_module.bias = nn.Parameter(bias)
-        else:
-            self.conv_module.bias.data = bias        
+            c = FakeQuantize.apply(c,self.qoc)
+        
+        hidden = (h,c)
+        return x, hidden
+
+    def quantize_inference(self, x, hidden=None):
+        # if self.has_hidden and hidden is None:
+        #     raise ValueError("LSTM layer with has_hidden must accept hidden")
+        # if not self.has_hidden and hidden is not None:
+        #     raise ValueError("LSTM layer without has_hidden cannot accept hidden")
+
+        # if self.has_hidden:
+        #     x, hidden = self.lstm_module(x,hidden)
+        # else:
+        #     x, hidden = self.lstm_module(x)
+
+        # h,c = hidden
+        # if hasattr(self,'qox'):
+        #     x = FakeQuantize.apply(x, self.qox)
+        # if hasattr(self,'qoh'):
+        #     h = FakeQuantize.apply(h, self.qoh)
+        # if hasattr(self,'qoc'):
+        #     c = FakeQuantize.apply(c, self.qoc)
+
+        # hidden = (h,c)
+        # return x,hidden
+
+        # if self.has_hidden and hidden is None:
+        #     raise ValueError("LSTM layer with has_hidden must accept hidden")
+        if not self.has_hidden and hidden is not None:
+            raise ValueError("LSTM layer without has_hidden cannot accept hidden")
 
-    def fakefreeze(self):
-        pass
-        
-    
-    def forward(self, x):
+        if hasattr(self, 'qix'):
+            x = FakeQuantize.apply(x,self.qix)
+        if self.has_hidden and hidden is not None:
+            h,c = hidden
+            if hasattr(self, 'qih'):
+                h = FakeQuantize.apply(h, self.qih)
+            if hasattr(self, 'qic'):
+                c = FakeQuantize.apply(c, self.qic)
             
-        if self.training:
-            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
-                            stride=self.conv_module.stride,
-                            padding=self.conv_module.padding,
-                            dilation=self.conv_module.dilation,
-                            groups=self.conv_module.groups)
-            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
-            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
-            # mean = y.mean(1)
-            # var = y.var(1)
-            mean = y.mean(1).detach()
-            var = y.var(1).detach()
-            self.bn_module.running_mean = \
-                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
-                self.bn_module.momentum * mean
-            self.bn_module.running_var = \
-                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
-                self.bn_module.momentum * var
+        # 由于该层实现与Conv等不同，若使用_VF.lstm,weight需要flatten
+        # 为简化调用逻辑，仍使用nn.LSTM新建层进行调用
+        tmplayer = nn.LSTM(input_size = self.lstm_module.input_size,
+                           hidden_size = self.lstm_module.hidden_size,
+                           num_layers = 1,
+                           batch_first = False,
+                           bidirectional = self.lstm_module.bidirectional,
+                           bias = self.lstm_module.bias,
+                           dropout = self.lstm_module.dropout)
+        # 正向
+        tmplayer.weight_ih_l0.data = FakeQuantize.apply(self.lstm_module.weight_ih_l0.data, self.qwih)
+        tmplayer.weight_hh_l0.data = FakeQuantize.apply(self.lstm_module.weight_hh_l0.data, self.qwhh)
+        tmplayer.bias_ih_l0.data = self.lstm_module.bias_ih_l0.data
+        tmplayer.bias_hh_l0.data = self.lstm_module.bias_hh_l0.data
+        # 反向
+        tmplayer.weight_ih_l0_reverse.data = FakeQuantize.apply(self.lstm_module.weight_ih_l0_reverse.data, self.qwihr)
+        tmplayer.weight_hh_l0_reverse.data = FakeQuantize.apply(self.lstm_module.weight_hh_l0_reverse.data, self.qwhhr)
+        tmplayer.bias_ih_l0_reverse.data = self.lstm_module.bias_ih_l0_reverse.data
+        tmplayer.bias_hh_l0_reverse.data = self.lstm_module.bias_hh_l0_reverse.data
+
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        tmplayer.to(device)
+
+        if self.has_hidden and hidden is not None:
+            x, (h,c) = tmplayer(x,(h,c))
         else:
-            mean = Variable(self.bn_module.running_mean)
-            var = Variable(self.bn_module.running_var)
-
-        std = torch.sqrt(var + self.bn_module.eps)
-
-        weight, bias = self.fold_bn(mean, std)
+            x, (h,c) = tmplayer(x)
 
+        if hasattr(self,'qox'):
+            x = FakeQuantize.apply(x,self.qox)
+        if hasattr(self,'qoh'):
+            h = FakeQuantize.apply(h,self.qoh)
+        if hasattr(self,'qoc'):
+            c = FakeQuantize.apply(c,self.qoc)
         
-        x = F.conv2d(x, weight, bias, 
-                stride=self.conv_module.stride,
-                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
-                groups=self.conv_module.groups)
-
-        return x
-    
-    def quantize_inference(self, x):
-        x = self.conv_module(x)  
-        return x
+        hidden = (h,c)
+        return x, hidden
 
         
-class ConvBNReLU6(nn.Module):
-    def __init__(self,conv_module, bn_module):
-        super(ConvBNReLU6, self).__init__()
-        self.conv_module = conv_module
-        self.bn_module = bn_module
 
-    def fold_bn(self, mean, std):
-        if self.bn_module.affine:
-            gamma_ = self.bn_module.weight / std
-            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
-            if self.conv_module.bias is not None:
-                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
-            else:
-                bias = self.bn_module.bias - gamma_ * mean
-        else:
-            gamma_ = 1 / std
-            weight = self.conv_module.weight * gamma_
-            if self.conv_module.bias is not None:
-                bias = gamma_ * self.conv_module.bias - gamma_ * mean
-            else:
-                bias = -gamma_ * mean
-            
-        return weight, bias
-
-    def freeze(self):
-
-        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
+class QEmbedding(QModule):
+    def __init__(self,quant_type,embedding_module,qi=False, qo=True, num_bits=8, e_bits=3):
+        super(QEmbedding, self).__init__(quant_type,qi,qo, num_bits, e_bits)
+        self.embedding_module = embedding_module
+        self.qw = QParam(quant_type, num_bits, e_bits)
+        self.register_buffer('M', torch.tensor([], requires_grad=False)) 
 
-        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
+    def freeze(self, qo=None):
+        #输入为index，不量化
+        if hasattr(self, 'qo') and qo is not None:
+            raise ValueError('qo has been provided in init function.')
+        if not hasattr(self, 'qo') and qo is None:
+            raise ValueError('qo is not existed, should be provided.')
 
-        self.conv_module.weight.data = weight.data
-        if self.conv_module.bias is None:
-            self.conv_module.bias = nn.Parameter(bias)
-        else:
-            self.conv_module.bias.data = bias        
+        if qo is not None:
+            self.qo = qo
 
-    def fakefreeze(self):
-        pass
+        self.M.data = (self.qw.scale / self.qo.scale).data
         
+        self.embedding_module.weight.data = self.qw.quantize_tensor(self.embedding_module.weight.data)
         
     def forward(self, x):
+        self.qw.update(self.embedding_module.weight.data)
+        tmp_wgt = FakeQuantize.apply(self.embedding_module.weight, self.qw)
+        # tmp_wgt = self.embedding_module.weight
+        x = F.embedding(x, tmp_wgt)
 
-        if self.training:
-            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
-                            stride=self.conv_module.stride,
-                            padding=self.conv_module.padding,
-                            dilation=self.conv_module.dilation,
-                            groups=self.conv_module.groups)
-            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
-            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
-            # mean = y.mean(1)
-            # var = y.var(1)
-            mean = y.mean(1).detach()
-            var = y.var(1).detach()
-            self.bn_module.running_mean = \
-                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
-                self.bn_module.momentum * mean
-            self.bn_module.running_var = \
-                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
-                self.bn_module.momentum * var
-        else:
-            mean = Variable(self.bn_module.running_mean)
-            var = Variable(self.bn_module.running_var)
-
-        std = torch.sqrt(var + self.bn_module.eps)
-
-        weight, bias = self.fold_bn(mean, std)
-
-
-        x = F.conv2d(x, weight, bias, 
-                stride=self.conv_module.stride,
-                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
-                groups=self.conv_module.groups)
-
-        x = F.relu6(x)
+        if hasattr(self, 'qo'):
+            self.qo.update(x)
+            x = FakeQuantize.apply(x, self.qo)
         
         return x
 
     def quantize_inference(self, x):
-        x = self.conv_module(x)  
-        x.clamp_(min=0,max=6)
+        # x = self.embedding_module(x)
+        # x = self.M * x
+        # x = get_nearest_val(self.quant_type, x)
+        # return x
+    
+        tmp_wgt = FakeQuantize.apply(self.embedding_module.weight, self.qw)
+        # tmp_wgt = self.embedding_module.weight
+        x = F.embedding(x, tmp_wgt)
+        x = FakeQuantize.apply(x, self.qo)
         return x
\ No newline at end of file

mzh/new_mzh/LSTM-OCR/param_flops_lstm-ocr.txt

+CRNN(
+  11.77 M, 100.000% Params, 1.26 GMac, 100.000% MACs, 
+  (layers): ModuleDict(
+    11.77 M, 100.000% Params, 1.26 GMac, 100.000% MACs, 
+    (conv1): Conv2d(640, 0.005% Params, 3.28 MMac, 0.260% MACs, 1, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (relu1): ReLU(0, 0.000% Params, 327.68 KMac, 0.026% MACs, )
+    (pool1): MaxPool2d(0, 0.000% Params, 327.68 KMac, 0.026% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
+    (conv2): Conv2d(73.86 k, 0.628% Params, 94.54 MMac, 7.494% MACs, 64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (relu2): ReLU(0, 0.000% Params, 163.84 KMac, 0.013% MACs, )
+    (pool2): MaxPool2d(0, 0.000% Params, 163.84 KMac, 0.013% MACs, kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)
+    (conv3): Conv2d(295.17 k, 2.508% Params, 94.45 MMac, 7.487% MACs, 128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (bn3): BatchNorm2d(512, 0.004% Params, 163.84 KMac, 0.013% MACs, 256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (relu3): ReLU(0, 0.000% Params, 81.92 KMac, 0.006% MACs, )
+    (conv4): Conv2d(590.08 k, 5.014% Params, 188.83 MMac, 14.968% MACs, 256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (relu4): ReLU(0, 0.000% Params, 81.92 KMac, 0.006% MACs, )
+    (pool4): MaxPool2d(0, 0.000% Params, 81.92 KMac, 0.006% MACs, kernel_size=(2, 2), stride=(2, 1), padding=(0, 1), dilation=1, ceil_mode=False)
+    (conv5): Conv2d(1.18 M, 10.029% Params, 193.55 MMac, 15.342% MACs, 256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (bn5): BatchNorm2d(1.02 k, 0.009% Params, 167.94 KMac, 0.013% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (relu5): ReLU(0, 0.000% Params, 83.97 KMac, 0.007% MACs, )
+    (conv6): Conv2d(2.36 M, 20.053% Params, 387.01 MMac, 30.677% MACs, 512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
+    (relu6): ReLU(0, 0.000% Params, 83.97 KMac, 0.007% MACs, )
+    (pool6): MaxPool2d(0, 0.000% Params, 83.97 KMac, 0.007% MACs, kernel_size=(2, 2), stride=(2, 1), padding=(0, 1), dilation=1, ceil_mode=False)
+    (conv7): Conv2d(1.05 M, 8.915% Params, 43.01 MMac, 3.409% MACs, 512, 512, kernel_size=(2, 2), stride=(1, 1))
+    (bn7): BatchNorm2d(1.02 k, 0.009% Params, 41.98 KMac, 0.003% MACs, 512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
+    (relu7): ReLU(0, 0.000% Params, 20.99 KMac, 0.002% MACs, )
+    (lstm1): LSTM(1.58 M, 13.401% Params, 64.87 MMac, 5.142% MACs, 512, 256, bidirectional=True)
+    (fc1): Linear(131.33 k, 1.116% Params, 5.37 MMac, 0.426% MACs, in_features=512, out_features=256, bias=True)
+    (lstm2): LSTM(1.05 M, 8.945% Params, 43.37 MMac, 3.438% MACs, 256, 256, bidirectional=True)
+    (fc2): Linear(3.46 M, 29.364% Params, 141.41 MMac, 11.209% MACs, in_features=512, out_features=6736, bias=True)
+  )
+)
\ No newline at end of file

mzh/new_mzh/LSTM-OCR/ptq.py

 # -*- coding: utf-8 -*-
 from torch.serialization import load
 from model import *
-# from extract_ratio import *
+from extract_ratio import *
+
+from train import parse_arg
+from model import *
 from utils import *
+from lstm_utils import *
+from dataset import get_dataset
+from config.alphabets import *
+from torch.utils.data import DataLoader
 
 import gol
 import openpyxl
@@ -18,161 +25,117 @@ import torch.utils.bottleneck as bn
 import os
 import os.path as osp
 from torch.utils.tensorboard import SummaryWriter
-import json
-from decoder import seq_mnist_decoder
-from data import seq_mnist_train, seq_mnist_val
-from torch.utils.data import DataLoader
-import random
 
-class objdict(dict):
-    def __getattr__(self, name):
-        if name in self:
-            return self[name]
-        else:
-            raise AttributeError("No such attribute: " + name)
 
-    def __setattr__(self, name, value):
-        self[name] = value
+def direct_quantize(config, val_loader, dataset, converter, model, criterion, device):
 
-    def __delattr__(self, name):
-        if name in self:
-            del self[name]
-        else:
-            raise AttributeError("No such attribute: " + name)
-
-
-# def direct_quantize(model, test_loader,device):
-#     for i, (data, target) in enumerate(test_loader, 1):
-#         data = data.to(device)
-#         output = model.quantize_forward(data).cpu()
-#         if i % 500 == 0:
-#             break
-#     print('direct quantization finish')
-
-
-# def full_inference(model, test_loader, device):
-#     correct = 0
-#     for i, (data, target) in enumerate(test_loader, 1):
-#         data = data.to(device)
-#         output = model(data).cpu()
-#         pred = output.argmax(dim=1, keepdim=True)
-#         correct += pred.eq(target.view_as(pred)).sum().item()
-#     print('\nTest set: Full Model Accuracy: {:.2f}%'.format(100. * correct / len(test_loader.dataset)))
-#     return 100. * correct / len(test_loader.dataset)
-
-
-# def quantize_inference(model, test_loader, device):
-#     correct = 0
-#     for i, (data, target) in enumerate(test_loader, 1):
-#         data = data.to(device)
-#         output = model.quantize_inference(data).cpu()
-#         pred = output.argmax(dim=1, keepdim=True)
-#         correct += pred.eq(target.view_as(pred)).sum().item()
-#     print('Test set: Quant Model Accuracy: {:.2f}%'.format(100. * correct / len(test_loader.dataset)))
-#     return 100. * correct / len(test_loader.dataset)
-
-def direct_quantize(model, val_loader , val_data ,args , trainer_params, decoder, criterion):
     model.eval()
-    loss_value = 0
-    for i, (item) in enumerate(val_loader):           
-        data, labels, output_len, lab_len = item
 
-        data = Variable(data.transpose(1,0), requires_grad=False)
-        labels = Variable(labels.view(-1), requires_grad=False)
-        output_len = Variable(output_len.view(-1), requires_grad=False)
-        lab_len = Variable(lab_len.view(-1), requires_grad=False)
+    n_correct = 0
+    with torch.no_grad():
+        for i, (inp, idx) in enumerate(val_loader):
+            inp = inp.to(device)
+            # inference
+            preds = model.quantize_forward(inp)
 
+    print('direct quantization finish')
 
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 
-        data = data.to(device)
 
-        output = model.quantize_forward(data)
-#         if i % 500 == 0:
-# #             break
+def full_inference(config, val_loader, dataset, converter, model, criterion, device):
+    losses = AverageMeter()
+    model.eval()
 
-    print('direct quantization finish')
+    n_correct = 0
+    with torch.no_grad():
+        for i, (inp, idx) in enumerate(val_loader):
 
+            # 一个batch的label的list
+            labels = get_batch_label(dataset, idx)
+            inp = inp.to(device)
 
+            # inference
+            preds = model(inp).cpu()
 
-    # loss_value /= (len(val_data)//trainer_params.test_batch_size)
-    # # loss_value = loss_value[0]
-    # loss_value = loss_value.item()
-    # print("Average Loss Value for Val Data is = {:.4f}\n".format(float(loss_value)))
+            # compute loss
+            batch_size = inp.size(0)
+            text, length = converter.encode(labels)
+            preds_size = torch.IntTensor([preds.size(0)] * batch_size)
+            loss = criterion(preds, text, preds_size, length)
 
-def full_inference(model, val_loader , val_data ,args , trainer_params, decoder, criterion):
-    model.eval()
-    loss_value = 0
-    for i, (item) in enumerate(val_loader):           
-        data, labels, output_len, lab_len = item
+            losses.update(loss.item(), inp.size(0))
 
-        data = Variable(data.transpose(1,0), requires_grad=False)
-        labels = Variable(labels.view(-1), requires_grad=False)
-        output_len = Variable(output_len.view(-1), requires_grad=False)
-        lab_len = Variable(lab_len.view(-1), requires_grad=False)
+            _, preds = preds.max(2)
+            preds = preds.transpose(1, 0).contiguous().view(-1)
+            # sim_preds是decode后的，得到了一个str或者一个str list (也是一个batch的)
+            sim_preds = converter.decode(preds.data, preds_size.data, raw=False)
+            # target应该是一个list  这里相当于在判断字符串是否相等
+            for pred, target in zip(sim_preds, labels):
+                if pred == target:
+                    n_correct += 1
 
+            # if (i + 1) % config.PRINT_FREQ == 0:
+            #     print('Epoch: [{0}][{1}/{2}]'.format(epoch, i, len(val_loader)))
 
-            # data = data.cuda()
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 
-        data = data.to(device)
+    # # 只打印展示前 config.TEST.NUM_TEST_DISP 个句子的对比
+    # raw_preds = converter.decode(preds.data, preds_size.data, raw=True)[:config.TEST.NUM_TEST_DISP]
+    # for raw_pred, pred, gt in zip(raw_preds, sim_preds, labels):
+    #     print('%-20s => %-20s, gt: %-20s' % (raw_pred, pred, gt))
 
-        output = model(data)
 
-        index = random.randint(0,trainer_params.test_batch_size-1)      
-        label = labels[index*trainer_params.word_size:(index+1)*trainer_params.word_size].data.numpy()
-        label = label-1
-        prediction = decoder.decode(output[:,index,:], output_len[index], lab_len[index])
-        accuracy = decoder.hit(prediction, label)
+    num_test_sample = len(dataset)
+    print("[#Full Precision Model: correct:{} / #total:{}]".format(n_correct, num_test_sample))
+    accuracy = n_correct / float(num_test_sample)
+    print('Full Precision Model: Test loss: {:.4f}, accuray: {:.4f}'.format(losses.avg, accuracy))
+    return accuracy
 
-        print("Sample Label      = {}".format(decoder.to_string(label))) 
-        print("Sample Prediction = {}".format(decoder.to_string(prediction)))
-        print("Full Model Accuracy on Sample = {:.2f}%\n\n".format(accuracy))
 
-        loss = criterion(output, labels, output_len, lab_len)
-        # loss_value += loss.data.numpy()
-        loss_value += loss.cpu().data.numpy()
 
+def quantize_inference(config, val_loader, dataset, converter, model, criterion, device):
+    losses = AverageMeter()
+    model.eval()
 
-    loss_value /= (len(val_data)//trainer_params.test_batch_size)
-    # loss_value = loss_value[0]
-    loss_value = loss_value.item()
-    print("Full Model Average Loss Value for Val Data is = {:.4f}\n".format(float(loss_value)))
+    n_correct = 0
+    with torch.no_grad():
+        for i, (inp, idx) in enumerate(val_loader):
 
-def quantize_inference(model, val_loader , val_data ,args , trainer_params, decoder, criterion):
-    model.eval()
-    loss_value = 0
-    for i, (item) in enumerate(val_loader):           
-        data, labels, output_len, lab_len = item
+            # 一个batch的label的list
+            labels = get_batch_label(dataset, idx)
+            inp = inp.to(device)
 
-        data = Variable(data.transpose(1,0), requires_grad=False)
-        labels = Variable(labels.view(-1), requires_grad=False)
-        output_len = Variable(output_len.view(-1), requires_grad=False)
-        lab_len = Variable(lab_len.view(-1), requires_grad=False)
+            # inference
+            preds = model.quantize_inference(inp).cpu()
 
-            # data = data.cuda()
-        device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 
-        data = data.to(device)
+            # compute loss
+            batch_size = inp.size(0)
+            text, length = converter.encode(labels)
+            preds_size = torch.IntTensor([preds.size(0)] * batch_size)
+            loss = criterion(preds, text, preds_size, length)
 
-        output = model.quantize_inference(data)
+            losses.update(loss.item(), inp.size(0))
 
-        index = random.randint(0,trainer_params.test_batch_size-1)      
-        label = labels[index*trainer_params.word_size:(index+1)*trainer_params.word_size].data.numpy()
-        label = label-1
-        prediction = decoder.decode(output[:,index,:], output_len[index], lab_len[index])
-        accuracy = decoder.hit(prediction, label)
+            _, preds = preds.max(2)
+            preds = preds.transpose(1, 0).contiguous().view(-1)
+            # sim_preds是decode后的，得到了一个str或者一个str list (也是一个batch的)
+            sim_preds = converter.decode(preds.data, preds_size.data, raw=False)
+            # target应该是一个list  这里相当于在判断字符串是否相等
+            for pred, target in zip(sim_preds, labels):
+                if pred == target:
+                    n_correct += 1
 
-        print("Sample Label      = {}".format(decoder.to_string(label))) 
-        print("Sample Prediction = {}".format(decoder.to_string(prediction)))
-        print("Quantize Model Accuracy on Sample = {:.2f}%\n\n".format(accuracy))
+    #         if (i + 1) % config.PRINT_FREQ == 0:
+    #             print('Epoch: [{0}][{1}/{2}]'.format(epoch, i, len(val_loader)))
 
-        loss = criterion(output, labels, output_len, lab_len)
-        # loss_value += loss.data.numpy()
-        loss_value += loss.cpu().data.numpy()
+    # # 只打印展示前 config.TEST.NUM_TEST_DISP 个句子的对比
+    # raw_preds = converter.decode(preds.data, preds_size.data, raw=True)[:config.TEST.NUM_TEST_DISP]
+    # for raw_pred, pred, gt in zip(raw_preds, sim_preds, labels):
+    #     print('%-20s => %-20s, gt: %-20s' % (raw_pred, pred, gt))
 
 
-    loss_value /= (len(val_data)//trainer_params.test_batch_size)
-    # loss_value = loss_value[0]
-    loss_value = loss_value.item()
-    print("Quantize Model Average Loss Value for Val Data is = {:.4f}\n".format(float(loss_value)))
+    num_test_sample = len(dataset)
+    print("[#Quantization Model: correct:{} / #total:{}]".format(n_correct, num_test_sample))
+    accuracy = n_correct / float(num_test_sample)
+    print('Quantization Model: Test loss: {:.4f}, accuray: {:.4f}'.format(losses.avg, accuracy))
+    return accuracy
 
 
 
@@ -190,86 +153,79 @@ def js_div(p_output, q_output, get_softmax=True):
 
 if __name__ == "__main__":
 
+    # load config
+    config = parse_arg()
 
-    parser = argparse.ArgumentParser(description='PyTorch FP32 Training')
-    parser.add_argument('-m', '--model', metavar='MODEL ARCH', default='ResNet18')
-    parser.add_argument('-b', '--batch_size', default=128, type=int, metavar='BATCH SIZE', help='mini-batch size (default: 128)')
-    parser.add_argument('-j','--workers', default=4, type=int, metavar='WORKERS',help='number of data loading workers (default: 4)')
-    parser.add_argument('-s', '--save',  default=False, type=bool)
-    # parser.add_argument('-t', '--test', dest='test', action='store_true', help='test model on test set')
-    parser.add_argument('--params', '-p', type=str, default="default_trainer_params.json", help='Path to params JSON file. Default ignored when resuming.')
-    # 训练参数
-    args = parser.parse_args()
-
-    with open(args.params) as d:
-        trainer_params = json.load(d)
-            # trainer_params = json.load(d, object_hook=ascii_encode_dict)
-    trainer_params = objdict(trainer_params)
-
-    batch_size = args.batch_size
-    num_workers = args.workers
-
-    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
-    print(device)
-
-    labels = [i for i in range(trainer_params.num_classes-1)]
-    decoder = seq_mnist_decoder(labels=labels)
-    criterion = nn.CTCLoss(blank=0, reduction='mean', zero_infinity=False)
-
-    random.seed(trainer_params.random_seed)
-    torch.manual_seed(trainer_params.random_seed)
-    # if args.cuda:
-    torch.cuda.manual_seed_all(trainer_params.random_seed)
+    # construct face related neural networks
+    model = get_crnn(config)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
-    train_data = seq_mnist_train(trainer_params)
-    val_data = seq_mnist_val(trainer_params) 
 
-    train_loader = DataLoader(train_data, batch_size=trainer_params.batch_size, \
-                                        shuffle=True, num_workers=trainer_params.num_workers)
+    # define loss function
+    criterion = torch.nn.CTCLoss()
 
-    val_loader = DataLoader(val_data, batch_size=trainer_params.test_batch_size, \
-                                        shuffle=False, num_workers=trainer_params.num_workers)      
+    train_dataset = get_dataset(config)(config, is_train=True)
+    train_loader = DataLoader(
+        dataset=train_dataset,
+        batch_size=config.TRAIN.BATCH_SIZE_PER_GPU,
+        shuffle=config.TRAIN.SHUFFLE,
+        num_workers=config.WORKERS,
+        pin_memory=config.PIN_MEMORY,
+    )
 
-    if args.model == 'LSTM-OCR':
-        model = BiLSTM(trainer_params)
+    val_dataset = get_dataset(config)(config, is_train=False)
+    val_loader = DataLoader(
+        dataset=val_dataset,
+        batch_size=config.TEST.BATCH_SIZE_PER_GPU,
+        shuffle=config.TEST.SHUFFLE,
+        num_workers=config.WORKERS,
+        pin_memory=config.PIN_MEMORY,
+    )
 
+    converter = strLabelConverter(config.DATASET.ALPHABETS)
 
-    # writer = SummaryWriter(log_dir='log/' + args.model  +  '/ptq')
-    save_dir = 'ckpt'
-    full_file = save_dir + '/mnist_' + trainer_params.reduce_bidirectional +'_' + str(trainer_params.bidirectional) + '.pt'
+    writer = SummaryWriter(log_dir='log/' + config.MODEL.NAME  +  '/ptq')
+    full_file = 'ckpt/360cc_' + config.MODEL.NAME + '.pt'
     model.load_state_dict(torch.load(full_file))
     model.to(device)
 
-    load_ptq = False
-    ptq_file_prefix = 'ckpt/mnist_' + trainer_params.reduce_bidirectional +'_' + str(trainer_params.bidirectional)  + '_ptq_'
+    load_ptq = True
+    ptq_file_prefix = 'ckpt/360cc_' + config.MODEL.NAME + '_ptq_'
 
     model.eval()
-    full_acc = full_inference(model, val_loader, val_data, args, trainer_params, decoder,criterion)
+    full_acc = full_inference(config, val_loader, val_dataset, converter, model, criterion, device)
 
-    # model_fold = fold_model(model)   # 
   
-    # full_params = []
+    model_fold = fold_model(model)   # 可以得到conv，bn，relu，fc的各个层 
     
-    # layer, par_ratio, flop_ratio = extract_ratio(args.model)
     
-    # layer = []
-    # for name, param in model.named_parameters():
-    #     if 'weight' in name:
-    #         n = name.split('.')
-    #         pre = '.'.join(n[:len(n)-1])
-    #         # 提取出weight前的名字(就是这个层的名字，if weight是避免bias重复提取一遍名字)
-    #         layer.append(pre)
+    full_params = []
 
-    # print('===================')
+    layer, par_ratio, flop_ratio = extract_ratio('lstm-ocr')
+    # print(layer)
+    layer = []
+    for name, param in model.named_parameters():
+        # 提取出weight前的名字(就是这个层的名字)
+        n = name.split('.')
+        pre = '.'.join(n[:len(n)-1])
+        # 避免重复添加
+        if pre not in layer:
+            layer.append(pre)
         
-    # par_ratio, flop_ratio = fold_ratio(layer, par_ratio, flop_ratio)
 
-    # for name, param in model_fold.named_parameters():
-    #     if 'bn' in name or 'sample.1' in name:
-    #         continue
-    #     param_norm = param.data.cpu()
-    #     full_params.append(param_norm)  # 没统计bn的 只统计了conv的 而且还是fold后的
-    #     writer.add_histogram(tag='Full_' + name + '_data', values=param.data)
+    print('===================')
+    # print(layer)
+    par_ratio, flop_ratio = fold_ratio(layer, par_ratio, flop_ratio)
+    # sys.exit()
+    for name, param in model_fold.named_parameters():
+        if 'bn' in name or 'sample.1' in name:
+            continue
+        # param_norm = F.normalize(param.data.cpu(),p=2,dim=-1)
+        param_data = param.data.cpu()
+        full_params.append(param_data)  # 统计了fold后的conv以及fc的weight和bias
+        writer.add_histogram(tag='Full_' + name + '_data', values=param.data)
+
+        # print(f"full-precision after Fold: name:{name}, param:{param.data}, param_data:{param_data}")
     
     
     gol._init()
@@ -291,10 +247,10 @@ if __name__ == "__main__":
         for num_bits in num_bit_list:
             e_bit_list = ebit_list(quant_type,num_bits)
             for e_bits in e_bit_list:
-                # model_ptq = resnet18()
-                if args.model == 'LSTM-OCR':
-                    model_ptq = BiLSTM(trainer_params)
-
+                # model_ptq = MobileNetV2()
+                model_ptq = get_crnn(config)
+                device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+                model_ptq = model_ptq.to(device)
 
                 if quant_type == 'FLOAT':
                     title = '%s_%d_E%d' % (quant_type, num_bits, e_bits)
@@ -319,59 +275,62 @@ if __name__ == "__main__":
                     model_ptq.to(device)
                     model_ptq.quantize(quant_type,num_bits,e_bits)
                     model_ptq.eval()
-                    direct_quantize(model_ptq, val_loader, val_data, args, trainer_params, decoder,criterion)
-
+                    direct_quantize(config, val_loader, val_dataset, converter, model_ptq, criterion, device)
                     # if args.save == True:
                     #     torch.save(model_ptq.state_dict(), ptq_file_prefix + title + '.pt')
 
-
                 model_ptq.freeze()
-                quantize_inference(model_ptq, val_loader, val_data, args, trainer_params, decoder,criterion)
-                
-                # ptq_acc = quantize_inference(model_ptq, val_loader, val_data, args, trainer_params, decoder,criterion)
-                # ptq_acc_list.append(ptq_acc)
-                # acc_loss = (full_acc - ptq_acc) / full_acc
-                # acc_loss_list.append(acc_loss)
-                # idx = -1
-
+                ptq_acc = quantize_inference(config, val_loader, val_dataset, converter, model_ptq, criterion, device)
+                ptq_acc_list.append(ptq_acc)
+                acc_loss = (full_acc - ptq_acc) / full_acc
+                acc_loss_list.append(acc_loss)
+                idx = -1
+
+                # 对权值参数反量化
+                model_ptq.fakefreeze()
                 # 获取计算量/参数量下的js-div
                 js_flops = 0.
                 js_param = 0.
+                for name, param in model_ptq.named_parameters():
+                    # if '.' not in name or 'bn' in name:
+                    if 'bn' in name or 'sample.1' in name:
+                        continue
+                    writer.add_histogram(tag=title +':'+ name + '_data', values=param.data)
+                    idx = idx + 1
+                    # renset中有多个. 需要改写拼一下
+                    # prefix = name.split('.')[0]
+                    n = name.split('.')
+                    prefix = '.'.join(n[:len(n) - 1])
+                     # weight和bias 1:1 ? 对于ratio，是按层赋予的，此处可以对weight和bias再单独赋予不同的权重，比如(8:2)
+                     # layer中只有conv,bn,fc, prefix只可能是conv，fc (顺序的，weight和bias都有)
+                    if prefix in layer:
+                        layer_idx = layer.index(prefix)
+                        ptq_param = param.data.cpu()
+                        # 取L2范数
+                        # ptq_norm = F.normalize(ptq_param,p=2,dim=-1)
+                        writer.add_histogram(tag=title +':'+ name + '_data', values=ptq_param)
+                        # print(name)
+                        # print('=========')
+                        # print(ptq_norm)
+                        # print('=========')
+                        # print(full_params[idx])
+
+                        # full_params中只有conv和fc 且是顺序下来的(weight和bias都有)
+                        js = js_div(ptq_param,full_params[idx])   # 这里算了fold后的量化前后模型的js距离
+                        js = js.item()
+                        origin_js = js
+                        # print()                               
+                        if js < 0.:
+                            js = 0.
+                        js_flops = js_flops + js * flop_ratio[layer_idx]
+                        js_param = js_param + js * par_ratio[layer_idx]
+                        print(f"prefix:{prefix}, layer_idx:{layer_idx}")
+                        # print(f"layer name:{prefix} layer_idx is:{layer_idx}  origin_js is {origin_js}  js is {js}  flop_ratio is {flop_ratio[layer_idx]}  par_ratio is {par_ratio[layer_idx]}  js_flops is {js_flops}  js_param is {js_param}")
+                js_flops_list.append(js_flops)
+                js_param_list.append(js_param)
+
+                print(title + ': js_flops: %f js_param: %f acc_loss: %f' % (js_flops, js_param, acc_loss))
     
-                # for name, param in model_ptq.named_parameters():
-                #     # if '.' not in name or 'bn' in name:
-                #     if 'bn' in name or 'sample.1' in name:
-                #         continue
-                #     writer.add_histogram(tag=title +':'+ name + '_data', values=param.data)
-                #     idx = idx + 1
-                #     # renset中有多个. 需要改写拼一下
-                #     # prefix = name.split('.')[0]
-                #     n = name.split('.')
-                #     prefix = '.'.join(n[:len(n) - 1])
-                #      # weight和bias 1:1 ? 对于ratio，是按层赋予的，此处可以对weight和bias再单独赋予不同的权重，比如(8:2)
-                #     if prefix in layer:
-                #         layer_idx = layer.index(prefix)
-                #         ptq_param = param.data.cpu()
-                #         # 取L2范数
-                #         # ptq_norm = F.normalize(ptq_param,p=2,dim=-1)
-                #         ptq_norm = ptq_param
-                #         writer.add_histogram(tag=title +':'+ name + '_data', values=ptq_param)
-                #         # print(name)
-                #         # print('=========')
-                #         # print(ptq_norm)
-                #         # print('=========')
-                #         # print(full_params[idx])
-                #         js = js_div(ptq_norm,full_params[idx])   # 这里算了fold后的量化前后模型的js距离
-                #         js = js.item()
-                #         if js < 0.:
-                #             js = 0.
-                #         js_flops = js_flops + js * flop_ratio[layer_idx]
-                #         js_param = js_param + js * par_ratio[layer_idx]
-                # js_flops_list.append(js_flops)
-                # js_param_list.append(js_param)
-
-                # print(title + ': js_flops: %f js_param: %f acc_loss: %f' % (js_flops, js_param, acc_loss))
-    sys.exit()
     # 写入xlsx
     workbook = openpyxl.Workbook()
     worksheet = workbook.active
@@ -389,10 +348,10 @@ if __name__ == "__main__":
         worksheet.cell(row=i+4, column=4, value=ptq_acc_list[i])
         worksheet.cell(row=i+4, column=5, value=acc_loss_list[i])
     
-    workbook.save('ptq_result_' + args.model + '.xlsx')
+    workbook.save('ptq_result_' + config.MODEL.NAME + '.xlsx')
     writer.close()
 
-    ft = open('ptq_result_' + args.model + '.txt','w')
+    ft = open('ptq_result_' + config.MODEL.NAME + '.txt','w')
 
     print('title_list:',file=ft)
     print(" ".join(title_list),file=ft)

mzh/new_mzh/LSTM-OCR/ptq_result_lstm-ocr.txt

+title_list:
+INT_2 INT_3 INT_4 INT_5 INT_6 INT_7 INT_8 INT_9 INT_10 INT_11 INT_12 INT_13 INT_14 INT_15 INT_16 POT_2 POT_3 POT_4 POT_5 POT_6 POT_7 POT_8 FLOAT_3_E1 FLOAT_4_E1 FLOAT_4_E2 FLOAT_5_E1 FLOAT_5_E2 FLOAT_5_E3 FLOAT_6_E1 FLOAT_6_E2 FLOAT_6_E3 FLOAT_6_E4 FLOAT_7_E1 FLOAT_7_E2 FLOAT_7_E3 FLOAT_7_E4 FLOAT_7_E5 FLOAT_8_E1 FLOAT_8_E2 FLOAT_8_E3 FLOAT_8_E4 FLOAT_8_E5 FLOAT_8_E6
+js_flops_list:
+938.7733335552051 422.2961529040284 112.00572935619252 28.15632049948652 6.580025053557225 1.593147596698573 0.39127389747221836 0.09726359211545822 0.02424534011691382 0.006096811349829773 0.0015819717562818567 0.0004088973671629054 0.00010591612996765123 4.215430786749398e-05 3.358851529044255e-05 938.0795941095593 276.1298308688724 43.56510410351131 43.31743097440866 43.31651992748872 43.317170796987085 43.316909690885424 245.27340648023244 80.55429874312716 50.01809924502505 38.66623068838267 17.323188075553762 12.265772098808744 26.78403199897619 9.167642811282255 3.1779184189380314 12.23167946700013 22.667666751272108 6.595860316723723 0.8016993676302422 3.1649891963442665 12.231686340919987 21.030391880048075 5.630180110297391 0.20824827230948417 0.7939111200807045 3.164995719146672 12.23168958358928
+js_param_list:
+1083.2152054453256 407.68102883090796 142.77271018968833 40.694342955319094 9.528033474559528 2.3056188772063395 0.5663514708991146 0.1405112916168392 0.03504902772537398 0.008813186313906787 0.0022363630716486275 0.000566627677334268 0.00014559649495622196 4.1442517690921754e-05 3.062477331841085e-05 1083.0293894297583 286.18087410960214 48.04799916302135 47.56581240023642 47.5640587789577 47.565362072997914 47.56507051619791 306.72068771644035 110.02866499298202 52.861756413549614 51.56753626886545 17.228751722832826 13.602914776865827 35.21196010598172 8.706064889198597 3.4432562752359903 13.53603392058786 29.65906818846962 6.118510949236464 0.8693586052802382 3.417883320602458 13.536039876528541 27.482646086738573 5.183734622246258 0.22716556760816775 0.8541492170710019 3.417896091306703 13.536039813988335
+ptq_acc_list:
+0.0 0.0 0.0 0.172 0.627 0.772 0.7856666666666666 0.791 0.791 0.7936666666666666 0.793 0.7936666666666666 0.794 0.7936666666666666 0.794 0.0 0.0 0.087 0.09433333333333334 0.086 0.09533333333333334 0.09366666666666666 0.0 0.001 0.0033333333333333335 0.18733333333333332 0.343 0.591 0.4836666666666667 0.5843333333333334 0.7566666666666667 0.5906666666666667 0.557 0.657 0.7823333333333333 0.7503333333333333 0.5883333333333334 0.597 0.6843333333333333 0.792 0.785 0.7486666666666667 0.5933333333333334
+acc_loss_list:
+1.0 1.0 1.0 0.7834662190516157 0.21065883340327318 0.02811582039446074 0.010910616869492292 0.004196391103650818 0.004196391103650818 0.0008392782207302194 0.001678556441460299 0.0008392782207302194 0.00041963911036503983 0.0008392782207302194 0.00041963911036503983 1.0 1.0 0.8904741921947126 0.8812421317666806 0.8917331095258079 0.8799832144355854 0.8820814099874108 1.0 0.9987410826689047 0.9958036088963492 0.7641628199748216 0.5681913554343264 0.2559798573227025 0.39110365086026017 0.26437263953000417 0.04741921947125467 0.25639949643306753 0.2987830465799412 0.17289131347041542 0.01510700797314311 0.055392362568191404 0.25933697020562313 0.24842635333613097 0.13848090642047836 0.002937473772555558 0.011749895090222373 0.057490558120016744 0.2530423835501468

mzh/new_mzh/LSTM-OCR/readme.md

+## update 2023.5.14
+
+### 采用了CRNN的结构实现LSTM-OCR，相比于之前的版本有了更好、更通用的处理OCR问题的能力。我使用了360CC数据集(中文OCR训练数据集，共360w张图片，我只用了其中3w训练数据和3k张测试数据)来训练和测试模型。
+
+1. config文件夹里是配置文件:
+
+  - alphabets.py是字典
+  - 360CC_config.yaml是使用360CC数据集时的配置文件
+  - OWN_config.yaml是使用其他数据集时的配置文件
+  
+
+2. dataset文件夹:
+
+- txt文件夹中的char_std_5990.txt是360cc数据集的字典
+- test.txt中是配置测试集数据
+- train.txt中是配置训练集数据
+- _360cc.py用于构建360cc dataset来训练/测试
+- _own.py用于构建自己的dataset来训练/测试
+
+3. lstm_utils.py中是LSTM-OCR中需要用到的一些特定的函数、类等
+
+4. module.py中增加了对LSTM量化相关的组件，补充了对于BiLSTM的部分权值参数的伪量化，修改了QLinear的quantize_inference方法，使其仍然是真量化。
+
+5. train.py中的train和validate函数考虑了OCR问题的特点，有了较大修改。同理ptq.py中的direct_inference, full_inference, quantize_inference也有一定修改。
+
+6. 由于QLSTM采用伪量化，而其他层采用真量化，因此在model.py中与QLSTM直接连接的层需要通过dequantize_tensor(x)或quantize_tensor(x)来处理下数据。
+
+7. ptq_result_lstm-ocr.txt和ptq_result_lstm-ocr.xlsx是实验结果，仍采用acc作为评价指标，当一个句子被完全识别正确时，算是识别正确一次。
+
+8. 拟合图像如下 <br>
+flops:
+<img src = "fig/flops_lstmocr.png" class="h-90 auto">
+params:
+<img src = "fig/params_lstmocr.png" class="h-90 auto">
+
+
 
 ## update 2023.5.4
 

mzh/new_mzh/LSTM-OCR/train.py

+import argparse
+from easydict import EasyDict as edict
+import yaml
+import os
+import torch
+import torch.backends.cudnn as cudnn
+from torch.utils.data import DataLoader
+from model import *
+from utils import *
+from lstm_utils import *
+from dataset import get_dataset
+from config.alphabets import *
+import time
+import sys
+
+from torch.utils.tensorboard import SummaryWriter
+
+def parse_arg():
+    parser = argparse.ArgumentParser(description="train crnn")
+
+    # parser.add_argument('--cfg', help='experiment configuration filename', required=True, type=str)
+    parser.add_argument('--cfg', help='experiment configuration filename', type=str, default='config/360CC_config.yaml')
+
+    args = parser.parse_args()
+
+    with open(args.cfg, 'r') as f:
+        # config = yaml.load(f, Loader=yaml.FullLoader)
+        config = yaml.load(f,Loader=yaml.FullLoader)
+        config = edict(config)
+
+    config.DATASET.ALPHABETS = alphabet
+    config.MODEL.NUM_CLASSES = len(config.DATASET.ALPHABETS)
+
+    return config
+
+
+def train(config, train_loader, dataset, converter, model, criterion, optimizer, device, epoch):
+
+    batch_time = AverageMeter()
+    data_time = AverageMeter()
+    losses = AverageMeter()
+
+    model.train()
+
+    end = time.time()
+    for i, (inp, idx) in enumerate(train_loader):
+        # measure data time
+        data_time.update(time.time() - end)
+
+        labels = get_batch_label(dataset, idx)
+        inp = inp.to(device)
+
+        # inference
+        preds = model(inp).cpu()
+
+        # compute loss
+        batch_size = inp.size(0)
+        text, length = converter.encode(labels)                    # length = 一个batch中的总字符长度, text = 一个batch中的字符所对应的下标
+        preds_size = torch.IntTensor([preds.size(0)] * batch_size) # timestep * batchsize
+        loss = criterion(preds, text, preds_size, length)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        losses.update(loss.item(), inp.size(0))
+
+        batch_time.update(time.time()-end)
+        if i % config.PRINT_FREQ == 0:
+            msg = 'Epoch: [{0}][{1}/{2}]\t' \
+                  'Time {batch_time.val:.3f}s ({batch_time.avg:.3f}s)\t' \
+                  'Speed {speed:.1f} samples/s\t' \
+                  'Data {data_time.val:.3f}s ({data_time.avg:.3f}s)\t' \
+                  'Loss {loss.val:.5f} ({loss.avg:.5f})\t'.format(
+                      epoch, i, len(train_loader), batch_time=batch_time,
+                      speed=inp.size(0)/batch_time.val,
+                      data_time=data_time, loss=losses)
+            print(msg)
+
+        end = time.time()
+
+
+def validate(config, val_loader, dataset, converter, model, criterion, device, epoch):
+
+    losses = AverageMeter()
+    model.eval()
+
+    n_correct = 0
+    with torch.no_grad():
+        for i, (inp, idx) in enumerate(val_loader):
+
+            # 一个batch的label的list
+            labels = get_batch_label(dataset, idx)
+            inp = inp.to(device)
+
+            # inference
+            preds = model(inp).cpu()
+
+            # compute loss
+            batch_size = inp.size(0)
+            text, length = converter.encode(labels)
+            preds_size = torch.IntTensor([preds.size(0)] * batch_size)
+            loss = criterion(preds, text, preds_size, length)
+
+            losses.update(loss.item(), inp.size(0))
+
+            _, preds = preds.max(2)
+            preds = preds.transpose(1, 0).contiguous().view(-1)
+            # sim_preds是decode后的，得到了一个str或者一个str list (也是一个batch的)
+            sim_preds = converter.decode(preds.data, preds_size.data, raw=False)
+            # target应该是一个list  这里相当于在判断字符串是否相等
+            for pred, target in zip(sim_preds, labels):
+                if pred == target:
+                    n_correct += 1
+
+            if (i + 1) % config.PRINT_FREQ == 0:
+                print('Epoch: [{0}][{1}/{2}]'.format(epoch, i, len(val_loader)))
+
+            # if i == config.TEST.NUM_TEST_BATCH:  # 只检查这些数据的  (config.TEST.NUM_TEST_BATCH个batch)
+            #     break
+
+    # 只打印展示前 config.TEST.NUM_TEST_DISP 个句子的对比
+    raw_preds = converter.decode(preds.data, preds_size.data, raw=True)[:config.TEST.NUM_TEST_DISP]
+    for raw_pred, pred, gt in zip(raw_preds, sim_preds, labels):
+        print('%-20s => %-20s, gt: %-20s' % (raw_pred, pred, gt))
+
+    # num_test_sample = config.TEST.NUM_TEST_BATCH * config.TEST.BATCH_SIZE_PER_GPU
+    # if num_test_sample > len(dataset):
+    #     num_test_sample = len(dataset)
+
+    num_test_sample = len(dataset)
+    print("[#correct:{} / #total:{}]".format(n_correct, num_test_sample))
+    accuracy = n_correct / float(num_test_sample)
+    print('Test loss: {:.4f}, accuray: {:.4f}'.format(losses.avg, accuracy))
+
+
+    return accuracy
+
+def main():
+
+    # load config
+    config = parse_arg()
+
+    # construct face related neural networks
+    model = get_crnn(config)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    model = model.to(device)
+
+    # define loss function
+    criterion = torch.nn.CTCLoss()
+
+    # 上一轮训练的epoch  (起始为0 )
+    last_epoch = config.TRAIN.BEGIN_EPOCH
+
+
+    train_dataset = get_dataset(config)(config, is_train=True)
+    train_loader = DataLoader(
+        dataset=train_dataset,
+        batch_size=config.TRAIN.BATCH_SIZE_PER_GPU,
+        shuffle=config.TRAIN.SHUFFLE,
+        num_workers=config.WORKERS,
+        pin_memory=config.PIN_MEMORY,
+    )
+
+    val_dataset = get_dataset(config)(config, is_train=False)
+    val_loader = DataLoader(
+        dataset=val_dataset,
+        batch_size=config.TEST.BATCH_SIZE_PER_GPU,
+        shuffle=config.TEST.SHUFFLE,
+        num_workers=config.WORKERS,
+        pin_memory=config.PIN_MEMORY,
+    )
+
+    optimizer = get_optimizer(config, model)
+    if isinstance(config.TRAIN.LR_STEP, list):
+        lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(
+            optimizer, config.TRAIN.LR_STEP,
+            config.TRAIN.LR_FACTOR, last_epoch-1
+        )
+    else:
+        lr_scheduler = torch.optim.lr_scheduler.StepLR(
+            optimizer, config.TRAIN.LR_STEP,
+            config.TRAIN.LR_FACTOR, last_epoch - 1
+        )
+
+
+    save_dir = 'ckpt'
+    if not os.path.isdir(save_dir):
+        os.makedirs(save_dir, mode=0o777)
+        os.chmod(save_dir, mode=0o777)
+        
+    converter = strLabelConverter(config.DATASET.ALPHABETS)
+
+    # for resume, only 
+    if config.TRAIN.RESUME.IS_RESUME:       
+       model.load_state_dict(torch.load(save_dir + '/360cc_' + config.MODEL.NAME + '.pt'))
+       acc = validate(config, val_loader, val_dataset, converter, model, criterion, device, config.TRAIN.END_EPOCH)
+       print(f"test accuracy: {acc:.2f}%")
+       sys.exit()
+
+
+
+    best_acc = 0.0
+    
+    for epoch in range(last_epoch, config.TRAIN.END_EPOCH):
+
+        # train
+        train(config, train_loader, train_dataset, converter, model, criterion, optimizer, device, epoch)
+        lr_scheduler.step()
+
+        # validate
+        acc = validate(config, val_loader, val_dataset, converter, model, criterion, device, epoch)
+
+        # save best ckpt
+        if config.TRAIN.SAVE and acc>best_acc:
+           torch.save(model.state_dict(), save_dir + '/360cc_' + config.MODEL.NAME + '.pt')
+
+        best_acc = max(acc, best_acc)
+        print("best acc is:", best_acc)
+
+if __name__ == '__main__':
+
+    main()
\ No newline at end of file

mzh/new_mzh/LSTM-OCR/utils.py

 import torch
 import torch.nn as nn
 import torch.nn.functional as F
+import torch.optim as optim
+
 
 def js_div(p_output, q_output, get_softmax=True):
     """
@@ -167,3 +169,5 @@ def fold_bn(conv, bn):
     
 
     return conv
+
+

mzh/new_mzh/Naive_MIA/attack.py

+#!/usr/bin/python
+
+import torch
+import torch.nn as nn
+import torchvision
+import torchvision.transforms as transforms
+from torch.utils.data.dataset import TensorDataset
+from torch.utils.data.sampler import SubsetRandomSampler
+import model
+from model import init_params as w_init
+from train import  train_model, train_attack_model, prepare_attack_data
+from sklearn.metrics import classification_report
+from sklearn.metrics import precision_score, recall_score
+import argparse
+import numpy as np
+import os
+import copy
+import random
+import matplotlib.pyplot as plt
+from torch.optim.lr_scheduler import CosineAnnealingLR
+
+
+#set the seed for reproducibility
+np.random.seed(1234)
+#Flag to enable early stopping
+need_earlystop = False
+
+########################
+# Model Hyperparameters
+########################
+#Number of layer's parameters for target and shadow models 
+# target_filters = [128, 256, 256]
+# shadow_filters = [64, 128, 128]
+#New FC layers size for pretrained model
+# n_fc= [256, 128] 
+#For CIFAR-10 and MNIST dataset
+num_classes = 10
+# #No. of training epocs
+# num_epochs = 100
+# #how many samples per batch to load
+# batch_size = 128
+# #learning rate
+# learning_rate = 0.001
+# #Learning rate decay 
+# lr_decay = 0.0001
+#Regularizer
+reg=1e-4
+#percentage of dataset to use for shadow model
+shadow_split = 0.6
+#Number of validation samples
+n_validation = 1000
+#Number of processes
+num_workers = 2
+#Hidden units for MNIST model
+n_hidden_mnist = 32
+
+
+################################
+#Attack Model Hyperparameters
+################################
+NUM_EPOCHS = 100
+BATCH_SIZE = 10
+#Learning rate
+LR_ATTACK = 0.1 
+#L2 Regulariser
+REG = 1e-7
+#weight decay
+LR_DECAY = 0.96
+#No of hidden units
+n_hidden = 256
+#Binary Classsifier
+out_classes = 2
+
+
+
+def get_cmd_arguments():
+    parser = argparse.ArgumentParser(prog="Membership Inference Attack")
+    parser.add_argument('--model', default='resnet18', type=str, choices=['resnet18', 'resnet50', 'resnet152', 'mobilenetv2'], help='Which model to use (resnet18,50,152 or mobilenet)')
+    parser.add_argument('--dataset', default='CIFAR10', type=str, choices=['CIFAR10', 'CIFAR100' ,'MNIST'], help='Which dataset to use (CIFAR10 or MNIST)')
+    parser.add_argument('--dataPath', default='/lustre/datasets', type=str, help='Path to store data')
+    parser.add_argument('--modelPath', default='./ckpt',type=str, help='Path to save or load model checkpoints')
+
+    parser.add_argument('-e','--num_epochs', default=100, type=int, metavar='EPOCHS', help='number of total epochs to run')
+    parser.add_argument('-b', '--batch_size', default=128, type=int, metavar='BATCH SIZE', help='mini-batch size (default: 128)')
+    parser.add_argument('-lr', '--learning_rate', default=0.001, type=float, metavar='LR', help='initial learning rate')
+    parser.add_argument('-wd','--lr_decay',default=0.0001,type=float,metavar='WD',help='lr schduler weight decay')
+    parser.add_argument('-j','--num_workers', default=2, type=int, metavar='WORKERS',help='number of data loading workers (default: 4)')
+
+    parser.add_argument('--trainTargetModel', action='store_true', help='Train a target model, if false then load an already trained model')
+    parser.add_argument('--trainShadowModel', action='store_true', help='Train a shadow model, if false then load an already trained model')
+    parser.add_argument('--trainAttackModel', action='store_true', help='Train an attack model, if false then load an already trained model')
+    parser.add_argument('--need_augm',action='store_true', help='To use data augmentation on target and shadow training set or not')
+    parser.add_argument('--need_topk',action='store_true', help='Flag to enable using Top 3 posteriors for attack data')
+    parser.add_argument('--param_init', action='store_true', help='Flag to enable custom model params initialization')
+    parser.add_argument('--verbose',action='store_true', help='Add Verbosity')
+    return parser.parse_args()
+
+def get_data_transforms(dataset, augm=False):
+
+    if dataset == 'CIFAR10':
+        normalize = transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010))
+        test_transforms = transforms.Compose([transforms.ToTensor(),
+                                            normalize])
+
+        if augm:
+            train_transforms = transforms.Compose([transforms.RandomRotation(5),
+                                                transforms.RandomHorizontalFlip(p=0.5),
+                                                transforms.RandomCrop(32, padding=4),
+                                                transforms.ToTensor(),
+                                                normalize]) 
+        else:
+            train_transforms = transforms.Compose([transforms.ToTensor(),
+                                                normalize])
+
+    elif dataset == 'CIFAR100':
+        normalize = transforms.Normalize((0.507, 0.487, 0.441), (0.267, 0.256, 0.276))
+        test_transforms = transforms.Compose([transforms.ToTensor(),
+                                            normalize])
+
+        if augm:
+            train_transforms = transforms.Compose([transforms.RandomRotation(5),
+                                                transforms.RandomHorizontalFlip(p=0.5),
+                                                transforms.RandomCrop(32, padding=4),
+                                                transforms.ToTensor(),
+                                                normalize]) 
+        else:
+            train_transforms = transforms.Compose([transforms.ToTensor(),
+                                                normalize])
+    else:
+        #The values 0.1307 and 0.3081 used for the Normalize() transformation below are the global mean and standard deviation 
+        #of the MNIST dataset
+        test_transforms = torchvision.transforms.Compose([torchvision.transforms.ToTensor(),
+                                                        torchvision.transforms.Normalize((0.1307,), (0.3081,))])
+        if augm:
+            train_transforms = torchvision.transforms.Compose([transforms.RandomRotation(5),
+                                                    transforms.RandomHorizontalFlip(p=0.5),
+                                                    torchvision.transforms.ToTensor(),
+                                                    torchvision.transforms.Normalize((0.1307,), (0.3081,))])
+        else:
+      
+            train_transforms = torchvision.transforms.Compose([torchvision.transforms.ToTensor(),
+                                                    torchvision.transforms.Normalize((0.1307,), (0.3081,))])
+        
+    return train_transforms, test_transforms
+
+
+def split_dataset(train_dataset):
+    #For simplicity we are only using orignal training set and splitting into 4 equal parts
+    #and assign it to Target train/test and Shadow train/test.
+    total_size = len(train_dataset)
+    split1 = total_size // 4
+    split2 = split1*2
+    split3 = split1*3
+    
+    # 0 ~ total_size-1
+    indices = list(range(total_size))
+    
+    np.random.shuffle(indices)
+    
+    #Shadow model train and test set
+    s_train_idx = indices[:split1]
+    s_test_idx = indices[split1:split2]
+
+    #Target model train and test set
+    t_train_idx = indices[split2:split3]
+    t_test_idx = indices[split3:]
+    
+    return s_train_idx, s_test_idx,t_train_idx,t_test_idx
+    
+
+#--------------------------------------------------------------------------------
+# Get dataloaders for Shadow and Target models 
+# Data Strategy:
+# - Split the entire training dataset into 4 parts(T_tain, T_test, S_train, S_test)
+#  Target -  Train on T_train and T_test
+#  Shadow -  Train on S_train and S_test
+#  Attack - Use T_train and T_test for evaluation
+#           Use S_train and S_test for training
+#--------------------------------------------------------------------------------
+
+def get_data_loader(dataset,
+                    data_dir,
+                    batch,
+                    shadow_split=0.5,  # 目前还没用上
+                    augm_required=False,
+                    num_workers=1):
+    """
+     Utility function for loading and returning train and valid
+     iterators over the CIFAR-10 and MNIST dataset.
+    """ 
+    # 暂时固定了shadow split为1：1：1：1
+    # error_msg = "[!] shadow_split should be in the range [0, 1]."
+    # assert ((shadow_split >= 0) and (shadow_split <= 1)), error_msg
+    
+    
+    train_transforms, test_transforms = get_data_transforms(dataset,augm_required)
+        
+    #Download test and train dataset
+    print(f"Using Dataset:{dataset}")
+    if dataset == 'CIFAR10':
+        #CIFAR10 training set
+        train_set = torchvision.datasets.CIFAR10(root=data_dir,
+                                                    train=True,
+                                                    transform=train_transforms,
+                                                    download=False)  
+        #CIFAR10 test set 在之前版中没用上，现可以考虑将其与train_set合并，构成一个统一的train set
+        test_set = torchvision.datasets.CIFAR10(root=data_dir, 
+                                                train = False,  
+                                                transform = test_transforms)
+        train_set = torch.utils.data.ConcatDataset([train_set, test_set])
+        # 原本只分割了train set (包含了到in，out)，效果等同于对train set再做划分得到t_in(t_train),t_out(t_test),s_in(s_train),s_out(s_test)，后续版本将test_set也合并进来了
+        s_train_idx, s_out_idx, t_train_idx, t_out_idx = split_dataset(train_set)   
+    elif dataset == 'CIFAR100':
+        train_set = torchvision.datasets.CIFAR100(root=data_dir,
+                                                    train=True,
+                                                    transform=train_transforms,
+                                                    download=False)  
+        test_set = torchvision.datasets.CIFAR100(root=data_dir, 
+                                                train = False,  
+                                                transform = test_transforms)
+        train_set = torch.utils.data.ConcatDataset([train_set, test_set])
+        s_train_idx, s_out_idx, t_train_idx, t_out_idx = split_dataset(train_set)  
+    elif dataset == 'MNIST':
+        #MNIST train set
+        train_set = torchvision.datasets.MNIST(root=data_dir,
+                                        train=True,
+                                        transform=train_transforms,
+                                        download=False)
+        #MNIST test set
+        test_set = torchvision.datasets.MNIST(root=data_dir, 
+                                        train = False,  
+                                        transform = test_transforms)
+        train_set = torch.utils.data.ConcatDataset([train_set, test_set])
+        
+        s_train_idx, s_out_idx, t_train_idx, t_out_idx = split_dataset(train_set)
+   
+    
+    # Data samplers
+    s_train_sampler = SubsetRandomSampler(s_train_idx)
+    s_out_sampler = SubsetRandomSampler(s_out_idx)
+    t_train_sampler = SubsetRandomSampler(t_train_idx)
+    t_out_sampler = SubsetRandomSampler(t_out_idx)
+       
+
+    #In our implementation we are keeping validation set to measure training performance
+    #From the held out set for target and shadow, we take n_validation samples. 
+    #As train set is already small we decided to take valid samples from held out set
+    #as these are samples not used in training. 
+    # if dataset == 'CIFAR10':
+        # out中的前n_validation个元素构成了in的val集 (没用于训练，可以)
+        # target_val_idx = t_out_idx[:n_validation]
+        # shadow_val_idx = s_out_idx[:n_validation]
+        # 将val set 等同于 test set
+    target_val_idx = t_out_idx
+    shadow_val_idx = s_out_idx
+    
+    t_val_sampler = SubsetRandomSampler(target_val_idx)
+    s_val_sampler = SubsetRandomSampler(shadow_val_idx)
+    # elif dataset == 'CIFAR100':
+    #     target_val_idx = t_out_idx
+    #     shadow_val_idx = s_out_idx
+
+    #     t_val_sampler = SubsetRandomSampler(target_val_idx)
+    #     s_val_sampler = SubsetRandomSampler(shadow_val_idx)
+    # # elif dataset == 'MNIST':
+    #     # target_val_idx = t_out_idx[:n_validation]
+    #     # shadow_val_idx = s_out_idx[:n_validation]
+    #     target_val_idx = t_out_idx
+    #     shadow_val_idx = s_out_idx
+
+    #     t_val_sampler = SubsetRandomSampler(target_val_idx)
+    #     s_val_sampler = SubsetRandomSampler(shadow_val_idx)
+    
+
+    #-------------------------------------------------
+    # Data loader
+    #-------------------------------------------------
+    # if dataset == 'CIFAR10':
+
+    t_train_loader = torch.utils.data.DataLoader(dataset=train_set, 
+                                            batch_size=batch, 
+                                            sampler = t_train_sampler,
+                                            num_workers=num_workers)
+                                            
+    t_out_loader = torch.utils.data.DataLoader(dataset=train_set,
+                                            batch_size=batch,
+                                            sampler = t_out_sampler,
+                                            num_workers=num_workers)
+                                            
+    t_val_loader = torch.utils.data.DataLoader(dataset=train_set,
+                                            batch_size=batch,
+                                            sampler=t_val_sampler,
+                                            num_workers=num_workers)
+        
+    s_train_loader = torch.utils.data.DataLoader(dataset=train_set,
+                                            batch_size=batch,
+                                            sampler=s_train_sampler,
+                                            num_workers=num_workers)
+                                            
+    s_out_loader = torch.utils.data.DataLoader(dataset=train_set,
+                                            batch_size=batch,
+                                            sampler=s_out_sampler,
+                                            num_workers=num_workers)
+        
+    s_val_loader = torch.utils.data.DataLoader(dataset=train_set,
+                                            batch_size=batch,
+                                            sampler=s_val_sampler,
+                                            num_workers=num_workers)
+
+    
+    # elif dataset == 'CIFAR100':
+
+    #     t_train_loader = torch.utils.data.DataLoader(dataset=train_set, 
+    #                                         batch_size=batch, 
+    #                                         sampler = t_train_sampler,
+    #                                         num_workers=num_workers)
+                                            
+    #     t_out_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler = t_out_sampler,
+    #                                         num_workers=num_workers)
+                                            
+    #     t_val_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=t_val_sampler,
+    #                                         num_workers=num_workers)
+        
+    #     s_train_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=s_train_sampler,
+    #                                         num_workers=num_workers)
+                                            
+    #     s_out_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=s_out_sampler,
+    #                                         num_workers=num_workers)
+        
+    #     s_val_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=s_val_sampler,
+    #                                         num_workers=num_workers)
+        
+    # elif dataset == 'MNIST':
+    #     t_train_loader = torch.utils.data.DataLoader(dataset=train_set, 
+    #                                         batch_size=batch, 
+    #                                         sampler=t_train_sampler,
+    #                                         num_workers=num_workers)
+    
+    #     t_out_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=t_out_sampler,
+    #                                         num_workers=num_workers)
+        
+    #     t_val_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=t_val_sampler,
+    #                                         num_workers=num_workers)
+        
+    #     s_train_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=s_train_sampler,
+    #                                         num_workers=num_workers)
+                                            
+    #     s_out_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=s_out_sampler,
+    #                                         num_workers=num_workers)
+        
+    #     s_val_loader = torch.utils.data.DataLoader(dataset=train_set,
+    #                                         batch_size=batch,
+    #                                         sampler=s_val_sampler,
+    #                                         num_workers=num_workers)
+    
+
+      
+    print('Total Combined Test samples in {} dataset : {}'.format(dataset, len(train_set))) 
+    # print('Total Train samples in {} dataset : {}'.format(dataset, len(train_set)))  
+    print('Number of Target train samples: {}'.format(len(t_train_sampler)))
+    print('Number of Target valid samples: {}'.format(len(t_val_sampler)))  #从test中取出来了部分做val
+    print('Number of Target test samples: {}'.format(len(t_out_sampler)))
+    print('Number of Shadow train samples: {}'.format(len(s_train_sampler)))
+    print('Number of Shadow valid samples: {}'.format(len(s_val_sampler)))  #从test中取出来了部分做val
+    print('Number of Shadow test samples: {}'.format(len(s_out_sampler)))
+   
+
+    return t_train_loader, t_val_loader, t_out_loader, s_train_loader, s_val_loader, s_out_loader
+
+
+def attack_inference(model,
+                    test_X,
+                    test_Y,
+                    device):
+    
+    print('----Attack Model Testing----')
+
+    targetnames= ['Non-Member', 'Member']
+    pred_y = []
+    true_y = []
+    
+    #Tuple of tensors => full tensor    
+    # (batch_size, channels, height, width)=>(total_samples, channels, height, width)
+    X = torch.cat(test_X)
+    Y = torch.cat(test_Y)
+    print(f"Y:{Y}")
+
+    #Create Inference dataset (reconstructed)
+    inferdataset = TensorDataset(X,Y) 
+    print('Length of Attack Model test dataset : [{}]'.format(len(inferdataset)))
+
+    dataloader = torch.utils.data.DataLoader(dataset=inferdataset,
+                                            batch_size=50,
+                                            shuffle=False,
+                                            num_workers=num_workers)
+
+    #Evaluation of Attack Model
+    model.eval() 
+    with torch.no_grad():
+        correct = 0
+        total = 0
+        for i, (inputs, labels) in enumerate(dataloader):
+            inputs = inputs.to(device)
+            labels = labels.to(device)
+            
+            outputs = model(inputs)
+            
+            #Predictions for accuracy calculations
+            _, predictions = torch.max(outputs.data, 1)
+            total+=labels.size(0)
+            correct+=(predictions == labels).sum().item()
+            
+            # print('True Labels for Batch [{}] are : {}'.format(i,labels))
+            # print('Predictions for Batch [{}] are : {}'.format(i,predictions))
+            
+            true_y.append(labels.cpu())
+            pred_y.append(predictions.cpu())
+        
+    attack_acc = correct / total
+    print('Attack Test Accuracy is  : {:.2f}%'.format(100*attack_acc))
+    
+    true_y =  torch.cat(true_y).numpy()
+    pred_y = torch.cat(pred_y).numpy()
+
+    print('---Detailed Results----')
+    print(classification_report(true_y,pred_y, target_names=targetnames))
+
+
+#Main Method to initate model training and attack
+def create_attack(modelname, dataset, dataPath, modelPath, num_epochs , batch_size,  learning_rate, lr_decay , num_workers,trainTargetModel, trainShadowModel,trainAttackModel, need_augm, need_topk, param_init, verbose):
+ 
+    dataset = dataset
+    need_augm = need_augm
+    verbose = verbose
+    #For using top 3 posterior probabilities 
+    top_k = need_topk
+
+    if dataset == 'CIFAR10' or dataset == 'CIFAR100':
+        img_size = 32
+        #Input Channels for the Image
+        input_dim = 3
+    else:#MNIST
+        img_size = 28
+        input_dim = 1
+
+    datasetDir = os.path.join(dataPath,dataset)
+    modelDir = os.path.join(modelPath, dataset)  
+    modelDir = os.path.join(modelDir,modelname)
+    
+    #Create dataset and model directories
+    if not os.path.exists(datasetDir):
+        try:
+            os.makedirs(datasetDir)
+        except OSError:
+            pass
+    
+    if not os.path.exists(modelDir):
+        try:
+            os.makedirs(modelDir)
+        except OSError:
+            pass 
+
+    # setting device on GPU if available, else CPU
+    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    print('Using device:', device)
+
+    #Creating data loaders
+    # val是由test提取出来的部分
+    t_train_loader, t_val_loader, t_test_loader,\
+    s_train_loader, s_val_loader, s_test_loader = get_data_loader(dataset, 
+                                                                datasetDir,
+                                                                batch_size,
+                                                                shadow_split,
+                                                                need_augm,
+                                                                num_workers)
+    
+     
+    if (trainTargetModel):
+
+        if dataset == 'CIFAR10':          
+            # target_model = model.TargetNet(input_dim,target_filters,img_size,num_classes).to(device)
+            if modelname =='resnet18':
+                target_model = model.resnet18().to(device)
+            elif modelname == 'resnet50':
+                target_model = model.resnet50().to(device)
+            elif modelname == 'resnet152':
+                target_model = model.resnet152().to(device)
+            elif modelname == 'mobilenetv2':
+                target_model = model.MobileNetV2().to(device)
+        elif dataset == 'CIFAR100':
+            if modelname =='resnet18':
+                target_model = model.resnet18(num_classes=100).to(device)
+            elif modelname == 'resnet50':
+                target_model = model.resnet50(num_classes=100).to(device)
+            elif modelname == 'resnet152':
+                target_model = model.resnet152(num_classes=100).to(device)
+            elif modelname == 'mobilenetv2':
+                target_model = model.MobileNetV2(num_classes=100).to(device)
+        else:
+            target_model = model.MNISTNet(input_dim, n_hidden_mnist, num_classes).to(device)
+
+        if (param_init):
+            #Initialize params
+            target_model.apply(w_init) 
+        
+        
+        # Print the model we just instantiated
+        if verbose:
+            print('----Target Model Architecure----')
+            print(target_model)
+            print('----Model Learnable Params----')
+            for name,param in target_model.named_parameters():
+                 if param.requires_grad == True:
+                    print("\t",name)
+        
+
+        # Loss and optimizer for Tager Model
+        loss = nn.CrossEntropyLoss()
+        
+        # optimizer = torch.optim.Adam(target_model.parameters(), lr=learning_rate, weight_decay=reg)
+        # lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer,gamma=lr_decay)
+        optimizer = torch.optim.Adam(target_model.parameters(), lr=learning_rate)
+        lr_scheduler = CosineAnnealingLR(optimizer, T_max=num_epochs)
+
+        # return attackX，attckY (均由CIFAR0的train set构建)
+        targetX, targetY = train_model(target_model,
+                                    t_train_loader,
+                                    t_val_loader,
+                                    t_test_loader,  # 来源于CIFAR10的train+test set
+                                    loss,
+                                    optimizer,
+                                    lr_scheduler,
+                                    device,
+                                    modelDir,
+                                    verbose,
+                                    num_epochs,
+                                    top_k,
+                                    need_earlystop,
+                                    is_target=True)
+
+    else: #Target model training not required, load the saved checkpoint
+        target_file = os.path.join(modelDir,'best_target_model.pt')
+        print('Use Target model at the path ====> [{}] '.format(modelDir))
+        #Instantiate Target Model Class
+        if dataset == 'CIFAR10': 
+            # target_model = model.TargetNet(input_dim,target_filters,img_size,num_classes).to(device)
+            # target_model = model.resnet18().to(device)
+            if modelname =='resnet18':
+                target_model = model.resnet18().to(device)
+            elif modelname == 'resnet50':
+                target_model = model.resnet50().to(device)
+            elif modelname == 'resnet152':
+                target_model = model.resnet152().to(device)
+            elif modelname == 'mobilenetv2':
+                target_model = model.MobileNetV2().to(device)
+        elif dataset == 'CIFAR100':
+            if modelname =='resnet18':
+                target_model = model.resnet18(num_classes=100).to(device)
+            elif modelname == 'resnet50':
+                target_model = model.resnet50(num_classes=100).to(device)
+            elif modelname == 'resnet152':
+                target_model = model.resnet152(num_classes=100).to(device)
+            elif modelname == 'mobilenetv2':
+                target_model = model.MobileNetV2(num_classes=100).to(device)
+        else:
+            target_model = model.MNISTNet(input_dim,n_hidden_mnist,num_classes).to(device)
+
+        target_model.load_state_dict(torch.load(target_file))
+        print('---Peparing Attack Training data---')
+        t_trainX = []
+        t_trainY = []
+        t_testX = []
+        t_testY = []
+        # test dataset for attack model : in
+        t_trainX, t_trainY = prepare_attack_data(t_trainX,t_trainY,target_model,t_train_loader,device,top_k)
+        # print(f"len of t_trainX:{len(t_trainX)},len of t_trainY:{len(t_trainY)}")
+        # test dataset for attack model : out
+        t_testX, t_testY = prepare_attack_data(t_testX,t_testY,target_model,t_test_loader,device,top_k,test_dataset=True)
+        # print(f"len of t_testX:{len(t_testX)},len of t_testY:{len(t_testY)}")
+        # list concat
+        targetX = t_trainX + t_testX
+        targetY = t_trainY + t_testY
+        # print(f"len of targetX:{len(targetX)},len of targetY:{len(targetY)}")
+        # print(f"targetX:{targetX}")
+        # print(f"targetY:{targetY}")
+
+    if (trainShadowModel):
+
+        if dataset == 'CIFAR10' :
+            # shadow_model = model.ShadowNet(input_dim,shadow_filters,img_size,num_classes).to(device)
+            if modelname =='resnet18':
+                shadow_model = model.resnet18().to(device)
+            elif modelname == 'resnet50':
+                shadow_model = model.resnet50().to(device)
+            elif modelname == 'resnet152':
+                shadow_model = model.resnet152().to(device)
+            elif modelname == 'mobilenetv2':
+                shadow_model = model.MobileNetV2().to(device)
+        elif dataset == 'CIFAR100':
+            if modelname =='resnet18':
+                shadow_model = model.resnet18(num_classes=100).to(device)
+            elif modelname == 'resnet50':
+                shadow_model = model.resnet50(num_classes=100).to(device)
+            elif modelname == 'resnet152':
+                shadow_model = model.resnet152(num_classes=100).to(device)
+            elif modelname == 'mobilenetv2':
+                shadow_model = model.MobileNetV2(num_classes=100).to(device)
+        else:
+            #Using less hidden units than target model to mimic the architecture
+            n_shadow_hidden = 16 
+            shadow_model = model.MNISTNet(input_dim,n_shadow_hidden,num_classes).to(device)
+
+        if (param_init):
+            #Initialize params
+            shadow_model.apply(w_init)
+
+        # Print the model we just instantiated
+        if verbose:
+            print('----Shadow Model Architecure---')
+            print(shadow_model)
+            print('---Model Learnable Params----')
+            for name,param in shadow_model.named_parameters():
+                 if param.requires_grad == True:
+                    print("\t",name)
+        
+        # Loss and optimizer
+        shadow_loss = nn.CrossEntropyLoss()
+        # shadow_optimizer = torch.optim.Adam(shadow_model.parameters(), lr=learning_rate, weight_decay=reg)
+        # shadow_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(shadow_optimizer,gamma=lr_decay)
+
+        # shadow_optimizer = torch.optim.Adam(shadow_model.parameters(), lr=learning_rate)
+        shadow_optimizer = torch.optim.Adam(shadow_model.parameters(), lr=learning_rate)
+        shadow_lr_scheduler = CosineAnnealingLR(shadow_optimizer, T_max=num_epochs)
+
+        # 返回的是由shadow model得到的attack model的训练数据集
+        shadowX, shadowY = train_model(shadow_model,
+                                    s_train_loader,
+                                    s_val_loader,
+                                    s_test_loader,
+                                    shadow_loss,
+                                    shadow_optimizer,
+                                    shadow_lr_scheduler,
+                                    device,
+                                    modelDir,
+                                    verbose,
+                                    num_epochs,
+                                    top_k,
+                                    need_earlystop,
+                                    is_target=False)
+    else: #Shadow model training not required, load the saved checkpoint
+        print('Using Shadow model at the path  ====> [{}] '.format(modelDir))
+        shadow_file = os.path.join(modelDir,'best_shadow_model.pt')
+        assert os.path.isfile(shadow_file), 'Shadow Mode Checkpoint not found, aborting load'
+        #Instantiate Shadow Model Class
+        if dataset == 'CIFAR10' :
+            # shadow_model = model.ShadowNet(input_dim,shadow_filters,img_size,num_classes).to(device)
+            # shadow_model = model.resnet18().to(device)
+            if modelname =='resnet18':
+                shadow_model = model.resnet18().to(device)
+            elif modelname == 'resnet50':
+                shadow_model = model.resnet50().to(device)
+            elif modelname == 'resnet152':
+                shadow_model = model.resnet152().to(device)
+            elif modelname == 'mobilenetv2':
+                shadow_model = model.MobileNetV2().to(device)
+        elif dataset =='CIFAR100':
+            if modelname =='resnet18':
+                shadow_model = model.resnet18(num_classes=100).to(device)
+            elif modelname == 'resnet50':
+                shadow_model = model.resnet50(num_classes=100).to(device)
+            elif modelname == 'resnet152':
+                shadow_model = model.resnet152(num_classes=100).to(device)
+            elif modelname == 'mobilenetv2':
+                shadow_model = model.MobileNetV2(num_classes=100).to(device)
+
+        else:
+            #Using less hidden units than target model to mimic the architecture
+            n_shadow_hidden = 16
+            shadow_model = model.MNISTNet(input_dim,n_shadow_hidden,num_classes).to(device)
+
+        #Load the saved model
+        shadow_model.load_state_dict(torch.load(shadow_file))
+        #Prepare dataset for training attack model
+        print('----Preparing Attack training data---')
+        # 构造attack model的数据集 (train和test放在一起了)  (只能用shadow model的各种输出)
+        # in training dataset
+        trainX =[]
+        trainY = []
+        testX=[]
+        testY=[]
+        # train dataset for attack model: in
+        trainX, trainY = prepare_attack_data(trainX,trainY,shadow_model,s_train_loader,device,top_k)
+        # train dataset for attack model: out
+        testX, testY = prepare_attack_data(testX,testY,shadow_model,s_test_loader,device,top_k,test_dataset=True)
+        shadowX = trainX + testX
+        shadowY = trainY + testY    
+    
+
+    ###################################
+    # Attack Model Training
+    ##################################
+    #The input dimension to MLP attack model
+    input_size = shadowX[0].size(1)
+    print('Input Feature dim for Attack Model : [{}]'.format(input_size))
+    
+    attack_model = model.AttackMLP(input_size,n_hidden,out_classes).to(device)
+
+    if (trainAttackModel):
+        if (param_init):
+            #Initialize params
+            attack_model.apply(w_init)
+
+        # Loss and optimizer
+        attack_loss = nn.CrossEntropyLoss()
+        # attack_optimizer = torch.optim.Adam(attack_model.parameters(), lr=LR_ATTACK, weight_decay=REG)
+        # attack_lr_scheduler = torch.optim.lr_scheduler.ExponentialLR(attack_optimizer,gamma=LR_DECAY)
+        
+        # attack_optimizer = torch.optim.Adam(attack_model.parameters(), lr=learning_rate)
+        
+        attack_optimizer = torch.optim.SGD(attack_model.parameters(), lr=learning_rate,momentum=0.7, weight_decay=0.0001)
+        attack_lr_scheduler = CosineAnnealingLR(attack_optimizer, T_max=NUM_EPOCHS)
+    
+        #Feature vector and labels for training Attack model
+        # 只能用shadow model的train/test set (都属于CIFAR10的train set) 作为attack model的train set
+        attackdataset = (shadowX, shadowY)
+    
+        attack_valacc = train_attack_model(attack_model, attackdataset, attack_loss,
+                       attack_optimizer, attack_lr_scheduler, device, modelDir,
+                        NUM_EPOCHS, BATCH_SIZE, num_workers, verbose, earlystopping=False)
+    
+    
+        print('Validation Accuracy for the Best Attack Model is: {:.2f} %'.format(100* attack_valacc))
+    
+    
+   
+    else:
+        # TODO
+        #Load the trained attack model
+        attack_path = os.path.join(modelDir,'best_attack_model.pt')
+        attack_model.load_state_dict(torch.load(attack_path))
+    
+    #Inference on trained attack model
+    # 在target model的train/test set上去推理 (起始还是CIFAR10的train set的一部分) 
+    # train attack model的时候用的是shadow model的，因此这里target model的数据集成为了attack model的测试集
+    # target_X中是target model的output vector，与量化相关
+    attack_inference(attack_model, targetX, targetY, device)
+
+if __name__ == '__main__':
+    #get command line arguments from the user
+    args = get_cmd_arguments()
+    print(args)
+    #Generate Membership inference attack
+    create_attack(args.model,args.dataset, args.dataPath, args.modelPath, args.num_epochs , args.batch_size, args.learning_rate, args.lr_decay , args.num_workers,args.trainTargetModel, args.trainShadowModel, args.trainAttackModel,args.need_augm, args.need_topk, args.param_init, args.verbose)
\ No newline at end of file

mzh/new_mzh/Naive_MIA/function.py

+from torch.autograd import Function
+
+
+class FakeQuantize(Function):
+
+    @staticmethod
+    def forward(ctx, x, qparam):
+        x = qparam.quantize_tensor(x)
+        x = qparam.dequantize_tensor(x)
+        return x
+
+    @staticmethod
+    def backward(ctx, grad_output):
+        return grad_output, None

mzh/new_mzh/Naive_MIA/global_var.py

+class GlobalVariables:
+     SELF_INPLANES = 0

mzh/new_mzh/Naive_MIA/gol.py

+# -*- coding: utf-8 -*-
+
+# 用于多个module之间共享全局变量
+def _init():  # 初始化
+    global _global_dict
+    _global_dict = {}
+ 
+def set_value(value,is_bias=False):
+    # 定义一个全局变量
+    if is_bias:
+        _global_dict[0] = value
+    else:
+        _global_dict[1] = value
+ 
+ 
+def get_value(is_bias=False): # 给bias独立于各变量外的精度
+    if is_bias:
+        return _global_dict[0]
+    else:
+        return _global_dict[1]  
+

mzh/new_mzh/Naive_MIA/model.py

+import torch
+import torch.nn as nn
+from torchvision import models
+
+import torch.nn.functional as F
+
+from module import *
+import module
+from global_var import GlobalVariables
+
+
+#Below methods to claculate input featurs to the FC layer
+#and weight initialization for CNN model is based on the below github repo
+#Based on :https://github.com/Lab41/cyphercat/blob/master/Utils/models.py
+ 
+def size_conv(size, kernel, stride=1, padding=0):
+    out = int(((size - kernel + 2*padding)/stride) + 1)
+    return out
+    
+    
+def size_max_pool(size, kernel, stride=None, padding=0): 
+    if stride == None: 
+        stride = kernel
+    out = int(((size - kernel + 2*padding)/stride) + 1)
+    return out
+
+#Calculate in_features for FC layer in Shadow Net
+def calc_feat_linear_cifar(size):
+    feat = size_conv(size,3,1,1)
+    feat = size_max_pool(feat,2,2)
+    feat = size_conv(feat,3,1,1)
+    out = size_max_pool(feat,2,2)
+    return out
+    
+#Calculate in_features for FC layer in Shadow Net
+def calc_feat_linear_mnist(size):
+    feat = size_conv(size,5,1)
+    feat = size_max_pool(feat,2,2)
+    feat = size_conv(feat,5,1)
+    out = size_max_pool(feat,2,2)
+    return out
+
+#Parameter Initialization
+def init_params(m): 
+    if isinstance(m, nn.Conv2d):
+        nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+        if m.bias is not None:
+            nn.init.zeros_(m.bias)
+    elif isinstance(m, nn.BatchNorm2d):
+        nn.init.constant_(m.weight, 1)
+        nn.init.zeros_(m.bias)
+    elif isinstance(m, nn.Linear): 
+        nn.init.xavier_normal_(m.weight.data)
+        nn.init.zeros_(m.bias)
+
+#####################################################
+# Define Target, Shadow and Attack Model Architecture
+#####################################################
+
+
+#Target Model
+class TargetNet(nn.Module):
+    def __init__(self, input_dim, hidden_layers, size, out_classes):
+        super(TargetNet, self).__init__()
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(in_channels=input_dim, out_channels=hidden_layers[0], kernel_size=3, padding=1),
+            nn.BatchNorm2d(hidden_layers[0]),
+            # nn.Dropout(p=0.5),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        self.conv2 = nn.Sequential(
+            nn.Conv2d(in_channels=hidden_layers[0], out_channels=hidden_layers[1], kernel_size=3, padding=1),
+            nn.BatchNorm2d(hidden_layers[1]),
+            # nn.Dropout(p=0.5),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        
+        features = calc_feat_linear_cifar(size)
+        self.classifier = nn.Sequential(
+            nn.Flatten(),
+            nn.Linear((features**2 * hidden_layers[1]), hidden_layers[2]),
+            nn.ReLU(inplace=True),
+            nn.Linear(hidden_layers[2], out_classes)
+        )
+        
+    def forward(self, x):
+        out = self.conv1(x)
+        out = self.conv2(out)
+        out = self.classifier(out)
+        return out
+        
+    
+#Shadow Model mimicking target model architecture, for our implememtation is different than target
+class ShadowNet(nn.Module):
+    def __init__(self, input_dim, hidden_layers,size,out_classes):
+        super(ShadowNet, self).__init__()
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(in_channels=input_dim, out_channels=hidden_layers[0], kernel_size=3, padding=1),
+            nn.BatchNorm2d(hidden_layers[0]),
+            # nn.Dropout(p=0.5),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        self.conv2 = nn.Sequential(
+            nn.Conv2d(in_channels=hidden_layers[0], out_channels=hidden_layers[1], kernel_size=3, padding=1),
+            nn.BatchNorm2d(hidden_layers[1]),
+            # nn.Dropout(p=0.5),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        
+        features = calc_feat_linear_cifar(size)
+        self.classifier = nn.Sequential(
+            nn.Flatten(),
+            nn.Linear((features**2 * hidden_layers[1]), hidden_layers[2]),
+            nn.ReLU(inplace=True),
+            nn.Linear(hidden_layers[2], out_classes)
+        )
+        
+        
+    def forward(self, x):
+        out = self.conv1(x)
+        out = self.conv2(out)
+        out = self.classifier(out)
+        return out
+
+#Pretrained VGG11 model for Target
+class VggModel(nn.Module):
+    def __init__(self, num_classes,layer_config,pretrained=True):
+        super(VggModel, self).__init__()
+        #Load the pretrained VGG11_BN model
+        if pretrained:
+            pt_vgg = models.vgg11_bn(pretrained=pretrained)
+
+            #Deleting old FC layers from pretrained VGG model
+            print('### Deleting Avg pooling and FC Layers ####')
+            del pt_vgg.avgpool
+            del pt_vgg.classifier
+
+            self.model_features = nn.Sequential(*list(pt_vgg.features.children()))
+            
+            #Adding new FC layers with BN and RELU for CIFAR10 classification
+            self.model_classifier = nn.Sequential(
+                nn.Linear(layer_config[0], layer_config[1]),
+                nn.BatchNorm1d(layer_config[1]),
+                nn.ReLU(inplace=True),
+                nn.Linear(layer_config[1], num_classes),
+            )
+
+    def forward(self, x):
+        x = self.model_features(x)
+        x = x.squeeze()
+        out = self.model_classifier(x)
+        return out
+
+#Target/Shadow Model for MNIST
+class MNISTNet(nn.Module):
+    def __init__(self, input_dim, n_hidden,out_classes=10,size=28):
+        super(MNISTNet, self).__init__()
+        self.conv1 = nn.Sequential(
+            nn.Conv2d(in_channels=input_dim, out_channels=n_hidden, kernel_size=5),
+            nn.BatchNorm2d(n_hidden),
+            # nn.Dropout(p=0.5),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        self.conv2 = nn.Sequential(
+            nn.Conv2d(in_channels=n_hidden, out_channels=n_hidden*2, kernel_size=5),
+            nn.BatchNorm2d(n_hidden*2),
+            # nn.Dropout(p=0.5),
+            nn.ReLU(inplace=True),
+            nn.MaxPool2d(kernel_size=2, stride=2)
+        )
+        
+        features = calc_feat_linear_mnist(size)
+        self.classifier = nn.Sequential(
+            nn.Flatten(),
+            nn.Linear(features**2 * (n_hidden*2), n_hidden*2),
+            nn.ReLU(inplace=True),
+            nn.Linear(n_hidden*2, out_classes)
+        )
+        
+    def forward(self, x):
+        out = self.conv1(x)
+        out = self.conv2(out)
+        out = self.classifier(out)
+        return out
+
+#Attack MLP Model
+class AttackMLP(nn.Module):
+    def __init__(self, input_size, hidden_size=1024,out_classes=2):
+        super(AttackMLP, self).__init__()
+        self.classifier = nn.Sequential(
+            nn.Linear(input_size, hidden_size),
+            nn.ReLU(inplace=True),
+            nn.Linear(hidden_size, out_classes)
+        )    
+    def forward(self, x):
+        out = self.classifier(x)
+        return out          
+
+
+class ResNet(nn.Module):
+    def __init__(self, block, layers, num_classes=10): # 这里将类别数设置为10
+        
+        super(ResNet, self).__init__()
+
+
+        self.inplanes = 16 # 因为 CIFAR-10 图片较小，所以开始时需要更少的通道数
+        GlobalVariables.SELF_INPLANES = self.inplanes
+        # print('resnet init:'+ str(GlobalVariables.SELF_INPLANES))
+        # 输入层
+        self.conv1 = nn.Conv2d(3, 16, kernel_size=3, stride=1, padding=1,
+                               bias=False)
+        self.bn1 = nn.BatchNorm2d(16)
+        self.relu = nn.ReLU()
+
+        # 残差层（4 个阶段，每个阶段包含 6n+2 个卷积层）
+        self.layer1 = MakeLayer_ResNet(block, 16, layers[0])
+        self.layer2 = MakeLayer_ResNet(block, 32, layers[1], stride=2)
+        self.layer3 = MakeLayer_ResNet(block, 64, layers[2], stride=2)
+        self.layer4 = MakeLayer_ResNet(block, 128, layers[3], stride=2)
+
+        # 分类层
+        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
+        self.fc = nn.Linear(128 * block.expansion, num_classes)
+
+        # 参数初始化
+        for m in self.modules():
+            if isinstance(m, nn.Conv2d):
+                nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
+            elif isinstance(m, nn.BatchNorm2d):
+                nn.init.constant_(m.weight, 1)
+                nn.init.constant_(m.bias, 0)
+
+
+    def forward(self, x):
+        # 输入层
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu(x)
+        # 这里相比于imagenet的，少了一个maxpool，因为cifar10本身图片就小，如果再pool就太小了
+
+        # 残差层
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+
+        # 分类层
+        x = self.avgpool(x)  # 输出的尺寸为 B,C,1,1 
+        x = x.view(x.size(0), -1)
+        x = self.fc(x)
+
+        # out = F.softmax(x,dim = 1)         # 这里不softmax也行 影响不大
+
+        return x
+
+    def quantize(self, quant_type, num_bits=8, e_bits=3):
+        self.qconvbnrelu1 = QConvBNReLU(quant_type,self.conv1,self.bn1,qi=True,qo=True,num_bits=num_bits,e_bits=e_bits)
+        # 没有输入num_bits 需修改
+        self.layer1.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)      
+        self.layer2.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)
+        self.layer3.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)
+        self.layer4.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)
+        self.qavgpool1 = QAdaptiveAvgPool2d(quant_type,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qfc1 = QLinear(quant_type, self.fc,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        # self.qfc1 = QLinear(quant_type, self.fc,qi=True,qo=True,num_bits=num_bits,e_bits=e_bits)
+
+    def quantize_forward(self, x):
+        # for _, layer in self.quantize_layers.items():
+        #     x = layer(x)
+
+        # out = F.softmax(x, dim=1)
+        # return out
+        x = self.qconvbnrelu1(x)
+        x = self.layer1.quantize_forward(x)
+        x = self.layer2.quantize_forward(x)
+        x = self.layer3.quantize_forward(x)
+        x = self.layer4.quantize_forward(x)
+        x = self.qavgpool1(x)
+        x = x.view(x.size(0), -1)   
+        x = self.qfc1(x)
+        
+        # out = F.softmax(x,dim = 1)         # 这里不softmax也行 影响不大
+        return x
+      
+
+    def freeze(self):
+        self.qconvbnrelu1.freeze()  # 因为作为第一层是有qi的，所以freeze的时候无需再重新提供qi
+        qo = self.layer1.freeze(qinput = self.qconvbnrelu1.qo)
+        qo = self.layer2.freeze(qinput = qo)
+        qo = self.layer3.freeze(qinput = qo)
+        qo = self.layer4.freeze(qinput = qo)
+        self.qavgpool1.freeze(qi=qo)
+        self.qfc1.freeze(qi=self.qavgpool1.qo)
+        # self.qfc1.freeze()
+
+    def fakefreeze(self):
+        self.qconvbnrelu1.fakefreeze()
+        self.layer1.fakefreeze()
+        self.layer2.fakefreeze()
+        self.layer3.fakefreeze()
+        self.layer4.fakefreeze()
+        self.qfc1.fakefreeze()
+
+    def quantize_inference(self, x):
+        qx = self.qconvbnrelu1.qi.quantize_tensor(x)
+        qx = self.qconvbnrelu1.quantize_inference(qx)
+        qx = self.layer1.quantize_inference(qx)
+        qx = self.layer2.quantize_inference(qx)
+        qx = self.layer3.quantize_inference(qx)
+        qx = self.layer4.quantize_inference(qx)
+        qx = self.qavgpool1.quantize_inference(qx)
+        qx = qx.view(qx.size(0), -1)
+        qx = self.qfc1.quantize_inference(qx) 
+        qx = self.qfc1.qo.dequantize_tensor(qx)
+        
+
+        # out = F.softmax(qx,dim = 1)         # 这里不softmax也行 影响不大
+        return qx
+
+
+
+
+# BasicBlock 类
+class BasicBlock(nn.Module):
+    expansion = 1
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None):
+        super(BasicBlock, self).__init__()
+
+        # 第一个卷积层
+        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=3, stride=stride,
+                               padding=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+
+        # 第二个卷积层
+        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1,
+                               padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+
+        # shortcut
+        self.relu = nn.ReLU()
+        self.downsample = downsample
+        self.stride = stride
+
+
+    def forward(self, x):
+
+        identity = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+
+        if self.downsample is not None:
+            identity = self.downsample(identity)
+
+        out += identity
+        out = self.relu(out)
+
+        return out
+    
+    def quantize(self, quant_type ,num_bits=8, e_bits=3):
+        self.qconvbnrelu1 = QConvBNReLU(quant_type,self.conv1,self.bn1,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qconvbn1 = QConvBN(quant_type,self.conv2,self.bn2,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        
+        if self.downsample is not None:
+            self.qconvbn2 =  QConvBN(quant_type,self.downsample[0],self.downsample[1],qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+           
+        self.qelementadd = QElementwiseAdd(quant_type,qi0=False, qi1=False, qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qrelu1 = QReLU(quant_type,qi= False,num_bits=num_bits,e_bits=e_bits)   # 需要qi
+        
+
+    def quantize_forward(self, x):
+        identity = x
+        out = self.qconvbnrelu1(x)
+        out = self.qconvbn1(out)
+
+        if self.downsample is not None:
+            identity = self.qconvbn2(identity)
+        
+        # residual add
+        # out = identity + out    # 这里是需要写一个elementwiseadd的变换的，待后续修改
+        out = self.qelementadd(out,identity)
+        out = self.qrelu1(out)
+        return out
+
+    def freeze(self, qinput):
+        # 这里的qconvbnrelu1其实是可以用前一层的qo的，但感觉不太好传参，就没用 
+        # 还需仔细检查
+        self.qconvbnrelu1.freeze(qi= qinput)   # 需要接前一个module的最后一个qo
+        self.qconvbn1.freeze(qi = self.qconvbnrelu1.qo)
+
+        if self.downsample is not None:
+            self.qconvbn2.freeze(qi = qinput) # 一条支路
+            self.qelementadd.freeze(qi0 = self.qconvbn1.qo, qi1 = self.qconvbn2.qo)
+        else:
+            self.qelementadd.freeze(qi0 = self.qconvbn1.qo, qi1 = qinput)
+        # 这里或许需要补充个层来处理elementwise add
+        self.qrelu1.freeze(qi = self.qelementadd.qo) 
+        return self.qrelu1.qi  # relu后的qo可用relu统计的qi 
+
+    def fakefreeze(self):
+        # 这里的qconvbnrelu1其实是可以用前一层的qo的，但感觉不太好传参，就没用 
+        # 还需仔细检查
+        self.qconvbnrelu1.fakefreeze()   # 需要接前一个module的最后一个qo
+        self.qconvbn1.fakefreeze()
+
+        if self.downsample is not None:
+            self.qconvbn2.fakefreeze() # 一条支路
+
+
+    def quantize_inference(self, x):
+        # 感觉是不需要进行初始的quantize_tensor和dequantize_tensor，因为他不是最前/后一层，只要中间的每层都在量化后的领域内，就不需要这种处理。
+        identity = x
+        out = self.qconvbnrelu1.quantize_inference(x)
+        out = self.qconvbn1.quantize_inference(out)
+
+        if self.downsample is not None:
+            identity = self.qconvbn2.quantize_inference(identity)
+        
+        # out = identity + out    # 这里可能需要写一个elementwiseadd的变换的，待后续修改
+        out = self.qelementadd.quantize_inference(out,identity)
+        out = self.qrelu1.quantize_inference(out)
+        return out
+
+
+    
+
+
+# Bottleneck 类
+class Bottleneck(nn.Module):
+    expansion = 4
+
+    def __init__(self, inplanes, planes, stride=1, downsample=None):
+        super(Bottleneck, self).__init__()
+
+        # 1x1 卷积层
+        self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
+        self.bn1 = nn.BatchNorm2d(planes)
+
+        # 3x3 卷积层
+        self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
+                               padding=1, bias=False)
+        self.bn2 = nn.BatchNorm2d(planes)
+
+        # 1x1 卷积层
+        self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1,
+                               bias=False)
+        self.bn3 = nn.BatchNorm2d(planes * self.expansion)
+
+        # shortcut
+        self.relu = nn.ReLU()
+        self.downsample = downsample
+        self.stride = stride
+
+    def forward(self, x):
+
+        identity = x
+
+        out = self.conv1(x)
+        out = self.bn1(out)
+        out = self.relu(out)
+
+        out = self.conv2(out)
+        out = self.bn2(out)
+        out = self.relu(out)
+
+        out = self.conv3(out)
+        out = self.bn3(out)
+
+        if self.downsample is not None:
+            identity = self.downsample(x)
+
+        out += identity  # 相加是在这里处理的
+        out = self.relu(out)
+
+        return out
+    def quantize(self, quant_type ,num_bits=8, e_bits=3):
+        self.qconvbnrelu1 = QConvBNReLU(quant_type,self.conv1,self.bn1,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qconvbnrelu2 = QConvBNReLU(quant_type,self.conv2,self.bn2,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qconvbn1 = QConvBN(quant_type,self.conv3,self.bn3,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        
+        if self.downsample is not None:
+            self.qconvbn2 =  QConvBN(quant_type,self.downsample[0],self.downsample[1],qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+           
+        self.qelementadd = QElementwiseAdd(quant_type,qi0=False, qi1=False, qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qrelu1 = QReLU(quant_type,qi= False,num_bits=num_bits,e_bits=e_bits)   # 需要qi
+    
+    def quantize_forward(self, x):
+        identity = x
+        out = self.qconvbnrelu1(x)
+        out = self.qconvbnrelu2(out)
+        out = self.qconvbn1(out)
+
+        if self.downsample is not None:
+            identity = self.qconvbn2(identity)
+        
+        # residual add
+        # out = identity + out    # 这里是需要写一个elementwiseadd的变换的，待后续修改
+        out = self.qelementadd(out,identity)
+        out = self.qrelu1(out)
+        return out
+    
+    def freeze(self, qinput):
+        # 这里的qconvbnrelu1其实是可以用前一层的qo的，但感觉不太好传参，就没用 
+        # 还需仔细检查
+        self.qconvbnrelu1.freeze(qi= qinput)   # 需要接前一个module的最后一个qo
+        self.qconvbnrelu2.freeze(qi=self.qconvbnrelu1.qo)
+        self.qconvbn1.freeze(qi = self.qconvbnrelu2.qo)
+
+
+        if self.downsample is not None:
+            self.qconvbn2.freeze(qi = qinput) # 一条支路
+            self.qelementadd.freeze(qi0 = self.qconvbn1.qo, qi1 = self.qconvbn2.qo)
+        else:
+            self.qelementadd.freeze(qi0 = self.qconvbn1.qo, qi1 = qinput)
+        # 这里或许需要补充个层来处理elementwise add
+        self.qrelu1.freeze(qi = self.qelementadd.qo)  # 需要自己统计qi
+        return self.qrelu1.qi  # relu后的qo可用relu统计的qi 
+    
+    def fakefreeze(self):
+        # 这里的qconvbnrelu1其实是可以用前一层的qo的，但感觉不太好传参，就没用 
+        # 还需仔细检查
+        self.qconvbnrelu1.fakefreeze()   
+        self.qconvbnrelu2.fakefreeze()
+        self.qconvbn1.fakefreeze()
+
+
+        if self.downsample is not None:
+            self.qconvbn2.fakefreeze() # 一条支路
+
+
+    
+    def quantize_inference(self, x):
+        # 感觉是不需要进行初始的quantize_tensor和dequantize_tensor，因为他不是最前/后一层，只要中间的每层都在量化后的领域内，就不需要这种处理。
+        identity = x
+        out = self.qconvbnrelu1.quantize_inference(x)
+        out = self.qconvbnrelu2.quantize_inference(out)
+        out = self.qconvbn1.quantize_inference(out)
+
+        if self.downsample is not None:
+            identity = self.qconvbn2.quantize_inference(identity)
+        
+        # out = identity + out    # 这里可能需要写一个elementwiseadd的变换的，待后续修改
+        out = self.qelementadd.quantize_inference(out,identity)
+        out = self.qrelu1.quantize_inference(out)
+        return out
+
+
+
+class MakeLayer_ResNet(nn.Module):
+
+    def __init__(self, block, planes, blocks, stride=1):
+        super(MakeLayer_ResNet, self).__init__()
+        # print('makelayer init:'+ str(GlobalVariables.SELF_INPLANES))
+        self.downsample = None
+        if stride != 1 or GlobalVariables.SELF_INPLANES != planes * block.expansion:
+            self.downsample = nn.Sequential(
+            nn.Conv2d(GlobalVariables.SELF_INPLANES, planes * block.expansion,kernel_size=1, stride=stride, bias=False), 
+            nn.BatchNorm2d(planes * block.expansion)
+            )
+        self.blockdict = nn.ModuleDict()
+        self.blockdict['block1'] = block(inplanes=GlobalVariables.SELF_INPLANES, planes=planes, stride=stride, downsample=self.downsample)
+        GlobalVariables.SELF_INPLANES = planes * block.expansion
+        for i in range(1, blocks):  # block的个数   这里只能用字典了
+            self.blockdict['block' + str(i+1)] = block(inplanes=GlobalVariables.SELF_INPLANES, planes=planes)  # 此处进行实例化了
+        
+    def forward(self,x):
+        
+        for _, layer in self.blockdict.items():
+            x = layer(x)
+        
+        return x
+
+    def quantize(self, quant_type, num_bits=8, e_bits=3):
+        # 需检查
+        for _, layer in self.blockdict.items():
+            layer.quantize(quant_type=quant_type,num_bits=num_bits,e_bits=e_bits)   # 这里是因为每一块都是block，而block中有具体的quantize策略, n_exp和mode已经在__init__中赋值了
+        
+
+    def quantize_forward(self, x):
+        for _, layer in self.blockdict.items():
+            x = layer.quantize_forward(x)   # 各个block中有具体的quantize_forward
+
+        return x
+        
+       
+    def freeze(self, qinput):  # 需要在 Module Resnet的freeze里传出来
+        # 这里的qconvbnrelu1其实是可以用前一层的qo的，但感觉不太好传参，就没用 
+        # 还需仔细检查
+        cnt = 0
+        for _, layer in self.blockdict.items():
+            if cnt == 0:
+                qo = layer.freeze(qinput = qinput)
+                cnt = 1
+            else:
+                qo = layer.freeze(qinput = qo)  # 各个block中有具体的freeze
+
+        return qo   # 供后续的层用
+
+    def fakefreeze(self):
+        for _, layer in self.blockdict.items():
+            layer.fakefreeze()
+
+    def quantize_inference(self, x):
+        # 感觉是不需要进行初始的quantize_tensor和dequantize_tensor，因为他不是最前/后一层，只要中间的每层都在量化后的领域内，就不需要这种处理。
+        for _, layer in self.blockdict.items():
+            x = layer.quantize_inference(x)  # 每个block中有具体的quantize_inference
+        
+        return x
+
+
+
+
+# 使用 ResNet18 模型
+def resnet18(**kwargs):
+    model = ResNet(BasicBlock, [2, 2, 2, 2], **kwargs)
+    return model
+
+
+# 使用 ResNet50 模型
+def resnet50(**kwargs):
+    model = ResNet(Bottleneck, [3, 4, 6, 3], **kwargs)
+    return model
+
+
+# 使用 ResNet152 模型
+def resnet152(**kwargs):
+    model = ResNet(Bottleneck, [3, 8, 36, 3], **kwargs)
+    return model
+
+
+
+
+# ==========================================================
+# MobileNetV2
+
+
+class MobileNetV2(nn.Module):
+    def __init__(self, num_classes=10):
+        super(MobileNetV2, self).__init__()
+
+        self.conv1 = nn.Conv2d(3, 32, 3, stride=1, padding=1)
+        self.bn1 = nn.BatchNorm2d(32)
+        self.relu1 = nn.ReLU6(inplace=True)
+         # Bottleneck 层次, t指channel扩充系数
+        self.layer1 = MakeLayer_Mobile(32, 16, 1, t=1, stride=1)
+        self.layer2 = MakeLayer_Mobile(16, 24, 2, t=6, stride=2)
+        self.layer3 = MakeLayer_Mobile(24, 32, 3, t=6, stride=2)
+         # 根据CIFAR-10图像大小调整层数
+        self.layer4 = MakeLayer_Mobile(32, 96, 3, t=6, stride=1)
+        self.layer5 = MakeLayer_Mobile(96, 160, 3, t=6, stride=2)
+        self.layer6 = MakeLayer_Mobile(160, 320, 1, t=6, stride=1)
+
+        self.conv2 = nn.Conv2d(320, 1280, 1)
+        self.avg1 = nn.AdaptiveAvgPool2d(1)
+
+
+        self.fc = nn.Linear(1280, num_classes)
+
+
+    def forward(self, x):
+        # x = self.layers(x)
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu1(x)
+        x = self.layer1(x)
+        x = self.layer2(x)
+        x = self.layer3(x)
+        x = self.layer4(x)
+        x = self.layer5(x)
+        x = self.layer6(x)
+        x = self.conv2(x)
+        x = self.avg1(x)
+        x = x.view(x.size(0), -1)
+        x = self.fc(x)
+        return x
+    
+    def quantize(self, quant_type, num_bits=8, e_bits=3):
+        self.qconvbnrelu1 = QConvBNReLU6(quant_type,self.conv1,self.bn1,qi=True,qo=True,num_bits=num_bits,e_bits=e_bits)
+        # 没有输入num_bits 需修改
+        
+        self.layer1.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)      
+        self.layer2.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)
+        self.layer3.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)
+        self.layer4.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)
+        self.layer5.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)
+        self.layer6.quantize(quant_type=quant_type,num_bits=num_bits, e_bits=e_bits)
+
+        self.qconv1 = QConv2d(quant_type, self.conv2, qi=False, qo=True, num_bits=num_bits, e_bits=e_bits)
+        self.qavgpool1 = QAdaptiveAvgPool2d(quant_type,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qfc1 = QLinear(quant_type, self.fc,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+
+    def quantize_forward(self, x):
+        # for _, layer in self.quantize_layers.items():
+        #     x = layer(x)
+
+        # out = F.softmax(x, dim=1)
+        # return out
+        x = self.qconvbnrelu1(x)
+        x = self.layer1.quantize_forward(x)
+        x = self.layer2.quantize_forward(x)
+        x = self.layer3.quantize_forward(x)
+        x = self.layer4.quantize_forward(x)
+        x = self.layer5.quantize_forward(x)
+        x = self.layer6.quantize_forward(x)
+        x = self.qconv1(x)
+        x = self.qavgpool1(x)
+        x = x.view(x.size(0), -1)   
+        x = self.qfc1(x)
+        
+        out = F.softmax(x,dim = 1)         # 这里不softmax也行 影响不大
+        return out
+
+    def freeze(self):
+        self.qconvbnrelu1.freeze()  # 因为作为第一层是有qi的，所以freeze的时候无需再重新提供qi
+        qo = self.layer1.freeze(qinput = self.qconvbnrelu1.qo)
+        qo = self.layer2.freeze(qinput = qo)
+        qo = self.layer3.freeze(qinput = qo)
+        qo = self.layer4.freeze(qinput = qo)
+        qo = self.layer5.freeze(qinput = qo)
+        qo = self.layer6.freeze(qinput = qo)
+        self.qconv1.freeze(qi = qo)
+        self.qavgpool1.freeze(qi=self.qconv1.qo)
+        self.qfc1.freeze(qi=self.qavgpool1.qo)
+        # self.qfc1.freeze()
+
+    def fakefreeze(self):
+        self.qconvbnrelu1.fakefreeze()
+        self.layer1.fakefreeze()
+        self.layer2.fakefreeze()
+        self.layer3.fakefreeze()
+        self.layer4.fakefreeze()
+        self.layer5.fakefreeze()
+        self.layer6.fakefreeze()
+        self.qconv1.fakefreeze()
+        self.qfc1.fakefreeze()
+
+    
+    def quantize_inference(self, x):
+
+        qx = self.qconvbnrelu1.qi.quantize_tensor(x)
+        qx = self.qconvbnrelu1.quantize_inference(qx)
+
+        qx = self.layer1.quantize_inference(qx)
+        qx = self.layer2.quantize_inference(qx)
+        qx = self.layer3.quantize_inference(qx)
+        qx = self.layer4.quantize_inference(qx)
+        qx = self.layer5.quantize_inference(qx)
+        qx = self.layer6.quantize_inference(qx)
+
+        qx = self.qconv1.quantize_inference(qx)
+        qx = self.qavgpool1.quantize_inference(qx)
+        qx = qx.view(qx.size(0), -1)
+        qx = self.qfc1.quantize_inference(qx) 
+        qx = self.qfc1.qo.dequantize_tensor(qx)
+        
+        out = F.softmax(qx,dim = 1)         # 这里不softmax也行 影响不大
+        return out
+
+
+
+ 
+
+class InvertedResidual(nn.Module):
+    def __init__(self, in_channels, out_channels, stride, expand_ratio):
+        super(InvertedResidual, self).__init__()
+        hidden_dims = int(in_channels * expand_ratio)
+        self.identity_flag = stride == 1 and in_channels == out_channels
+
+        # self.bottleneck = nn.Sequential(
+        #     # Pointwise Convolution
+        #     nn.Conv2d(in_channels, hidden_dims, 1),
+        #     nn.BatchNorm2d(hidden_dims),
+        #     nn.ReLU6(inplace=True),
+        #     # Depthwise Convolution
+        #     nn.Conv2d(hidden_dims, hidden_dims, 3, stride=stride, padding=1, groups=hidden_dims),
+        #     nn.BatchNorm2d(hidden_dims),
+        #     nn.ReLU6(inplace=True),
+        #     # Pointwise & Linear Convolution
+        #     nn.Conv2d(hidden_dims, out_channels, 1),
+        #     nn.BatchNorm2d(out_channels),
+        # )
+        self.conv1 = nn.Conv2d(in_channels, hidden_dims, 1)
+        self.bn1 = nn.BatchNorm2d(hidden_dims)
+        self.relu1 = nn.ReLU6(inplace=True)
+        # Depthwise Convolution
+        self.conv2 = nn.Conv2d(hidden_dims, hidden_dims, 3, stride=stride, padding=1, groups=hidden_dims)
+        self.bn2 = nn.BatchNorm2d(hidden_dims)
+        self.relu2 = nn.ReLU6(inplace=True)
+        # Pointwise & Linear Convolution
+        self.conv3 = nn.Conv2d(hidden_dims, out_channels, 1)
+        self.bn3 = nn.BatchNorm2d(out_channels)
+
+    def forward(self, x):
+        # if self.identity_flag:
+        #     return x + self.bottleneck(x)
+        # else:
+        #     return self.bottleneck(x)
+        identity = x
+
+        x = self.conv1(x)
+        x = self.bn1(x)
+        x = self.relu1(x)
+
+        x = self.conv2(x)
+        x = self.bn2(x)
+        x = self.relu2(x)
+
+        x = self.conv3(x)
+        x = self.bn3(x)
+
+        if self.identity_flag:
+            return identity + x
+        
+        else:
+            return x
+
+    
+    def quantize(self, quant_type ,num_bits=8, e_bits=3):
+        self.qconvbnrelu1 = QConvBNReLU6(quant_type,self.conv1,self.bn1,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qconvbnrelu2 = QConvBNReLU6(quant_type,self.conv2,self.bn2,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        self.qconvbn1 = QConvBN(quant_type,self.conv3,self.bn3,qi=False,qo=True,num_bits=num_bits,e_bits=e_bits)
+        
+
+        self.qelementadd = QElementwiseAdd(quant_type,qi0=False, qi1=False, qo=True,num_bits=num_bits,e_bits=e_bits)
+
+   
+    def quantize_forward(self, x):
+        identity = x
+        out = self.qconvbnrelu1(x)
+        out = self.qconvbnrelu2(out)
+        out = self.qconvbn1(out)
+
+        if self.identity_flag:
+            out = self.qelementadd(out, identity)
+
+        return out
+    
+    def freeze(self, qinput):
+        # 这里的qconvbnrelu1其实是可以用前一层的qo的，但感觉不太好传参，就没用 
+        # 还需仔细检查
+        self.qconvbnrelu1.freeze(qi= qinput)   # 需要接前一个module的最后一个qo
+        self.qconvbnrelu2.freeze(qi=self.qconvbnrelu1.qo)
+        self.qconvbn1.freeze(qi = self.qconvbnrelu2.qo)
+
+        if self.identity_flag:
+            self.qelementadd.freeze(qi0 = self.qconvbn1.qo, qi1 = qinput)
+            return self.qelementadd.qo
+        else:
+            return self.qconvbn1.qo
+    
+    def fakefreeze(self):
+        self.qconvbnrelu1.fakefreeze()
+        self.qconvbnrelu2.fakefreeze()
+        self.qconvbn1.fakefreeze()
+
+
+    
+    def quantize_inference(self, x):
+        # 感觉是不需要进行初始的quantize_tensor和dequantize_tensor，因为他不是最前/后一层，只要中间的每层都在量化后的领域内，就不需要这种处理。
+        identity = x
+        out = self.qconvbnrelu1.quantize_inference(x)
+        out = self.qconvbnrelu2.quantize_inference(out)
+        out = self.qconvbn1.quantize_inference(out)
+
+        if self.identity_flag:
+            out = self.qelementadd.quantize_inference(out, identity)        
+        
+        return out
+
+
+
+class MakeLayer_Mobile(nn.Module):
+
+    def __init__(self, in_channels, out_channels, n_repeat, t, stride):
+        super(MakeLayer_Mobile, self).__init__()
+        # print('makelayer init:'+ str(GlobalVariables.SELF_INPLANES))
+        self.layers = nn.ModuleList()
+        for i in range(n_repeat):
+            if i == 0:
+                self.layers.append(InvertedResidual(in_channels, out_channels, stride, t))
+            else:
+                self.layers.append(InvertedResidual(in_channels, out_channels, 1, t))
+            in_channels = out_channels
+        
+        # for l in self.layers:
+        #     print(l)
+
+
+    def forward(self,x):
+        
+        for layer in self.layers:
+            x = layer(x)
+        
+        return x
+
+    def quantize(self, quant_type, num_bits=8, e_bits=3):
+        # 需检查
+        # print('CHECK======')
+        for layer in self.layers:
+            layer.quantize(quant_type=quant_type,num_bits=num_bits,e_bits=e_bits)   # 这里是因为每一块都是block，而block中有具体的quantize策略, n_exp和mode已经在__init__中赋值了
+            
+            # print(layer)
+
+        # print('CHECK======')
+        
+
+    def quantize_forward(self, x):
+        for layer in self.layers:
+            x = layer.quantize_forward(x)   # 各个block中有具体的quantize_forward
+
+        return x
+        
+       
+    def freeze(self, qinput):  # 需要在 Module Resnet的freeze里传出来
+        # 这里的qconvbnrelu1其实是可以用前一层的qo的，但感觉不太好传参，就没用 
+        # 还需仔细检查
+        cnt = 0
+        for layer in self.layers:
+            if cnt == 0:
+                qo = layer.freeze(qinput = qinput)
+                cnt = 1
+            else:
+                qo = layer.freeze(qinput = qo)  # 各个block中有具体的freeze
+
+        return qo   # 供后续的层用
+
+    def fakefreeze(self):
+        for layer in self.layers:
+            layer.fakefreeze()
+
+    def quantize_inference(self, x):
+        # 感觉是不需要进行初始的quantize_tensor和dequantize_tensor，因为他不是最前/后一层，只要中间的每层都在量化后的领域内，就不需要这种处理。
+        for layer in self.layers:
+            x = layer.quantize_inference(x)  # 每个block中有具体的quantize_inference
+        
+        return x

mzh/new_mzh/Naive_MIA/module.py

+import math
+import numpy as np
+import gol
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+from function import FakeQuantize
+
+# 获取最近的量化值
+# def get_nearest_val(quant_type,x,is_bias=False):
+#     if quant_type=='INT':
+#         return x.round_()
+    
+#     plist = gol.get_value(is_bias)
+#     # print('get')
+#     # print(plist)
+#     # x = x / 64
+#     shape = x.shape
+#     xhard = x.view(-1)
+#     plist = plist.type_as(x)
+#     # 取最近幂次作为索引
+#     idx = (xhard.unsqueeze(0) - plist.unsqueeze(1)).abs().min(dim=0)[1]
+#     xhard = plist[idx].view(shape)
+#     xout = (xhard - x).detach() + x
+#     # xout = xout * 64
+#     return xout
+
+
+def get_nearest_val(quant_type, x, is_bias=False, block_size=1000000):
+    if quant_type == 'INT':
+        return x.round_()
+
+    plist = gol.get_value(is_bias)
+    shape = x.shape
+    xhard = x.view(-1)
+    xout = torch.zeros_like(xhard)
+
+    plist = plist.type_as(x)
+    n_blocks = (x.numel() + block_size - 1) // block_size
+
+    for i in range(n_blocks):
+        start_idx = i * block_size
+        end_idx = min(start_idx + block_size, xhard.numel())
+        block_size_i = end_idx - start_idx
+        # print(x.numel())
+        # print(block_size_i)
+        # print(start_idx)
+        # print(end_idx)
+        xblock = xhard[start_idx:end_idx]
+        # xblock = xblock.view(shape[start_idx:end_idx])
+        plist_block = plist.unsqueeze(1) #.expand(-1, block_size_i)
+        idx = (xblock.unsqueeze(0) - plist_block).abs().min(dim=0)[1]
+        # print(xblock.shape)
+        xhard_block = plist[idx].view(xblock.shape)
+        xout[start_idx:end_idx] = (xhard_block - xblock).detach() + xblock
+
+    xout = xout.view(shape)
+    return xout
+
+
+# 采用对称有符号量化时，获取量化范围最大值
+def get_qmax(quant_type,num_bits=None, e_bits=None):
+    if quant_type == 'INT':
+        qmax = 2. ** (num_bits - 1) - 1
+    elif quant_type == 'POT':
+        qmax = 1
+    else: #FLOAT
+        m_bits = num_bits - 1 - e_bits
+        dist_m = 2 ** (-m_bits)
+        e = 2 ** (e_bits - 1)
+        expo = 2 ** e
+        m = 2 ** m_bits -1
+        frac = 1. + m * dist_m
+        qmax = frac * expo
+    return qmax
+
+# 都采用有符号量化，zeropoint都置为0
+def calcScaleZeroPoint(min_val, max_val, qmax):
+    scale = torch.max(max_val.abs(),min_val.abs()) / qmax
+    zero_point = torch.tensor(0.)
+    return scale, zero_point
+
+# 将输入进行量化，输入输出都为tensor
+def quantize_tensor(quant_type, x, scale, zero_point, qmax, is_bias=False):
+    # 量化后范围，直接根据位宽确定
+    qmin = -qmax
+    q_x = zero_point + x / scale
+
+    q_x.clamp_(qmin, qmax)
+    q_x = get_nearest_val(quant_type, q_x, is_bias)
+    return q_x
+
+# bias使用不同精度，需要根据量化类型指定num_bits/e_bits
+def bias_qmax(quant_type):
+    if quant_type == 'INT':
+        return get_qmax(quant_type, 64)
+    elif quant_type == 'POT':
+        return get_qmax(quant_type)
+    else:
+        return get_qmax(quant_type, 16, 7)
+        
+
+# 转化为FP32，不需再做限制
+def dequantize_tensor(q_x, scale, zero_point):
+    return scale * (q_x - zero_point)
+
+
+class QParam(nn.Module):
+
+    def __init__(self,quant_type, num_bits=8, e_bits=3):
+        super(QParam, self).__init__()
+        self.quant_type = quant_type
+        self.num_bits = num_bits
+        self.e_bits = e_bits
+        self.qmax = get_qmax(quant_type, num_bits, e_bits)
+
+        scale = torch.tensor([], requires_grad=False)
+        zero_point = torch.tensor([], requires_grad=False)
+        min = torch.tensor([], requires_grad=False)
+        max = torch.tensor([], requires_grad=False)
+        # 通过注册为register，使得buffer可以被记录到state_dict
+        self.register_buffer('scale', scale)
+        self.register_buffer('zero_point', zero_point)
+        self.register_buffer('min', min)
+        self.register_buffer('max', max)
+
+    # 更新统计范围及量化参数
+    def update(self, tensor):
+        if self.max.nelement() == 0 or self.max.data < tensor.max().data:
+            self.max.data = tensor.max().data
+        self.max.clamp_(min=0)
+
+        if self.min.nelement() == 0 or self.min.data > tensor.min().data:
+            self.min.data = tensor.min().data
+        self.min.clamp_(max=0)
+
+        self.scale, self.zero_point = calcScaleZeroPoint(self.min, self.max, self.qmax)
+
+    def quantize_tensor(self, tensor):
+        return quantize_tensor(self.quant_type, tensor, self.scale, self.zero_point, self.qmax)
+
+    def dequantize_tensor(self, q_x):
+        return dequantize_tensor(q_x, self.scale, self.zero_point)
+
+    # 该方法保证了可以从state_dict里恢复
+    def _load_from_state_dict(self, state_dict, prefix, local_metadata, strict, missing_keys, unexpected_keys,
+                              error_msgs):
+        key_names = ['scale', 'zero_point', 'min', 'max']
+        for key in key_names:
+            value = getattr(self, key)
+            value.data = state_dict[prefix + key].data
+            state_dict.pop(prefix + key)
+
+    # 该方法返回值将是打印该对象的结果
+    def __str__(self):
+        info = 'scale: %.10f ' % self.scale
+        info += 'zp: %.6f ' % self.zero_point
+        info += 'min: %.6f ' % self.min
+        info += 'max: %.6f' % self.max
+        return info
+
+
+# 作为具体量化层的父类，qi和qo分别为量化输入/输出
+class QModule(nn.Module):
+
+    def __init__(self,quant_type, qi=True, qo=True, num_bits=8, e_bits=3):
+        super(QModule, self).__init__()
+        if qi:
+            self.qi = QParam(quant_type,num_bits, e_bits)
+        if qo:
+            self.qo = QParam(quant_type,num_bits, e_bits)
+        self.quant_type = quant_type
+        self.num_bits = num_bits
+        self.e_bits = e_bits
+        self.bias_qmax = bias_qmax(quant_type)
+
+    def freeze(self):
+        pass  # 空语句
+    
+    def fakefreeze(self):
+        pass
+
+    def quantize_inference(self, x):
+        raise NotImplementedError('quantize_inference should be implemented.')
+
+
+"""
+QModule  量化卷积
+
+:quant_type: 量化类型
+:conv_module: 卷积模块
+:qi: 是否量化输入特征图
+:qo: 是否量化输出特征图
+:num_bits: 8位bit数
+"""
+
+
+class QConv2d(QModule):
+
+    def __init__(self, quant_type, conv_module, qi=True, qo=True, num_bits=8, e_bits=3):
+        super(QConv2d, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+        self.conv_module = conv_module
+        self.qw = QParam(quant_type, num_bits,e_bits)
+        self.register_buffer('M', torch.tensor([], requires_grad=False))  # 将M注册为buffer
+
+    # freeze方法可以固定真量化的权重参数，并将该值更新到原全精度层上，便于散度计算
+    def freeze(self, qi=None, qo=None):
+
+        if hasattr(self, 'qi') and qi is not None:
+            raise ValueError('qi has been provided in init function.')
+        if not hasattr(self, 'qi') and qi is None:
+            raise ValueError('qi is not existed, should be provided.')
+
+        if hasattr(self, 'qo') and qo is not None:
+            raise ValueError('qo has been provided in init function.')
+        if not hasattr(self, 'qo') and qo is None:
+            raise ValueError('qo is not existed, should be provided.')
+
+        # 这里因为在池化或者激活的输入，不需要对最大值和最小是进行额外的统计，会共享相同的输出
+        if qi is not None:
+            self.qi = qi
+        if qo is not None:
+            self.qo = qo
+
+        # 根据https://zhuanlan.zhihu.com/p/156835141, 这是式3 的系数
+        self.M.data = (self.qw.scale * self.qi.scale / self.qo.scale).data
+
+        self.conv_module.weight.data = self.qw.quantize_tensor(self.conv_module.weight.data)
+        self.conv_module.weight.data = self.conv_module.weight.data - self.qw.zero_point
+
+        self.conv_module.bias.data = quantize_tensor(self.quant_type,
+                                                    self.conv_module.bias.data, scale=self.qi.scale * self.qw.scale,
+                                                    zero_point=0.,qmax=self.bias_qmax, is_bias=True)
+        
+    def fakefreeze(self):
+        self.conv_module.weight.data = self.qw.dequantize_tensor(self.conv_module.weight.data)
+        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale * self.qw.scale, zero_point=0.)
+        
+    
+    def forward(self, x):  # 前向传播,输入张量,x为浮点型数据
+        if hasattr(self, 'qi'):
+            self.qi.update(x)
+            x = FakeQuantize.apply(x, self.qi)  # 对输入张量X完成量化
+
+        # foward前更新qw，保证量化weight时候scale正确
+        self.qw.update(self.conv_module.weight.data)
+        # 注意:此处主要为了统计各层x和weight范围，未对bias进行量化操作
+        # tmp_wgt = FakeQuantize.apply(self.conv_module.weight, self.qw)
+        # x = F.conv2d(x, tmp_wgt, self.conv_module.bias,
+        #              stride=self.conv_module.stride,
+        #              padding=self.conv_module.padding, dilation=self.conv_module.dilation,
+        #              groups=self.conv_module.groups)
+        
+        x = F.conv2d(x, FakeQuantize.apply(self.conv_module.weight, self.qw), self.conv_module.bias,
+                     stride=self.conv_module.stride,
+                     padding=self.conv_module.padding, dilation=self.conv_module.dilation,
+                     groups=self.conv_module.groups)
+
+        if hasattr(self, 'qo'):
+            self.qo.update(x)
+            x = FakeQuantize.apply(x, self.qo)
+
+        return x
+
+    # 利用公式 q_a = M(\sigma(q_w-Z_w)(q_x-Z_x) + q_b)
+    def quantize_inference(self, x):  # 此处input为已经量化的qx
+        x = x - self.qi.zero_point
+        x = self.conv_module(x)
+        x = self.M * x
+
+        # if self.quant_type is 'INT':
+        x = get_nearest_val(self.quant_type,x)
+
+        x = x + self.qo.zero_point
+        return x
+
+
+class QLinear(QModule):
+
+    def __init__(self, quant_type, fc_module, qi=True, qo=True, num_bits=8, e_bits=3):
+        super(QLinear, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+        self.fc_module = fc_module
+        self.qw = QParam(quant_type, num_bits, e_bits)
+        self.register_buffer('M', torch.tensor([], requires_grad=False))  # 将M注册为buffer
+
+    def freeze(self, qi=None, qo=None):
+
+        if hasattr(self, 'qi') and qi is not None:
+            raise ValueError('qi has been provided in init function.')
+        if not hasattr(self, 'qi') and qi is None:
+            raise ValueError('qi is not existed, should be provided.')
+
+        if hasattr(self, 'qo') and qo is not None:
+            raise ValueError('qo has been provided in init function.')
+        if not hasattr(self, 'qo') and qo is None:
+            raise ValueError('qo is not existed, should be provided.')
+
+        if qi is not None:
+            self.qi = qi
+        if qo is not None:
+            self.qo = qo
+        
+        self.M.data = (self.qw.scale * self.qi.scale / self.qo.scale).data
+
+        self.fc_module.weight.data = self.qw.quantize_tensor(self.fc_module.weight.data)
+        self.fc_module.weight.data = self.fc_module.weight.data - self.qw.zero_point
+        self.fc_module.bias.data = quantize_tensor(self.quant_type,
+                                                    self.fc_module.bias.data, scale=self.qi.scale * self.qw.scale,
+                                                    zero_point=0., qmax=self.bias_qmax, is_bias=True)
+
+    def fakefreeze(self):
+        self.fc_module.weight.data = self.qw.dequantize_tensor(self.fc_module.weight.data)
+        self.fc_module.bias.data = dequantize_tensor(self.fc_module.bias.data, scale=self.qi.scale * self.qw.scale, zero_point=0.)
+        
+    
+    
+    def forward(self, x):
+        if hasattr(self, 'qi'):
+            self.qi.update(x)
+            x = FakeQuantize.apply(x, self.qi)
+
+        self.qw.update(self.fc_module.weight.data)
+        # tmp_wgt = FakeQuantize.apply(self.fc_module.weight, self.qw)
+        # x = F.linear(x, tmp_wgt, self.fc_module.bias)
+        x = F.linear(x, FakeQuantize.apply(self.fc_module.weight, self.qw), self.fc_module.bias)
+
+        if hasattr(self, 'qo'):
+            self.qo.update(x)
+            x = FakeQuantize.apply(x, self.qo)
+
+        return x
+
+    def quantize_inference(self, x):
+        x = x - self.qi.zero_point
+        x = self.fc_module(x)
+        x = self.M * x
+
+        # if self.quant_type is 'INT':
+        x = get_nearest_val(self.quant_type,x)
+
+       
+        x = x + self.qo.zero_point
+
+        return x
+
+
+class QReLU(QModule):
+
+    def __init__(self,quant_type, qi=False, qo=True, num_bits=8, e_bits=3):
+        super(QReLU, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+
+    def freeze(self, qi=None):
+
+        if hasattr(self, 'qi') and qi is not None:
+            raise ValueError('qi has been provided in init function.')
+        if not hasattr(self, 'qi') and qi is None:
+            raise ValueError('qi is not existed, should be provided.')
+
+        if qi is not None:
+            self.qi = qi
+
+    def forward(self, x):
+        if hasattr(self, 'qi'):
+            self.qi.update(x)
+            x = FakeQuantize.apply(x, self.qi)
+
+        x = F.relu(x)
+
+        return x
+
+    def quantize_inference(self, x):
+        x = x.clone()
+        # x[x < self.qi.zero_point] = self.qi.zero_point
+        device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+        a = self.qi.zero_point.float().to(device)
+        x[x < a] = a
+        return x
+
+
+class QMaxPooling2d(QModule):
+
+    def __init__(self, quant_type, kernel_size=3, stride=1, padding=0, qi=False, qo=True, num_bits=8,e_bits=3):
+        super(QMaxPooling2d, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+        self.kernel_size = kernel_size
+        self.stride = stride
+        self.padding = padding
+        
+    def freeze(self, qi=None):
+        if hasattr(self, 'qi') and qi is not None:
+            raise ValueError('qi has been provided in init function.')
+        if not hasattr(self, 'qi') and qi is None:
+            raise ValueError('qi is not existed, should be provided.')
+        if qi is not None:
+            self.qi = qi
+
+    def forward(self, x):
+        if hasattr(self, 'qi'):
+            self.qi.update(x)
+            x = FakeQuantize.apply(x, self.qi)
+
+        x = F.max_pool2d(x, self.kernel_size, self.stride, self.padding)
+
+        return x
+
+    def quantize_inference(self, x):
+        return F.max_pool2d(x, self.kernel_size, self.stride, self.padding)
+
+class QConvBNReLU(QModule):
+
+    def __init__(self, quant_type, conv_module, bn_module, qi=True, qo=True, num_bits=8, e_bits=3):
+        super(QConvBNReLU, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+        self.conv_module = conv_module
+        self.bn_module = bn_module
+        self.qw = QParam(quant_type, num_bits,e_bits)
+        self.register_buffer('M', torch.tensor([], requires_grad=False))  # 将M注册为buffer
+
+    def fold_bn(self, mean, std):
+        if self.bn_module.affine:
+            gamma_ = self.bn_module.weight / std
+            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
+            else:
+                bias = self.bn_module.bias - gamma_ * mean
+        else:
+            gamma_ = 1 / std
+            weight = self.conv_module.weight * gamma_
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean
+            else:
+                bias = -gamma_ * mean
+            
+        return weight, bias
+
+    def freeze(self, qi=None, qo=None):
+        if hasattr(self, 'qi') and qi is not None:
+            raise ValueError('qi has been provided in init function.')
+        if not hasattr(self, 'qi') and qi is None:
+            raise ValueError('qi is not existed, should be provided.')
+
+        if hasattr(self, 'qo') and qo is not None:
+            raise ValueError('qo has been provided in init function.')
+        if not hasattr(self, 'qo') and qo is None:
+            raise ValueError('qo is not existed, should be provided.')
+
+        if qi is not None:
+            self.qi = qi
+        if qo is not None:
+            self.qo = qo
+
+        self.M.data = (self.qw.scale * self.qi.scale / self.qo.scale).data
+
+        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
+
+        self.conv_module.weight.data = self.qw.quantize_tensor(weight.data)
+        self.conv_module.weight.data = self.conv_module.weight.data - self.qw.zero_point
+
+        if self.conv_module.bias is None:
+            self.conv_module.bias = nn.Parameter(quantize_tensor(self.quant_type,
+                                                     bias, scale=self.qi.scale * self.qw.scale,
+                                                     zero_point=0., qmax=self.bias_qmax,is_bias=True))
+        else:
+            self.conv_module.bias.data = quantize_tensor(self.quant_type,
+                                                     bias, scale=self.qi.scale * self.qw.scale,
+                                                     zero_point=0., qmax=self.bias_qmax,is_bias=True)
+        
+
+    def fakefreeze(self):
+        self.conv_module.weight.data = self.qw.dequantize_tensor(self.conv_module.weight.data)
+        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale * self.qw.scale, zero_point=0.)
+        
+    
+    def forward(self, x):
+
+        if hasattr(self, 'qi'):
+            self.qi.update(x)
+            x = FakeQuantize.apply(x, self.qi)
+
+        if self.training:
+            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
+                            stride=self.conv_module.stride,
+                            padding=self.conv_module.padding,
+                            dilation=self.conv_module.dilation,
+                            groups=self.conv_module.groups)
+            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
+            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
+            # mean = y.mean(1)
+            # var = y.var(1)
+            mean = y.mean(1).detach()
+            var = y.var(1).detach()
+            self.bn_module.running_mean = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
+                self.bn_module.momentum * mean
+            self.bn_module.running_var = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
+                self.bn_module.momentum * var
+        else:
+            mean = Variable(self.bn_module.running_mean)
+            var = Variable(self.bn_module.running_var)
+
+        std = torch.sqrt(var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(mean, std)
+
+        self.qw.update(weight.data)
+
+        x = F.conv2d(x, FakeQuantize.apply(weight, self.qw), bias, 
+                stride=self.conv_module.stride,
+                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
+                groups=self.conv_module.groups)
+
+        x = F.relu(x)
+
+        if hasattr(self, 'qo'):
+            self.qo.update(x)
+            x = FakeQuantize.apply(x, self.qo)
+
+        return x
+    
+    def quantize_inference(self, x):
+        x = x - self.qi.zero_point
+        x = self.conv_module(x)
+        x = self.M * x
+
+        # if self.quant_type is 'INT':
+        x = get_nearest_val(self.quant_type,x)
+
+        
+        x = x + self.qo.zero_point        
+        x.clamp_(min=0)
+        return x
+        
+
+
+class QConvBN(QModule):
+
+    def __init__(self, quant_type, conv_module, bn_module, qi=True, qo=True, num_bits=8, e_bits=3):
+        super(QConvBN, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+        self.conv_module = conv_module
+        self.bn_module = bn_module
+        self.qw = QParam(quant_type, num_bits,e_bits)
+        self.register_buffer('M', torch.tensor([], requires_grad=False))  # 将M注册为buffer
+
+    def fold_bn(self, mean, std):
+        if self.bn_module.affine:
+            gamma_ = self.bn_module.weight / std
+            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
+            else:
+                bias = self.bn_module.bias - gamma_ * mean
+        else:
+            gamma_ = 1 / std
+            weight = self.conv_module.weight * gamma_
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean
+            else:
+                bias = -gamma_ * mean
+            
+        return weight, bias
+
+    def freeze(self, qi=None, qo=None):
+        if hasattr(self, 'qi') and qi is not None:
+            raise ValueError('qi has been provided in init function.')
+        if not hasattr(self, 'qi') and qi is None:
+            raise ValueError('qi is not existed, should be provided.')
+
+        if hasattr(self, 'qo') and qo is not None:
+            raise ValueError('qo has been provided in init function.')
+        if not hasattr(self, 'qo') and qo is None:
+            raise ValueError('qo is not existed, should be provided.')
+
+        if qi is not None:
+            self.qi = qi
+        if qo is not None:
+            self.qo = qo
+
+        self.M.data = (self.qw.scale * self.qi.scale / self.qo.scale).data
+
+        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
+
+        self.conv_module.weight.data = self.qw.quantize_tensor(weight.data)
+        self.conv_module.weight.data = self.conv_module.weight.data - self.qw.zero_point
+
+        if self.conv_module.bias is None:
+            self.conv_module.bias = nn.Parameter(quantize_tensor(self.quant_type,
+                                                     bias, scale=self.qi.scale * self.qw.scale,
+                                                     zero_point=0., qmax=self.bias_qmax,is_bias=True))
+        else:
+            self.conv_module.bias.data = quantize_tensor(self.quant_type,
+                                                     bias, scale=self.qi.scale * self.qw.scale,
+                                                     zero_point=0., qmax=self.bias_qmax,is_bias=True)
+
+    def fakefreeze(self):
+        self.conv_module.weight.data = self.qw.dequantize_tensor(self.conv_module.weight.data)
+        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale * self.qw.scale, zero_point=0.)
+    
+    def forward(self, x):
+
+        if hasattr(self, 'qi'):
+            self.qi.update(x)
+            x = FakeQuantize.apply(x, self.qi)
+
+        if self.training:
+            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
+                            stride=self.conv_module.stride,
+                            padding=self.conv_module.padding,
+                            dilation=self.conv_module.dilation,
+                            groups=self.conv_module.groups)
+            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
+            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
+            # mean = y.mean(1)
+            # var = y.var(1)
+            mean = y.mean(1).detach()
+            var = y.var(1).detach()
+            self.bn_module.running_mean = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
+                self.bn_module.momentum * mean
+            self.bn_module.running_var = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
+                self.bn_module.momentum * var
+        else:
+            mean = Variable(self.bn_module.running_mean)
+            var = Variable(self.bn_module.running_var)
+
+        std = torch.sqrt(var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(mean, std)
+
+        self.qw.update(weight.data)
+
+        x = F.conv2d(x, FakeQuantize.apply(weight, self.qw), bias, 
+                stride=self.conv_module.stride,
+                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
+                groups=self.conv_module.groups)
+
+        # x = F.relu(x)
+
+        if hasattr(self, 'qo'):
+            self.qo.update(x)
+            x = FakeQuantize.apply(x, self.qo)
+
+        return x
+    
+    def quantize_inference(self, x):
+        x = x - self.qi.zero_point
+        x = self.conv_module(x)
+        x = self.M * x
+        
+        # if self.quant_type is 'INT':
+        x = get_nearest_val(self.quant_type,x)
+        
+        
+        x = x + self.qo.zero_point        
+        # x.clamp_(min=0)
+        return x
+
+# 待修改  需要有qo吧
+class QAdaptiveAvgPool2d(QModule):
+    def __init__(self, quant_type, qi=False, qo=True, num_bits=8, e_bits=3):
+        super(QAdaptiveAvgPool2d, self).__init__(quant_type,qi,qo,num_bits,e_bits)
+        self.register_buffer('M', torch.tensor([], requires_grad=False))  # 将M注册为buffer
+
+    def freeze(self, qi=None, qo=None):
+        if hasattr(self, 'qi') and qi is not None:
+            raise ValueError('qi has been provided in init function.')
+        if not hasattr(self, 'qi') and qi is None:
+            raise ValueError('qi is not existed, should be provided.')
+        if qi is not None:
+            self.qi = qi
+
+        if hasattr(self, 'qo') and qo is not None:
+            raise ValueError('qo has been provided in init function.')
+        if not hasattr(self, 'qo') and qo is None:
+            raise ValueError('qo is not existed, should be provided.')
+        if qo is not None:
+            self.qo = qo
+        self.M.data = (self.qi.scale / self.qo.scale).data
+
+    def forward(self, x):
+        if hasattr(self, 'qi'):
+            self.qi.update(x)
+            x = FakeQuantize.apply(x, self.qi)   # 与ReLu一样，先更新qi的scale，再将x用PoT表示了 (不过一般前一层的qo都是True，则x已经被PoT表示了)
+
+
+        
+        x = F.adaptive_avg_pool2d(x,(1, 1))   # 对输入输出都量化一下就算是量化了
+        
+
+        if hasattr(self, 'qo'):
+            self.qo.update(x)
+            x = FakeQuantize.apply(x, self.qo)
+
+        return x
+
+    def quantize_inference(self, x):
+        
+        x = F.adaptive_avg_pool2d(x,(1, 1))   # 对输入输出都量化一下就算是量化了  
+        x = self.M * x
+        # if self.quant_type is 'INT':
+        x = get_nearest_val(self.quant_type,x)
+
+       
+
+        return x
+    
+class QConvBNReLU6(QModule):
+
+    def __init__(self, quant_type, conv_module, bn_module, qi=True, qo=True, num_bits=8, e_bits=3):
+        super(QConvBNReLU6, self).__init__(quant_type, qi, qo, num_bits, e_bits)
+        self.conv_module = conv_module
+        self.bn_module = bn_module
+        self.qw = QParam(quant_type, num_bits,e_bits)
+        self.register_buffer('M', torch.tensor([], requires_grad=False))  # 将M注册为buffer
+
+    def fold_bn(self, mean, std):
+        if self.bn_module.affine:
+            gamma_ = self.bn_module.weight / std
+            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
+            else:
+                bias = self.bn_module.bias - gamma_ * mean
+        else:
+            gamma_ = 1 / std
+            weight = self.conv_module.weight * gamma_
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean
+            else:
+                bias = -gamma_ * mean
+            
+        return weight, bias
+
+    def freeze(self, qi=None, qo=None):
+        if hasattr(self, 'qi') and qi is not None:
+            raise ValueError('qi has been provided in init function.')
+        if not hasattr(self, 'qi') and qi is None:
+            raise ValueError('qi is not existed, should be provided.')
+
+        if hasattr(self, 'qo') and qo is not None:
+            raise ValueError('qo has been provided in init function.')
+        if not hasattr(self, 'qo') and qo is None:
+            raise ValueError('qo is not existed, should be provided.')
+
+        if qi is not None:
+            self.qi = qi
+        if qo is not None:
+            self.qo = qo
+
+        self.M.data = (self.qw.scale * self.qi.scale / self.qo.scale).data
+
+        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
+
+        self.conv_module.weight.data = self.qw.quantize_tensor(weight.data)
+        self.conv_module.weight.data = self.conv_module.weight.data - self.qw.zero_point
+
+        self.conv_module.bias.data = quantize_tensor(self.quant_type,
+                                                     bias, scale=self.qi.scale * self.qw.scale,
+                                                     zero_point=0., qmax=self.bias_qmax,is_bias=True)
+
+    
+    def fakefreeze(self):
+        self.conv_module.weight.data = self.qw.dequantize_tensor(self.conv_module.weight.data)
+        self.conv_module.bias.data = dequantize_tensor(self.conv_module.bias.data,scale=self.qi.scale * self.qw.scale, zero_point=0.)
+    
+    
+    
+    def forward(self, x):
+
+        if hasattr(self, 'qi'):
+            self.qi.update(x)
+            x = FakeQuantize.apply(x, self.qi)
+
+        if self.training:
+            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
+                            stride=self.conv_module.stride,
+                            padding=self.conv_module.padding,
+                            dilation=self.conv_module.dilation,
+                            groups=self.conv_module.groups)
+            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
+            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
+            # mean = y.mean(1)
+            # var = y.var(1)
+            mean = y.mean(1).detach()
+            var = y.var(1).detach()
+            self.bn_module.running_mean = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
+                self.bn_module.momentum * mean
+            self.bn_module.running_var = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
+                self.bn_module.momentum * var
+        else:
+            mean = Variable(self.bn_module.running_mean)
+            var = Variable(self.bn_module.running_var)
+
+        std = torch.sqrt(var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(mean, std)
+
+        self.qw.update(weight.data)
+
+        x = F.conv2d(x, FakeQuantize.apply(weight, self.qw), bias, 
+                stride=self.conv_module.stride,
+                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
+                groups=self.conv_module.groups)
+
+        x = F.relu6(x)
+
+        if hasattr(self, 'qo'):
+            self.qo.update(x)
+            x = FakeQuantize.apply(x, self.qo)
+
+        return x
+    
+    def quantize_inference(self, x):
+        a = torch.tensor(6)
+        a = self.qo.quantize_tensor(a)
+
+
+
+        x = x - self.qi.zero_point
+        x = self.conv_module(x)
+        x = self.M * x
+
+        # if self.quant_type is not 'POT':
+        #     x = get_nearest_val(self.quant_type,x)
+        x = get_nearest_val(self.quant_type,x)
+        
+        x = x + self.qo.zero_point        # 属于qo范围的数据
+        x.clamp_(min=0, max=a.item())
+        return x
+
+class QModule_2(nn.Module):
+
+    def __init__(self,quant_type, qi0=True, qi1=True, qo=True, num_bits=8, e_bits=3):
+        super(QModule_2, self).__init__()
+        if qi0:
+            self.qi0 = QParam(quant_type,num_bits, e_bits)  # qi在此处就已经被num_bits和mode赋值了
+        if qi1:
+            self.qi1 = QParam(quant_type,num_bits, e_bits)  # qi在此处就已经被num_bits和mode赋值了
+        if qo:
+            self.qo = QParam(quant_type,num_bits, e_bits)  # qo在此处就已经被num_bits和mode赋值了
+
+        self.quant_type = quant_type
+        self.num_bits = num_bits
+        self.e_bits = e_bits
+        self.bias_qmax = bias_qmax(quant_type)
+
+
+    def freeze(self):
+        pass
+
+    def fakefreeze(self):
+        pass
+
+    def quantize_inference(self, x):
+        raise NotImplementedError('quantize_inference should be implemented.')
+
+
+class QElementwiseAdd(QModule_2):
+    def __init__(self, quant_type, qi0=True, qi1=True,  qo=True, num_bits=8, e_bits=3):
+        super(QElementwiseAdd, self).__init__(quant_type, qi0, qi1, qo, num_bits, e_bits)
+        self.register_buffer('M0', torch.tensor([], requires_grad=False))  # 将M注册为buffer
+        self.register_buffer('M1', torch.tensor([], requires_grad=False))  # 将M注册为buffer
+    
+    def freeze(self, qi0=None, qi1=None ,qo=None):
+
+        if hasattr(self, 'qi') and qi0 is not None:
+            raise ValueError('qi0 has been provided in init function.')
+        if not hasattr(self, 'qi') and qi0 is None:
+            raise ValueError('qi0 is not existed, should be provided.')
+        if hasattr(self, 'qi1') and qi0 is not None:
+            raise ValueError('qi1 has been provided in init function.')
+        if not hasattr(self, 'qi1') and qi0 is None:
+            raise ValueError('qi1 is not existed, should be provided.')
+
+        if hasattr(self, 'qo') and qo is not None:
+            raise ValueError('qo has been provided in init function.')
+        if not hasattr(self, 'qo') and qo is None:
+            raise ValueError('qo is not existed, should be provided.')
+
+        # 这里因为在池化或者激活的输入，不需要对最大值和最小是进行额外的统计，会共享相同的输出
+        if qi0 is not None:
+            self.qi0 = qi0
+        if qi1 is not None:
+            self.qi1 = qi1
+        if qo is not None:
+            self.qo = qo
+
+        # 根据https://zhuanlan.zhihu.com/p/156835141, 这是式3 的系数
+        self.M0.data = self.qi0.scale / self.qo.scale
+        self.M1.data = self.qi1.scale / self.qi0.scale
+        # self.M0.data = self.qi0.scale / self.qo.scale
+        # self.M1.data = self.qi1.scale / self.qo.scale
+
+    def forward(self, x0, x1):  # 前向传播,输入张量,x为浮点型数据
+        if hasattr(self, 'qi0'):
+            self.qi0.update(x0)
+            x0 = FakeQuantize.apply(x0, self.qi0)  # 对输入张量X完成量化
+        if hasattr(self, 'qi1'):
+            self.qi1.update(x1)
+            x1 = FakeQuantize.apply(x1, self.qi1)  # 对输入张量X完成量化
+
+        x = x0 + x1 
+
+        if hasattr(self, 'qo'):
+            self.qo.update(x)
+            x = FakeQuantize.apply(x, self.qo)
+
+        return x
+    
+    def quantize_inference(self, x0, x1):  # 此处input为已经量化的qx
+        x0 = x0 - self.qi0.zero_point
+        x1 = x1 - self.qi1.zero_point
+        
+        x = self.M0 * (x0 + x1*self.M1)
+
+        # if self.quant_type is 'INT':
+        x = get_nearest_val(self.quant_type,x)
+
+        x = x + self.qo.zero_point
+
+
+        return x
+
+
+
+# new modules for full-precision model - fold bn
+# inference应该也需要相应的适配
+
+
+
+class ConvBNReLU(nn.Module):
+
+    def __init__(self,conv_module, bn_module):
+        super(ConvBNReLU, self).__init__()
+        self.conv_module = conv_module
+        self.bn_module = bn_module
+
+    def fold_bn(self, mean, std):
+        if self.bn_module.affine:
+            gamma_ = self.bn_module.weight / std
+            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
+            else:
+                bias = self.bn_module.bias - gamma_ * mean
+        else:
+            gamma_ = 1 / std
+            weight = self.conv_module.weight * gamma_
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean
+            else:
+                bias = -gamma_ * mean
+            
+        return weight, bias
+
+    def freeze(self):
+
+        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
+
+        self.conv_module.weight.data = weight.data
+        if self.conv_module.bias is None:
+            self.conv_module.bias = nn.Parameter(bias)
+        else:
+            self.conv_module.bias.data = bias        
+
+    def fakefreeze(self):
+        pass
+        
+    
+    def forward(self, x):
+
+        if self.training:
+            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
+                            stride=self.conv_module.stride,
+                            padding=self.conv_module.padding,
+                            dilation=self.conv_module.dilation,
+                            groups=self.conv_module.groups)
+            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
+            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
+            # mean = y.mean(1)
+            # var = y.var(1)
+            mean = y.mean(1).detach()
+            var = y.var(1).detach()
+            self.bn_module.running_mean = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
+                self.bn_module.momentum * mean
+            self.bn_module.running_var = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
+                self.bn_module.momentum * var
+        else:
+            mean = Variable(self.bn_module.running_mean)
+            var = Variable(self.bn_module.running_var)
+
+        std = torch.sqrt(var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(mean, std)
+
+
+        x = F.conv2d(x, weight, bias, 
+                stride=self.conv_module.stride,
+                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
+                groups=self.conv_module.groups)
+
+        x = F.relu(x)
+
+        return x
+    
+    def quantize_inference(self, x):
+        x = self.conv_module(x)  
+        x.clamp_(min=0)
+        return x
+        
+
+class ConvBN(nn.Module):
+
+    def __init__(self,conv_module, bn_module):
+        super(ConvBN, self).__init__()
+        self.conv_module = conv_module
+        self.bn_module = bn_module
+
+    def fold_bn(self, mean, std):
+        if self.bn_module.affine:
+            gamma_ = self.bn_module.weight / std
+            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
+            else:
+                bias = self.bn_module.bias - gamma_ * mean
+        else:
+            gamma_ = 1 / std
+            weight = self.conv_module.weight * gamma_
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean
+            else:
+                bias = -gamma_ * mean
+            
+        return weight, bias
+
+    def freeze(self):
+
+        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
+
+        self.conv_module.weight.data = weight.data
+        if self.conv_module.bias is None:
+            self.conv_module.bias = nn.Parameter(bias)
+        else:
+            self.conv_module.bias.data = bias        
+
+    def fakefreeze(self):
+        pass
+        
+    
+    def forward(self, x):
+
+        if self.training:
+            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
+                            stride=self.conv_module.stride,
+                            padding=self.conv_module.padding,
+                            dilation=self.conv_module.dilation,
+                            groups=self.conv_module.groups)
+            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
+            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
+            # mean = y.mean(1)
+            # var = y.var(1)
+            mean = y.mean(1).detach()
+            var = y.var(1).detach()
+            self.bn_module.running_mean = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
+                self.bn_module.momentum * mean
+            self.bn_module.running_var = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
+                self.bn_module.momentum * var
+        else:
+            mean = Variable(self.bn_module.running_mean)
+            var = Variable(self.bn_module.running_var)
+
+        std = torch.sqrt(var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(mean, std)
+
+
+        x = F.conv2d(x, weight, bias, 
+                stride=self.conv_module.stride,
+                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
+                groups=self.conv_module.groups)
+
+        return x
+    
+    def quantize_inference(self, x):
+        x = self.conv_module(x)  
+        return x
+  
+
+class ConvBNReLU6(nn.Module):
+    def __init__(self,conv_module, bn_module):
+        super(ConvBNReLU6, self).__init__()
+        self.conv_module = conv_module
+        self.bn_module = bn_module
+
+    def fold_bn(self, mean, std):
+        if self.bn_module.affine:
+            gamma_ = self.bn_module.weight / std
+            weight = self.conv_module.weight * gamma_.view(self.conv_module.out_channels, 1, 1, 1)
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean + self.bn_module.bias
+            else:
+                bias = self.bn_module.bias - gamma_ * mean
+        else:
+            gamma_ = 1 / std
+            weight = self.conv_module.weight * gamma_
+            if self.conv_module.bias is not None:
+                bias = gamma_ * self.conv_module.bias - gamma_ * mean
+            else:
+                bias = -gamma_ * mean
+            
+        return weight, bias
+
+    def freeze(self):
+
+        std = torch.sqrt(self.bn_module.running_var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(self.bn_module.running_mean, std)
+
+        self.conv_module.weight.data = weight.data
+        if self.conv_module.bias is None:
+            self.conv_module.bias = nn.Parameter(bias)
+        else:
+            self.conv_module.bias.data = bias        
+
+    def fakefreeze(self):
+        pass
+        
+    
+    def forward(self, x):
+
+        if self.training:
+            y = F.conv2d(x, self.conv_module.weight, self.conv_module.bias, 
+                            stride=self.conv_module.stride,
+                            padding=self.conv_module.padding,
+                            dilation=self.conv_module.dilation,
+                            groups=self.conv_module.groups)
+            y = y.permute(1, 0, 2, 3) # NCHW -> CNHW
+            y = y.contiguous().view(self.conv_module.out_channels, -1) # CNHW -> C,NHW
+            # mean = y.mean(1)
+            # var = y.var(1)
+            mean = y.mean(1).detach()
+            var = y.var(1).detach()
+            self.bn_module.running_mean = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_mean + \
+                self.bn_module.momentum * mean
+            self.bn_module.running_var = \
+                (1 - self.bn_module.momentum) * self.bn_module.running_var + \
+                self.bn_module.momentum * var
+        else:
+            mean = Variable(self.bn_module.running_mean)
+            var = Variable(self.bn_module.running_var)
+
+        std = torch.sqrt(var + self.bn_module.eps)
+
+        weight, bias = self.fold_bn(mean, std)
+
+
+        x = F.conv2d(x, weight, bias, 
+                stride=self.conv_module.stride,
+                padding=self.conv_module.padding, dilation=self.conv_module.dilation, 
+                groups=self.conv_module.groups)
+
+        x = F.relu6(x)
+
+        return x
+    
+    def quantize_inference(self, x):
+        x = self.conv_module(x)  
+        x.clamp_(min=0,max=6)
+        return x

mzh/new_mzh/Naive_MIA/readme.md

+## Naive MIA 
+
+#### update 2023.5.24
+
+1\. 思路
+
+   （1）一共需要训练3个模型，分别是Target Model， Shadow Model，Attack Model. 其中Target Model是被攻击的模型。
+
+   （2）假设攻击者已知Target Model的结构并了解其训练所用的数据集的特征，于是Shadow Model采用与Target Model相同的结构。将数据集切分，分别用于Target Model和Shadow Model的训练和测试。
+
+   （3）在训练和测试Target Model和Shadow Model时，我们可以分别构建Attack Model的测试集和训练集。Attack Model的输入是Target Model或Shadow Model对一个图片输入的输出向量，输出是该图片是否属于Target Model或Shadow Model的训练集。我们构造Shadow Model就是为了构建Attack Model的训练集（因为我们作为攻击者，只知道Shadow Model的训练集，测试集是什么，而不知道Target Model的），Target Model的输出向量和训练集、测试集可以作为Attack Model的测试集，检验攻击成果。
+
+   （4）Attack Model是一个二分类网络，由若干FC，ReLU，BN层组成
+
+   （5）最终输出攻击的成功率 （即Attack Model的acc）和一些统计信息
+
+2\. 代码文件说明
+
+   attack.py: 
+   核心文件，``get_cmd_arguments()``负责了读取各种参数配置
+   ``split_dataset()``将数据集切分
+   ``get_data_loader()``构造data loader
+   `` attack_inference``负责Attack Model的推理
+   ``create_attack()``负责的工作较多，包括训练Target Model，训练Shadow Model，通过``train_model()``在训练中构造Attack Model的数据集（也可以load训练好的模型权值后通过``prepare_attack_data`构造）.    训练Attack Model，并测试。
+
+   model.py : 
+   支持的各种模型结构，目前用到的包括ResNet18，50，152，MobileNetV2
+
+   train.py：
+   ``prepare_attack_data()``构建Attack Model训练/测试所用的数据集，attack_X是Target Model或Shadow Model的输出，attack_Y是Target Model或Shadow Model是否为其训练集数据
+   ```train_per_epoch()```负责Target Model或Shadow Model每个epoch的训练
+
+   ``val_per_epoch()``负责Target Model或Shadow Model每个epoch的测试/验证
+
+   ``train_attack_model()``负责训练Attack Model的训练
+
+   ``train_model()``负责组织训练模型，调用上述函数  
+
+   
+
+   其余程序文件的含义与之前用到的基本相同
+
+3\. 结果
+
+   - 先后分别在CIFAR10和CIFAR100上尝试了对ResNet18，50，152，MobileNetV2的攻击，在CIFAR100上一般能比CIFAR10上取得更好的结果，原因可能是CIFAR100数据集每类数据图片更少，更难训练，更容易过拟合，因此对MIA更脆弱。
+
+     只有对使用CIFAR10训练的MobileNetV2的攻击取得了相对显著的结果(Attack Model acc=61.57%)，其余情况下，Attack Model的acc均在52%~56%，考虑到二分类问题随机情况下也有50%的acc，攻击效果不是很显著。
+
+   - 具体数据
+
+     Target Model和Shadow Model的optimizer是Adam，Attack Model的optimizer是SGD，lr_scheduler都是CosineAnnealingLR
+
+     * ResNet18 + CIFAR10：
+
+     ```
+     Validation Accuracy for the Best Attack Model is: 54.40 %
+     ----Attack Model Testing----
+     Y:tensor([1, 1, 1,  ..., 0, 0, 0])
+     Length of Attack Model test dataset : [30000]
+     Attack Test Accuracy is  : 53.26%
+     ---Detailed Results----
+                   precision    recall  f1-score   support
+     
+       Non-Member       0.55      0.38      0.45     15000
+           Member       0.52      0.69      0.59     15000
+     
+         accuracy                           0.53     30000
+        macro avg       0.54      0.53      0.52     30000
+     weighted avg       0.54      0.53      0.52     30000
+     ```
+
+     * ResNet50 + CIFAR10：
+
+     ```
+     Validation Accuracy for the Best Attack Model is: 55.70 %
+     ----Attack Model Testing----
+     Y:tensor([1, 1, 1,  ..., 0, 0, 0])
+     Length of Attack Model test dataset : [30000]
+     Attack Test Accuracy is  : 52.20%
+     ---Detailed Results----
+                   precision    recall  f1-score   support
+     
+       Non-Member       0.52      0.47      0.49     15000
+           Member       0.52      0.58      0.55     15000
+     
+         accuracy                           0.52     30000
+        macro avg       0.52      0.52      0.52     30000
+     weighted avg       0.52      0.52      0.52     30000
+     ```
+
+     * ResNet152 + CIFAR10
+
+     ```
+     ----Attack Model Testing----
+     Y:tensor([1, 1, 1,  ..., 0, 0, 0])
+     Length of Attack Model test dataset : [30000]
+     Attack Test Accuracy is  : 52.50%
+     ---Detailed Results----
+                   precision    recall  f1-score   support
+     
+       Non-Member       0.54      0.37      0.44     15000
+           Member       0.52      0.68      0.59     15000
+     
+         accuracy                           0.53     30000
+        macro avg       0.53      0.53      0.51     30000
+     weighted avg       0.53      0.53      0.51     30000
+     ```
+
+     * MobileNetV2 + CIFAR10
+
+     ```
+     Validation Accuracy for the Best Attack Model is: 65.20 %
+     ----Attack Model Testing----
+     Y:tensor([1, 1, 1,  ..., 0, 0, 0])
+     Length of Attack Model test dataset : [30000]
+     Attack Test Accuracy is  : 61.57%
+     ---Detailed Results----
+                   precision    recall  f1-score   support
+     
+       Non-Member       0.63      0.57      0.60     15000
+           Member       0.61      0.66      0.63     15000
+     
+         accuracy                           0.62     30000
+        macro avg       0.62      0.62      0.61     30000
+     weighted avg       0.62      0.62      0.61     30000
+     ```
+
+     * ResNet50 + CIFAR100：
+
+     ```
+     ----Attack Model Testing----
+     Y:tensor([1, 1, 1,  ..., 0, 0, 0])
+     Length of Attack Model test dataset : [30000]
+     Attack Test Accuracy is  : 54.30%
+     ---Detailed Results----
+                   precision    recall  f1-score   support
+     
+       Non-Member       0.54      0.61      0.57     15000
+           Member       0.55      0.47      0.51     15000
+     
+         accuracy                           0.54     30000
+        macro avg       0.54      0.54      0.54     30000
+     weighted avg       0.54      0.54      0.54     30000
+     ```
+
+     * ResNet152  + CIFAR100：
+
+     ```
+     Attack Test Accuracy is  : 56.11%
+     ---Detailed Results----
+                   precision    recall  f1-score   support
+     
+       Non-Member       0.56      0.60      0.58     15000
+           Member       0.57      0.52      0.54     15000
+     
+         accuracy                           0.56     30000
+        macro avg       0.56      0.56      0.56     30000
+     weighted avg       0.56      0.56      0.56     30000
+     ```
+
+     * MobileNetV2 + CIFAR100：
+
+     ```
+     Validation Accuracy for the Best Attack Model is: 55.50 %
+     ----Attack Model Testing----
+     Y:tensor([1, 1, 1,  ..., 0, 0, 0])
+     Length of Attack Model test dataset : [30000]
+     Attack Test Accuracy is  : 51.33%
+     ---Detailed Results----
+                   precision    recall  f1-score   support
+     
+       Non-Member       0.51      0.52      0.52     15000
+           Member       0.51      0.51      0.51     15000
+     
+         accuracy                           0.51     30000
+        macro avg       0.51      0.51      0.51     30000
+     weighted avg       0.51      0.51      0.51     30000
+     ```
+
+     
+
+4\. 问题及改进方向
+
+   Q1：大部分情况下攻击效果较差
+
+   A1：尝试换一种MIA攻击方式，尝试Loss Trajectory MIA。之后再仔细检查下有无代码问题。
+
+   Q2：对于CIFAR100，Target Model和Shadow Model的训练效果差
+
+   A2：因为CIFAR100每类的训练数据本来就比较少，还分别拆分给了Target Model和Shadow Model的train set和test set，导致训练数据过少，因此模型训练效果较差。可以考虑调整训练策略，或是**改用CINIC-10数据集**，CINIC-10 是 CIFAR-10 通过添加下采样的 ImageNet 图像扩展得到，有 270,000 张图像，是 CIFAR 的 4.5 倍，图像大小与 CIFAR 中的一样，不需要对代码进行大量修改，不过有一点问题是图像来源于 CIFAR 和 ImageNet，这些图像的分布不一定相同，可能不利于训练。
+
+   Q3：如何把量化也考虑进来
+
+   A3：我认为主要取决于我们假设的攻击情景是怎样的。
+
+   如果认为攻击者不知道要攻击的模型是全精度的还是量化后的，那么就会采用全精度的Shadow Model，进而根据Shadow Model训练好Attack Model。我们只需要把Target Model进行相应的量化，而后再根据其输出构造Attack Model的test set，就可以将量化考虑进来了。
+
+   如果认为攻击者知道要攻击的模型是量化的，同时还知道是用的什么量化数据表示方式、位宽等，那么会比较麻烦。简单的方法是还是训练一个全精度的Shadow Model然后量化，之后再构造Attack Model的训练集。其余步骤于上一个情况相同。可能会取得更好的攻击效果(因为是根据量化后的Shadow Model构建的Attack Model的训练集，Attack Model更能适应量化模型的output). 复杂的方法是引入QAT，但QAT目前在ResNet系列和MobileNetV2上还比较难训练起来。
+
+   
\ No newline at end of file

mzh/new_mzh/Naive_MIA/train.py

+import torch
+from torch.utils.data.dataset import TensorDataset
+import torch.nn.functional as F
+import copy
+import os
+
+
+#Prepare data for Attack Model
+def prepare_attack_data(attack_X,
+                        attack_Y,
+                        model,
+                        iterator,
+                        device,
+                        top_k=False,
+                        test_dataset=False):
+    
+
+    
+    model.eval()
+    with torch.no_grad():
+        for inputs, _ in iterator:  # train loader
+            # Move tensors to the configured device
+            inputs = inputs.to(device)
+            
+            #Forward pass through the model
+            outputs = model(inputs)
+            
+            #To get class probabilities
+            # posteriors = F.softmax(outputs, dim=1)
+            posteriors = outputs
+            if top_k:
+                #Top 3 posterior probabilities(high to low) for train samples
+                topk_probs, _ = torch.topk(posteriors, 3, dim=1)
+                attack_X.append(topk_probs.cpu())
+            else:
+                # 只用了output probabilities向量
+                # TODO 可以用loss等 (用target model的output和shadow model的output做loss,不过这个需要能获得target model的output)
+                attack_X.append(posteriors.cpu())
+
+            # size(0) = batch size   
+            if test_dataset:
+                attack_Y.append(torch.zeros(posteriors.size(0),dtype=torch.long))
+            else:
+                attack_Y.append(torch.ones(posteriors.size(0), dtype=torch.long))
+        
+    return attack_X, attack_Y
+    
+def train_per_epoch(model,
+                    train_iterator,
+                    criterion,
+                    optimizer,
+                    device,
+                    bce_loss=False):
+    epoch_loss = 0
+    epoch_acc = 0
+    correct = 0
+    total = 0
+    
+    model.train()
+    for _ , (features, target) in enumerate(train_iterator):
+        # Move tensors to the configured device
+        features = features.to(device)
+        target = target.to(device)
+        
+        # Forward pass
+        outputs = model(features)
+        
+        # For Attack Model
+        if bce_loss:
+            #For BCE loss
+            loss = criterion(outputs, target.unsqueeze(1))
+        # For Target Model and Shadow Model
+        else:
+            loss = criterion(outputs, target)
+        
+        # Backward pass and optimize
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+        
+        #Record Loss
+        epoch_loss += loss.item()
+
+        #Get predictions for accuracy calculation
+        _, predicted = torch.max(outputs.data, 1)
+        # 统计epoch内的总数
+        total += target.size(0)
+        correct += (predicted == target).sum().item()
+
+    #Per epoch valdication accuracy calculation
+    epoch_acc = correct / total
+    epoch_loss = epoch_loss / total
+
+    return epoch_loss, epoch_acc
+
+def val_per_epoch(model,
+                val_iterator,
+                criterion,
+                device,
+                bce_loss=False):
+
+    epoch_loss = 0
+    epoch_acc = 0
+    correct = 0
+    total =0
+
+    model.eval()
+    with torch.no_grad():
+        for _,(features,target) in enumerate(val_iterator):
+            features = features.to(device)
+            target = target.to(device)
+            
+            outputs = model(features)
+            #Caluclate the loss
+            if bce_loss:
+                #For BCE loss
+                loss = criterion(outputs, target.unsqueeze(1))
+            else:
+                loss = criterion(outputs,target)
+                
+            #record the loss
+            epoch_loss += loss.item()
+            
+            #Check Accuracy
+            _, predicted = torch.max(outputs.data, 1)
+            total += target.size(0)
+            correct += (predicted == target).sum().item()
+
+        #Per epoch valdication accuracy and loss calculation
+        epoch_acc = correct / total
+        epoch_loss = epoch_loss / total
+    
+    return epoch_loss, epoch_acc
+
+###############################
+# Training Attack Model
+###############################
+def train_attack_model(model,
+                    dataset,
+                    criterion,
+                    optimizer,
+                    lr_scheduler,
+                    device,
+                    model_path='./model',
+                    epochs=10,
+                    b_size=20,
+                    num_workers=1,
+                    verbose=False,
+                    earlystopping=False):
+        
+    n_validation = 1000 # number of validation samples
+    best_valacc = 0
+    stop_count = 0
+    patience = 10 # Early stopping
+
+    path = os.path.join(model_path,'best_attack_model.pt')
+        
+    train_loss_hist = []
+    valid_loss_hist = []
+    val_acc_hist = []
+
+    train_X, train_Y = dataset
+        
+    #Contacetnae list of tensors to a single tensor
+    t_X = torch.cat(train_X)
+    t_Y = torch.cat(train_Y)
+    
+    print(f"t_Y:{t_Y}")
+ 
+  
+    # #Create Attack Dataset
+    attackdataset = TensorDataset(t_X,t_Y)
+        
+    print('Shape of Attack Feature Data : {}'.format(t_X.shape))
+    print('Shape of Attack Target Data : {}'.format(t_Y.shape))
+    print('Length of Attack Model train dataset : [{}]'.format(len(attackdataset)))
+    print('Epochs [{}] and Batch size [{}] for Attack Model training'.format(epochs,b_size))
+        
+    #Create Train and Validation Split
+    n_train_samples = len(attackdataset) - n_validation
+    train_data, val_data = torch.utils.data.random_split(attackdataset, 
+                                                               [n_train_samples, n_validation])
+        
+
+    train_loader = torch.utils.data.DataLoader(dataset=train_data,
+                                                batch_size=b_size,
+                                                shuffle=True,
+                                                num_workers=num_workers)
+        
+    val_loader = torch.utils.data.DataLoader(dataset=val_data,
+                                                  batch_size=b_size,
+                                                  shuffle=False,
+                                                  num_workers=num_workers)
+    
+    
+    print('----Attack Model Training------')   
+    for i in range(epochs):
+            
+        train_loss, train_acc = train_per_epoch(model, train_loader, criterion, optimizer, device)
+        valid_loss, valid_acc = val_per_epoch(model, val_loader, criterion, device)
+
+        valid_loss_hist.append(valid_loss)
+        train_loss_hist.append(train_loss)
+        val_acc_hist.append(valid_acc)
+        
+        lr_scheduler.step()
+        
+        print ('Epoch [{}/{}], Train Loss: {:.3f} | Train Acc: {:.2f}% | Val Loss: {:.3f} | Val Acc: {:.2f}%'
+                 .format(i+1, epochs, train_loss, train_acc*100, valid_loss, valid_acc*100))
+
+        if earlystopping: 
+            if best_valacc<valid_acc:
+                print('Saving model checkpoint')
+                best_valacc = valid_acc
+                #Store best model weights
+                best_model = copy.deepcopy(model.state_dict())
+                torch.save(best_model, path)
+                stop_count = 0
+            else:
+                stop_count+=1
+                if stop_count >=patience: #early stopping check
+                    print('End Training after [{}] Epochs'.format(epochs+1))
+                    break
+        else:#Continue model training for all epochs
+           if best_valacc < valid_acc:
+                print('Saving model checkpoint')
+                best_valacc = valid_acc
+                #Store best model weights
+                best_model = copy.deepcopy(model.state_dict())
+                torch.save(best_model, path)
+            
+    return best_valacc
+    
+
+
+###################################
+# Training Target and Shadow Model
+###################################         
+# 
+# 每次调用该函数，可以得到一份全新的attck model的训练数据 (不过对于各种量化target model，目前打算采用相同的shadow model? 其实感觉用一个个量化后的shadow model可能更合适吗，这样shadow model是更接近于模仿target model，进而能得到一个个不同的全精度的attack model)   
+def train_model(model,
+                train_loader,
+                val_loader,
+                test_loader,
+                loss,
+                optimizer,
+                scheduler,
+                device,
+                model_path,
+                verbose=False,
+                num_epochs=50,
+                top_k=False,
+                earlystopping=False,
+                is_target=False):
+    
+    best_valacc = 0
+    patience = 20 # Early stopping
+    stop_count= 0
+    train_loss_hist = []
+    valid_loss_hist = []
+    val_acc_hist = []
+    attack_X = []
+    attack_Y = []
+    
+    if is_target:
+        print('----Target model training----')
+    else:
+        print('---Shadow model training----')
+    
+    #Path for saving best target and shadow models
+    target_path = os.path.join(model_path,'best_target_model.pt')
+    shadow_path = os.path.join(model_path,'best_shadow_model.pt')
+    
+    for epoch in range(num_epochs):
+        
+        train_loss, train_acc = train_per_epoch(model, train_loader, loss, optimizer, device)
+        valid_loss, valid_acc = val_per_epoch(model, val_loader, loss, device)
+
+        valid_loss_hist.append(valid_loss)
+        train_loss_hist.append(train_loss)
+        val_acc_hist.append(valid_acc)
+
+        scheduler.step()
+
+        print ('Epoch [{}/{}], Train Loss: {:.3f} | Train Acc: {:.2f}% | Val Loss: {:.3f} | Val Acc: {:.2f}%'.format(epoch+1, num_epochs, train_loss, train_acc*100, valid_loss, valid_acc*100))
+        
+        if earlystopping:
+            if best_valacc<valid_acc:
+                print('Saving model checkpoint')
+                best_valacc = valid_acc
+                #Store best model weights
+                # best_model = copy.deepcopy(model.state_dict())
+                if is_target:
+                    torch.save(model.state_dict(), target_path)
+                else:
+                    torch.save(model.state_dict(), shadow_path)
+                stop_count = 0
+            else:
+                stop_count+=1
+                if stop_count >=patience: #early stopping check
+                    print('End Training after [{}] Epochs'.format(epoch+1))
+                    break
+        else:#Continue model training for all epochs
+            if best_valacc<valid_acc:
+                print('Saving model checkpoint')
+                best_valacc = valid_acc
+                #Store best model weights
+                # best_model = copy.deepcopy(model.state_dict())
+                if is_target:
+                    torch.save(model.state_dict(), target_path)
+                else:
+                    torch.save(model.state_dict(), shadow_path)
+
+            # best_valacc = valid_acc
+            #Store best model weights
+            # best_model = copy.deepcopy(model.state_dict())
+            # if is_target:
+            #     torch.save(best_model, target_path)
+            # else:
+            #     torch.save(best_model, shadow_path)
+    
+    
+    if is_target:
+        print('----Target model training finished----')
+        print('Validation Accuracy for the Target Model is: {:.2f} %'.format(100* best_valacc))
+    else:
+        print('----Shadow model training finished-----')
+        print('Validation Accuracy for the Shadow Model is: {:.2f} %'.format(100* best_valacc))
+
+    if is_target:
+        print('----LOADING the best Target model for Test----')
+        model.load_state_dict(torch.load(target_path))
+    else:
+        print('----LOADING the best Shadow model for Test----')
+        model.load_state_dict(torch.load(shadow_path))
+    
+    #As the model is fully trained, time to prepare data for attack model.
+    #Training Data for members would come from shadow train dataset, and member inference from target train dataset respectively.
+
+    # target model的应该不能用做attack model的训练数据，而是用于测试数据，只有shadow model的可以用于构造attack model data
+    # if not is_target:
+    attack_X, attack_Y = prepare_attack_data(attack_X,attack_Y,model,train_loader,device,top_k)
+        # dataset被t_train, t_test, s_train, s_test平均分割为了四份
+
+        # 和test set的数据是重复的 (val由test数据抽取得到)
+        # val_loader中的数据是从t_train中抽出来的1k个，t_train因此减少了相应的数据
+        # attack_X, attack_Y = prepare_attack_data(attack_X,attack_Y,model,val_loader,device,top_k,test_dataset=True)
+    
+    # In test phase, we don't need to compute gradients (for memory efficiency)
+    print('----Test the Trained Network----')
+    model.eval() 
+    with torch.no_grad():
+        correct = 0
+        total = 0
+        for inputs, labels in test_loader:
+            inputs = inputs.to(device)
+            labels = labels.to(device)
+            
+            test_outputs = model(inputs)
+            
+            #Predictions for accuracy calculations
+            _, predicted = torch.max(test_outputs.data, 1)
+            total += labels.size(0)
+            correct += (predicted == labels).sum().item()
+            
+            # Posterior and labels for non-members
+            # probs_test = F.softmax(test_outputs, dim=1)
+            probs_test = test_outputs
+            if top_k:
+                #Take top K posteriors ranked high ---> low
+                topk_t_probs, _ = torch.topk(probs_test, 3, dim=1)
+                attack_X.append(topk_t_probs.cpu())
+            else:
+                # 把test set的数据也作为attack model的训练数据的一部分 (out)
+                attack_X.append(probs_test.cpu())
+            attack_Y.append(torch.zeros(probs_test.size(0), dtype=torch.long))
+
+        if is_target:
+            print('Test Accuracy of the Target model: {:.2f}%'.format(100 * correct / total))
+        else:
+            print('Test Accuracy of the Shadow model: {:.2f}%'.format(100 * correct / total)) 
+            
+
+    return attack_X, attack_Y
+
+    
+
+