From fec5468eeb2c10b55f2626f9e0220d3f40906cad Mon Sep 17 00:00:00 2001 From: Maxy Date: Wed, 25 Aug 2021 07:22:07 +0000 Subject: [PATCH 1/3] add data augmentation(work in process) --- .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 160 bytes .../data/__pycache__/__init__.cpython-38.pyc | Bin 0 -> 165 bytes .../__pycache__/dataloader.cpython-38.pyc | Bin 0 -> 15996 bytes captioning/data/dataloader.py | 106 ++++++++++++++++- .../__pycache__/AoAModel.cpython-38.pyc | Bin 0 -> 7282 bytes .../__pycache__/AttModel.cpython-38.pyc | Bin 0 -> 29855 bytes .../__pycache__/BertCapModel.cpython-38.pyc | Bin 0 -> 3941 bytes .../__pycache__/CaptionModel.cpython-38.pyc | Bin 0 -> 14779 bytes .../models/__pycache__/FCModel.cpython-38.pyc | Bin 0 -> 6365 bytes .../__pycache__/M2Transformer.cpython-38.pyc | Bin 0 -> 4100 bytes .../__pycache__/ShowTellModel.cpython-38.pyc | Bin 0 -> 5631 bytes .../TransformerModel.cpython-38.pyc | Bin 0 -> 14812 bytes .../__pycache__/__init__.cpython-38.pyc | Bin 0 -> 1905 bytes .../cachedTransformer.cpython-38.pyc | Bin 0 -> 16425 bytes .../models/__pycache__/utils.cpython-38.pyc | Bin 0 -> 1269 bytes .../__pycache__/loss_wrapper.cpython-38.pyc | Bin 0 -> 2686 bytes .../modules/__pycache__/losses.cpython-38.pyc | Bin 0 -> 5806 bytes .../utils/__pycache__/__init__.cpython-38.pyc | Bin 0 -> 166 bytes .../utils/__pycache__/config.cpython-38.pyc | Bin 0 -> 4669 bytes .../__pycache__/eval_utils.cpython-38.pyc | Bin 0 -> 9279 bytes .../utils/__pycache__/misc.cpython-38.pyc | Bin 0 -> 8382 bytes .../utils/__pycache__/opts.cpython-38.pyc | Bin 0 -> 13288 bytes .../utils/__pycache__/resnet.cpython-38.pyc | Bin 0 -> 2545 bytes .../__pycache__/resnet_utils.cpython-38.pyc | Bin 0 -> 1117 bytes .../utils/__pycache__/rewards.cpython-38.pyc | Bin 0 -> 4770 bytes captioning/utils/eval_utils.py | 9 +- captioning/utils/opts.py | 12 ++ configs/fc.yml | 25 ++-- prepro.sh | 44 +++++++ scripts/prepro_feats.py | 3 +- scripts/prepro_feats_da.py | 112 ++++++++++++++++++ scripts/prepro_ngrams.py | 2 + scripts/write_f30k_da.py | 26 ++++ tools/train.py | 4 +- 34 files changed, 325 insertions(+), 18 deletions(-) create mode 100644 captioning/__pycache__/__init__.cpython-38.pyc create mode 100644 captioning/data/__pycache__/__init__.cpython-38.pyc create mode 100644 captioning/data/__pycache__/dataloader.cpython-38.pyc create mode 100644 captioning/models/__pycache__/AoAModel.cpython-38.pyc create mode 100644 captioning/models/__pycache__/AttModel.cpython-38.pyc create mode 100644 captioning/models/__pycache__/BertCapModel.cpython-38.pyc create mode 100644 captioning/models/__pycache__/CaptionModel.cpython-38.pyc create mode 100644 captioning/models/__pycache__/FCModel.cpython-38.pyc create mode 100644 captioning/models/__pycache__/M2Transformer.cpython-38.pyc create mode 100644 captioning/models/__pycache__/ShowTellModel.cpython-38.pyc create mode 100644 captioning/models/__pycache__/TransformerModel.cpython-38.pyc create mode 100644 captioning/models/__pycache__/__init__.cpython-38.pyc create mode 100644 captioning/models/__pycache__/cachedTransformer.cpython-38.pyc create mode 100644 captioning/models/__pycache__/utils.cpython-38.pyc create mode 100644 captioning/modules/__pycache__/loss_wrapper.cpython-38.pyc create mode 100644 captioning/modules/__pycache__/losses.cpython-38.pyc create mode 100644 captioning/utils/__pycache__/__init__.cpython-38.pyc create mode 100644 captioning/utils/__pycache__/config.cpython-38.pyc create mode 100644 captioning/utils/__pycache__/eval_utils.cpython-38.pyc create mode 100644 captioning/utils/__pycache__/misc.cpython-38.pyc create mode 100644 captioning/utils/__pycache__/opts.cpython-38.pyc create mode 100644 captioning/utils/__pycache__/resnet.cpython-38.pyc create mode 100644 captioning/utils/__pycache__/resnet_utils.cpython-38.pyc create mode 100644 captioning/utils/__pycache__/rewards.cpython-38.pyc create mode 100644 prepro.sh create mode 100644 scripts/prepro_feats_da.py create mode 100644 scripts/write_f30k_da.py diff --git a/captioning/__pycache__/__init__.cpython-38.pyc b/captioning/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf48a2d5e5c728ac060552793173321fecc4b594 GIT binary patch literal 160 zcmWIL<>g`kf}5Oji6Hthh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vNKP9mwQNN%l zvn;VBRX?|)Qa?FAB~{-uH!(fcIkBK5Ge0jgFI}&ovLwGKIYU1gE~FnHpP83g5+4tg VyTxIXo1apelWGSt{xc9W004TFCoBK} literal 0 HcmV?d00001 diff --git a/captioning/data/__pycache__/__init__.cpython-38.pyc b/captioning/data/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e5c5dbe2ef4537d99e8ac4489f41720ace5891ee GIT binary patch literal 165 zcmWIL<>g`kf}5Oji6Hthh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vDKP9mwQNN%l zvn;VBRX?|)Qa?FAB~{-uH!(fcIkBK5Ge0jgFI}&ovLwGKIYU1gE(FmSAD@|*SrQ)) WRB(&KCO1E&G$+*#WXES9W&i+A7byAw literal 0 HcmV?d00001 diff --git a/captioning/data/__pycache__/dataloader.cpython-38.pyc b/captioning/data/__pycache__/dataloader.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2a25f1cfe16200a8dfab3f13fdd31c0058386d5b GIT binary patch literal 15996 zcmbt*dyr(;S>Jv1<91I^&-Zfmt#X+WNp*OB8Wn`LFmiv;r6uxqfJ746E~ zUwiJI-JQA1iSxePZ#gxRQYU#RH^y-;nrkgff z%u3pcVou(Z^?Y%v8<#KU5tgn^*Jp||-LTo>tfZT(&pTkMzEE7~#w->WC1$C(^zD}N zROS^+We%L;GJIJz0pEnbu(NXD7FQ6GQQX>gi}b1&@Q)K&FTv#fkw{%H(fwt(M@n+0{V)GCMD zg=$bJ7gSaIm9VLwEQH&Bqp%nFg+^1?%eBI0wdNOU$X;!1^|G=N-IRBbQ1`>_rYb1ErK7?Org42==;dkb-m|5S-9gD zg7R&@P_EU;TVc~LhkM!&kQSvgrw%%50_!D6=vbiQ+V+pA+t+Fe~ zx05P)VMa}K-Py)>OTHZP{aZCD`6do?k$sP~PhyR9YYVYj4s6?MoR##`NdHS}n(5U{ zFHIw*W*HK>pi`)8N`I+JUZ<6X89KM|R${%;+OJ?{_J4G<>DTZC(;%UBXot=rins?R zu37i(jr)n%X|&=5Ct&Ob)w^-}{rjOGy!fH}c5L6hZ|ON?eCwqPsvMRtU?tuzhyI27 z-Te!drt&YmSTAq+&y`!8ikR8!t$ob?%JzjyKgeWcZdT!IB#FICwQ~KYDqlW_vi}br ztNr-!0$VSYs*P$`Dy?6tH7n&>a2XDhcRvr5^yaUCj!5=sW}<- zVXhj*ZEDith;1x#Ki;;KtZMcwL0c?S@{LPzRwez?JBRUY1Z7 zp_^gyFa=*)CGJ7ANlAqeqH8C6IKgnKE4O6bb5!!cIh?$Qx)5&Z!!8$TxEKOHE`h(p zeB|7-J8tA+j-)osXq^V@`V2^H>)6pZ*QQLrzKh{ptR~sFFhyefX^4fyRScC3L!4+r z)5T6Rh!fPVabl-gZOGun3GJ7a=8{}XXbwT_Vn!6xn~)i;y-+W)=&^BgxBT!j!UN7D z%b9jLG`(luDLB)1*3CNY>5<_`NCS?sXD71R(-KyJcjU)8aRpEC6376gCzd&#`aY`eYKt#Exb$v>$6>fKVYzg4RFjjeF|*gB=fgETJ>RXoz-L6VWBPSMt1>VJl=>f#a` z3k%33#Nt%8at^5TUGceHUqXDqC=!=Y2v3`$v*)-R|<>6wSa(@1OW%qZ6Wuv!*Q&U8a()yhz4p&L3klJ*hl ziAlA(Vo7hD1#hhX>>~WOwIzS*)EjK!qv5&X)CczS*2`alSY*E@*0=A@db(h zgv1Y*4<)m6UL_H0>#vQM{Rl4>X=O1Pj|D#iAJ;G>!(F)<=KT-|? zU_ES=dZh&1u1VU0l+ix(pd={2U3gFe*1h(4>^56rJV6-;0Ej*OR-A?`#@K27?PTT! zk{3z7gXB9&z7-_SipDN+WmJfv7;_f}pO3#;?joSYGq|u`vNBKpM z9mvTVJILgs8^<)901GA!IJyL$o|^~Mls_PvdBHR}JLfJZ^Y)_C4RPk-PB{qSACOKS zxfTVYPGdVW3)kR59*61L(_Ab-2m_4jveNF49nx1^zXK111(X*=lf7(-z!QWjK5n7i zWyCC5on(|$&Tb-jUx=;tf&D{vl#HBROTQkvz^O!($t{Ol@7J78DojO=h|s4x=`elh zVe1RL+VPMo6M2etCO_$HQhB7Jv{4V6&L!*CjmA^G^pRs~9^{L>$XS9S8P4BHMJa<^ zzZ-240n~o6+~}@6)r6U((F_aQVnAu^7i6Za8v{fI>0r9pE;m#SEA8f9SkTq2ZCFQ| zcOVa7C&5@YLWq;3;(lf=JD6Q@{@Eaa%>tX}4`|)g-5pii*Ui98#1{C8Qo>We^Lq zgxi;C$HT0_OtR16eadOi_h!Rzw#x8~;BLvo!v!0hLkwp`$K8U$!;{35Qg$bew}&Sa zQ8oVGPBzNkNOdMUxz1!K-PD8Sw!~evIExq(SL0KLIFx%DR#4Bs$4Xs*~w>sB;b$n@P-kJDO9|mn;~IX4EXqRLSR| zR1fUVJo^QTHVJq#kM#wpv@@;JE7q2)=DCuPZvpvcOd45HPjnVg`vUkz9Owd8gZPmX z`Awfe;lCk$E@=>BDVP1yFLiOO)N`E$^!uDzd_al++r%Hv-ecWJxicqVW%kQp?=OFbq%vQUSZo#V&6MFJ10;YZ=@v zqn<@G!k4Vqom>ApT0*HS7$GV9a0R`DUX5I}de1dHFdiUmJcfBPhFY4#_r&$RSzUAA=Vf zK|%ij+lZ|>wvOGyTtiB+QYo7DoDGprXDSoMhg+&@dXb*n*Jf^yh z{goZUzI|Ya_Kvgb>GPp`2nBo3Za9(iY3Y3FWg5T4$|knFbVG?XPp_f0ILq^h5+F@$ zAI82PVUWEW+qaJn4^Lubc(7eFhK&?B`z8jcpK3U38<~WD?FTO+~6d#l#80SgG5ROg|z)oL|h* ziZebMj3b`V=NuPgsGCfatl~T)SX(hBr{v9YSlQMkmMY4Z>p;SaGQo^Qs`P7#hX`A(oe7k> zfp_l2Qd6M^MV(UKJ?BQElkRv>;Ix}0c=jOmi*04d58&Wfgg?lckJ3@@Au5as(2TgV zT{nSAHu9FOc`M3A6Z6(X)@BkakkDlcd@V<)Tlh|m@J){Lgc(~j&!TOI)8P!-CdNUP zQj@R2?$Aj#^2j|)JB^wK@Mdf0Q0kAj&%!l-VtXgQ*J>=;dfxzX@~kl0f}s^n1RMP;;F;fLjvK$|1zxBov>2>pQRWc&~Tp zThKm^D;qpx4fn8Y%z>jM&^T_?g%c%QkLl|Kf?eta9w$-RO2Kn1_#|b!iiv8J?PEQ?RVo2c zQ?ML&)TKX((x7mW&Fo2Zfi3|F5=Nx30g3bigKH#48V1;Si*WoohKcx-jkn17RIS=5 z*S2Wy&~>vbuOk`SEgM$8DL>6F{byLfPm_o^mfid=Qzzb4*xruqJFw&%NTE9{S-MonJiPJ5P?zBn zgcV#tsO4rMN=`%6q-+QnxaJ{n-Urt*1kfsWH54&tAwht0w zkrf-ZShe5Rw`$LIWhJ>cR&Cg=`&MnZhiuifYE%BxGX4K&)n zuOB(aetkAH7Ta!W**aYuw_gw0pl$tGlyp{X*yoH58=J<@qJ}fFA(EJ$!tG{wRs4nIc`^M^YlW0n&c_fmT#nbBzhP{n&#N!X9gq ziH)~?{y}MJn(Zd0wQ%h*{c&!SK+9~;v`U+733~#?929$En>FH%3)@J$D{vL3 zhE21`Q=A^L&N5_X)FR7*w^+zUlJ}5MqZx%ISpNg`j`q3OVl{P;(RdBI2+$3}zeQI- zA_7nd^@fPdcT?z6S<0dP3AWSulWeDM-*O5a(zTNI^ah0Az)(u*+53$k)jvV~N091w zUr+I6eUjp!|~K_Py>*sAYt5H63jX=hFJ$GbCg{}IVby=HT{pmtY?8)4d1B|zRA%r zW=(f7tGQ9=0ipIVYZ{mprZ6=(gjs*C{Uo>rW*y=Axc(eRuk$15m1WO&%QonBLD1`0 zj-c1YW6k>nRiehK6V{@}d2h9~eq zdeBq^+}X32nCbdX&{K08s0$?TggH-yoBCIo{ntodBq7w) zze4isB)>r-=RrS1*D**_|0W~eAQ8Y*rt7yzq>+P`ehy)~XlZZ-4grMD6AuQ2&XXqt z!rtlqSWj)NCC8!J1=MB`*npt{S6@KvQ{;+!3(xkR(F{2PxLgEY!4nWYo(#BpS5nPg z4?k^gdmPxf=V9)9WV-h~vWSx#ABdYU_F`NJUc{7Mz~$CSMcd<&fo?`~jzBl^I)O)^ z+gIS|UQ6IXy1+LqF5ElP7im=DRY;hbQl#8w0<5GPSrbZk6=o@i@CWBMrHrE8KZ1G6 z!8wqlbx4So-kFf?vD|8T&F#32#gT9bJe6QLPKhYSXxpcS7FRMuT&qNWY#p&x$89C$ zy~bFa@xXRW-~$&$9b6fP-P#ank=ZaBn#%b)?hvPg?Y+%S+!IUC%5qg!B2Hzgz}M2a zie?~goE1xsIiyP)`wb2QTm9qQc!CFjxS_HO+i08}>_p>?(oI@za9%oXH1U^Fc5#L! zm&$uvb=(9j^BhS3Ji_D2=bAXe2?K+0x3r%fJ=XqM?hwf! ze}=t~LlO%u<@xA5{>(X8x2-%1lcXbSheDbSmc~8-2iV8@!S(=3{vJ#v(liN?S&7?u zm#|&vH$(W;92;zPVSq#>96l{=VB@gG9JFC{_~QH^LmBrVKid;L7L0A}l>zi!A5C|x zrjZ6@yA3$B@AX-9WD=S!>9dF*>qCk7b=E96XtZAT+2DfM$@LlyM`L@hW9*YFyaFbE zt6!nhUKncVH&fXI8Yx@u;~L3`-|p9WbYe)oC%5kbRmukcxGEX(J400tk0<+;*9Txf zz*V)A>piYtPppvW`wRUFNBi{wm2vj-3C@B1u)PT*3GOU10i4{ydqH0162hU8+|$RMPxyiF;CK73#4=MQ=P_S+`XvkM6=YpYnFaM# zxQ%Scprd#fN8ODxC0)5KPW@%Ze~Dz!^^}P5G`qH?{Y_k!+~#4(m)IjIL~&7_cG?$) zW*aPLI60#|esFR&o`0mNY-SpdH1#YTa)iUvN;$%b9QMXup>rj-(cmNm9!n zBmRr5g_F&)b4losw?`J`fa#49*~#7@M1BsB3_PwsS~$GIr%b%M-U%bvV5f&nT>F!FQZyMib9FvvFxGY!Zbx>77B1%Qzmg$oJzvdhM` z;z`=<^&g_xN&=+!qnm2rIXuB>kS>zM^2LWYG@rqB=u$5ZjU4ZV!Gc`kshs}L$fdtX z@<$*&81ttL7(|T?95MMH4F--k?^AAjwhxmAI0cFNKf3t{v*dj2bBkvYG&CY29&j<7 z>bX?n-oB^tMk|u&BdeZ6ArjehWbM`K=0t(#AF$u%g#QXonF%vC z5eVQh)c@>t)BiCubP?32nJ(ei{Q5CHLLgQu{W*JN5$P1!za!#rhQ=d57|C3-BfskL6UO38-Dc)Ao6Q7OfdSWArRvR?6=pwbz=ix8}au+UtD5auDUWm>y2aIqmS0g=+T!V3Cb_48*1mxSM z-WiJYh158|fOfrzT-XR>Y zKkhqvyl;$<9^>pANSh#MoUWE^m9hZ91OCSJ#fbJJi=2hR<|-VK-Qt}&kqlVZ*aYD& zqw@w=arf>Cx~k8k{$I$_8#sJ2YY>Z%@7HiIWsJ{1M$+sMpEwTl{;oYg;B!bxLZ#vV z$cYq1to}0+S%zG4BB0GV6gy(PhlB^2JX8~3=j54&0~*YO-pb?2cy2Z&N@VQ}KeXB0 z<0q>4qJ~|g**BWg-&mu?Xq<$%IL+_t1az6tWCQt7Og;|NPa{E`L>IL7A~ zqf-A%7WY?7laZ6=X0sNUo%0$4r{!a^^}VoK3)U4j<8e~lW|KZu->ZdHX!44Wd)AFD zJ;l;8B;Q8z1rphd%XTi^+oQAjLnG4f7kn0JJ}tV;B@%oK2(Mt`%QDO)99n|%jRzsK R40ZBXW`DrCG41EC{yl8!6dC{k literal 0 HcmV?d00001 diff --git a/captioning/data/dataloader.py b/captioning/data/dataloader.py index 7f2ed030..5eb87ac2 100644 --- a/captioning/data/dataloader.py +++ b/captioning/data/dataloader.py @@ -301,11 +301,115 @@ def __getitem__(self, index): def __len__(self): return len(self.info['images']) +class Dataset_DA(Dataset): + def __init__(self, opt): + self.opt = opt + self.seq_per_img = 1 + + # feature related options + self.use_fc = getattr(opt, 'use_fc', True) + self.use_att = getattr(opt, 'use_att', True) + self.use_box = getattr(opt, 'use_box', 0) + self.norm_att_feat = getattr(opt, 'norm_att_feat', 0) + self.norm_box_feat = getattr(opt, 'norm_box_feat', 0) + + # load the json file which contains additional information about the dataset + print('DataLoader da loading json file: ', opt.input_json_da) + self.info = json.load(open(self.opt.input_json_da)) + if 'ix_to_word' in self.info: + self.ix_to_word = self.info['ix_to_word'] + self.vocab_size = len(self.ix_to_word) + print('vocab size is ', self.vocab_size) + + # open the hdf5 file + print('DataLoader da loading h5 file: ', opt.input_fc_dir_da, opt.input_att_dir_da, opt.input_label_h5_da) + """ + Setting input_label_h5 to none is used when only doing generation. + For example, when you need to test on coco test set. + """ + if self.opt.input_label_h5 != 'none': + self.h5_label_file = h5py.File(self.opt.input_label_h5_da, 'r', driver='core') + # load in the sequence data + seq_size = self.h5_label_file['labels'].shape + self.label = self.h5_label_file['labels'][:] + self.seq_length = seq_size[1] + print('max sequence length in data is', self.seq_length) + # load the pointers in full to RAM (should be small enough) + self.label_start_ix = self.h5_label_file['label_start_ix'][:] + self.label_end_ix = self.h5_label_file['label_end_ix'][:] + else: + self.seq_length = 1 + + self.data_in_memory = getattr(opt, 'data_in_memory', False) + self.fc_loader = HybridLoader(self.opt.input_fc_dir_da, '.npy', in_memory=self.data_in_memory) + self.att_loader = HybridLoader(self.opt.input_att_dir_da, '.npz', in_memory=self.data_in_memory) + # self.box_loader = HybridLoader(self.opt.input_box_dir, '.npy', in_memory=self.data_in_memory) + + self.num_images = len(self.info['images']) # self.label_start_ix.shape[0] + print('read %d image features' %(self.num_images)) + + # separate out indexes for each of the provided splits + self.split_ix = {'train': [], 'val': [], 'test': []} + for ix in range(len(self.info['images'])): + img = self.info['images'][ix] + # data augmentation only for training + if img['split'] == 'train': + self.split_ix['train'].append(ix) + + print('assigned %d images to split train in da' %len(self.split_ix['train'])) + print('assigned %d images to split val in da' %len(self.split_ix['val'])) + print('assigned %d images to split test in da' %len(self.split_ix['test'])) + + def __getitem__(self, index): + """This function returns a tuple that is further passed to collate_fn + """ + ix, it_pos_now, wrapped = index #self.split_ix[index] + if self.use_att: + # att_feat = self.att_loader.get(str(self.info['images'][ix]['id'])) + att_feat = self.att_loader.get(f"{self.info['images'][ix]['imgid']}_{ix%5}") + # Reshape to K x C + att_feat = att_feat.reshape(-1, att_feat.shape[-1]) + if self.norm_att_feat: + att_feat = att_feat / np.linalg.norm(att_feat, 2, 1, keepdims=True) + if self.use_box: + box_feat = self.box_loader.get(str(self.info['images'][ix]['id'])) + # devided by image width and height + x1,y1,x2,y2 = np.hsplit(box_feat, 4) + h,w = self.info['images'][ix]['height'], self.info['images'][ix]['width'] + box_feat = np.hstack((x1/w, y1/h, x2/w, y2/h, (x2-x1)*(y2-y1)/(w*h))) # question? x2-x1+1?? + if self.norm_box_feat: + box_feat = box_feat / np.linalg.norm(box_feat, 2, 1, keepdims=True) + att_feat = np.hstack([att_feat, box_feat]) + # sort the features by the size of boxes + att_feat = np.stack(sorted(att_feat, key=lambda x:x[-1], reverse=True)) + else: + att_feat = np.zeros((0,0), dtype='float32') + if self.use_fc: + try: + # fc_feat = self.fc_loader.get(str(self.info['images'][ix]['id'])) + fc_feat = self.fc_loader.get(f"{self.info['images'][ix]['imgid']}_{ix%5}") + except: + # Use average of attention when there is no fc provided (For bottomup feature) + fc_feat = att_feat.mean(0) + else: + fc_feat = np.zeros((0), dtype='float32') + if hasattr(self, 'h5_label_file'): + seq = self.get_captions(ix, self.seq_per_img) + else: + seq = None + return (fc_feat, + att_feat, seq, + ix, it_pos_now, wrapped) + + class DataLoader: def __init__(self, opt): self.opt = opt self.batch_size = self.opt.batch_size - self.dataset = Dataset(opt) + if self.opt.data_augmentation: + self.dataset = data.ConcatDataset([Dataset(opt), Dataset_DA(opt)]) + else: + self.dataset = Dataset(opt) # Initialize loaders and iters self.loaders, self.iters = {}, {} diff --git a/captioning/models/__pycache__/AoAModel.cpython-38.pyc b/captioning/models/__pycache__/AoAModel.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7c09cb8be08fe09139f193ea1b7c432b03a2ac6b GIT binary patch literal 7282 zcmb_h+m9UCS+7&KuCDIs=^2kbw%2UTW-;5FFg9KUkytr#>}-PYPBv>Vmu8pRc2~9C zGu_qgQ`No<)eth4)*x8|-r#9QJner#q~tFM5)w#A9PtDak)J_4@WAl6{UeIliq*gxsVvr!)CiVTxu^3m)pzA`nDAZarLPcR}+8Z)V9-J0VRxUpwwoR)1cJj1yB}dlrx|- z;zdvvXOvY?n(-1SOG$`X&c^O-tF`aEU`M6ZwZ z&{R8fy79Y%QJQ3C`S#@QfE{j((ln9j&FTlEtk2Pp`dRW`lEm+g^rMcByWH*Jr!xD< z-a;j@+A6l%b{u|Y^%VL1wAyy!n($Ax-Onu7YI`_!-K@Sp8RY%%B%L^kZ;tZ)-Tw5i z{*uoRzpgz@sCmktz9Y{+e7!5;GFy;!F7DvXaD!HE<*Kl@?Hzj;S6$eJyJ8jU48&&B z&fdtKAE-I)yj56{TcCI36&^~z2=uMIQrO$11!)-DvGbI@g1nlCJ2i}y`F8EtjWK4$ zTE$9eug_Y!mc2{LLQz54D6EZ!{%5o_x>Z~m~( z_cq>)x>cb1xmy1mg;9_7GDuSV*R9KOC+}Q_$UW@j$>rhWCzrdBs>{DI?A%LkbjB2n zetK_h{3IXg?)}T%ebVJ2r7ydDeRN&q5M500?w~Wg8+WcGoP(> z#(3ET`luTqnP7e>)3}~xi6-~HrS(V~Cyq9ax^HS@J=%cgMVrw>UF8@zg|g}zt9F_i z|5iUuI@(kQ@|&6ZXp)a7c_eykDw8aUI-`yWZzm5XRQt}rIN!eY5!8LunRmLWM`H{# zwYaRv2p4#en#zpS3XPj3gPwk!D_7!3u2X-NUtWd_N%WUkZ*svSTkzwN%;$6C(F6T9 zx`Re8+cORkuTR(OGtU<*Q3}pqh4wKUame zueL6Jk<_er2^#Z<2oc~FgeV~S1q{PxRev={ZAa}|+lttr-@un0+q-HCEk(^?Eq13K z`l&VAp^(WRdwc`{Dqtr9CfsvT=Z&CAN_Z()}Ry+PP=shSEyuq%! zQ;nTi?b=hna87Ik8lBh{q;Gv@eWrSL?8lX-K^}57;F(j{J!jj#iIaWoO#NwP8sxRY z2OwRKcGo<^si!AxdxYPxyF6iPacU- z>z@O~RCC=)v+*cPU_$M+D@G0U3K~AQO_1L0;r}g-yQU`ygM3>uYDIS zf}jx7y{&gn63bp`GY@0ge1?{7++io%6x9M=ID;gG>h-r0Q=Oz4lsMT+^hNG~3a7u! z;@4PEGvyHaYp5A#t3TG1beTzig|x6c0tDQfj3$}>1vb$(i$)qx0NXg-POjl{LC0rn zwL&?s@vyXhV!Vf)!6Y%xX7WT|L%()C1KUF2CntSeE;DyTFtGi8QTecPMxlUkT5DW-a zsJ6cJ1q^Rh^dc52B|zMWeP2SXqHdzK1;ghPo72uPi6Rq51iJ~27@2y6b9M%0izZy@ z-$Y>+W@y$;27@`GiGT?73$_vQX?R3D0@b)AHlPd2k%v;wC*vi|-6nCGAjwYyI89FM zKD7?HOrBYSQH?%KdN2Vx5|8c(-0_UG>YO_J5XCDFkrg?B2)I6R>H1gzD+1z`m~M!d z`*49&;cf#A;c$2}7bg%$sVZ)TQ@RzxSxC%ztfIfif;=)+nvO^eO1%f4F`C`I@o6r4 zJ+WuH&aOc=^Evf5@Y(W8Wn*1)>-shenjI~y-Pb}{C;>g=9nyBtQuvSxU!7H3=U-vb zzDLoWvz+5tC%=SOL<;~397V$Q3|Q#^2HOA~!i>Hu?E>e7oNfCydID;^6WlBxQ=Y;r zy97i6!ua>h+66#~W71OPLD}nbTXgItz!48+jArprw!ZS}(T>5FnXkkn0S7F~w7fuY}fYL1Fjl$Vjl(?Ze_q-3#h7L}N zHeQM2mr3|#7FSuYy-#x|fc1i6(^@Lk>bKc0&?8P{HJuEj`v{jZ6X-OJ#8fng45EY_ ziFeixwcvxeVwHPI4rSD(hKd+0a8mNf_~JQc;%$l!>Wd(^+|mj-qj6r^5>NUar-kHj zM$1DjyrkY#^_S#@7j(S(@uv9Rz?)q}0l2YlTAwtg1Sr~gDBc6Vuy$}$yHK=JWo-Qk z`X1hBX>w?wbj0F79`^lf8y$TT(jc@tWvyecc4z{Bgv{ zNW2D9LBJDfBuwh@?7H$NA#UgM9!l`uEuKF^YDb zE&F`AOF9)n|2zs4kO4Q~9GO%7l;LTTLayOBmT(mSjiN!84|Pj=lb^y~DvW-E#V@j< zf__)AhH*bkZha&Ul>S||il58iKzC5UY>GYmeG^K^fVcR$1o;CAw_A%AC8 z18(OPxRe*2x81tBh%W}4rNKLvo$K9ssteOv1XAus5bQlg1lvLWX-6Uy)SUuW<{Z+L z=d4||Y9V4`cBrJ$&G6X59zD_Pc0b{a~c0f6( zq0QfQr_~fPEu+!5q{yzSMWx)sG{l&y#Hjy?o?#Il>~yv(>}#<7iO49-OMVLv;|m=@?vY zHFO_8`X-BSvEcpd9t+-r@zV|xDvwGzr39Bkxyd#!%aJ10BB~o8v=&MmXFtwOI7~We z)Jr;fCYjM$wKk>~JNw}H9AOzL4B0Z{-Dk7$y7J?o-mX$5MCOfO=0)0-ZZsI(>vxYc zN`H-gWEW6a_BWC7xMnwP#6+;6uR{>uv;+Jh&Jj~}L>OnL3bD({*6LHt`hC1}YxRaE zg-Uwlab{mXi~kh>>Ljgq#P6bw+IOaZfR6(kMeB{^I&s&@;d?Q!D)vS!m1JMmks%N)4zjWb@W)!8}PVFBwqw()1E^2qQ$&9qb>=0`|AS9`T@2h z!t~0Hp|eLbp$D+rvc<13>dDJ@cLsxg!YKuNzdUEh|Ax8CCFX8%lwfX1Pj2RY5mIMh zyDx|$)}{i1x(dk>U@fsx;F(Blf?hXDhIf;=?U(+ZI3%Vr`;ZGaZWsQS9&+7d!12eR zXS7-tqHrJiH$>d(0@vx+##1QUm;OQ45iOSIEz~80!^IkC`BMy|l&(nb!VL(n75o%=(}&3a+{TQn7`b z`uq6SQx-%TWN2_H#>pl_eVxQVV)2J4T7C&3yaxdft=TkTAreP#@g>|Ul`Qew@Y^L@{E?+dDT?&SQmhn{-}rK(lnBz2%L&UcjpbA+ zjhLttTgohFOIb;cFXfi|N`1@uQhvF=)Gz6YrNVNdRDeI}q@1*qak5U%>2vZn1eygIARVthY@o)h}nghBTf-9#UN$^eYs<4 z_wrkrQGE9+oKr>9@2gyBt}Ly&wQ_xVbw#;|N;vgP^=5sg0e@~))f;Yk zcCAr$5i|3OsUM53x%H)H(e`uityC9l&QrCEYqdtT<`1qSTwbj>PR%Jd!`J~tAd)%T zktjWLrn1UnpIC8fOTJn0txChU-T9ht)!~_4QHXJUYh~7tT&}zGe(XXGk+o8^GV9im zxY8&^)>i2@Y88at6+GE2*%gH%D$7+Yk@hDBUliXn_%@$};~ER5XF8^1y=XmawXBxy zTAtzAV}@s~+Z$L~j{TzDig@;VWW#7hU1L4!MBJEXUo$tX^|%+^FukZ}pEA6N7oD|0 z*67UB=vK<9HLJCTQ*X@OF#Ys}id&s8&(>AbRYJamA9Yt$b>5FOL2;EpTYS5^y5>i# zODm0-&m1#amPe}EGf%5AWGb2}$)AYYhaYu_&Otc7S(ZL(=LloQIpNkC?N)Zp zJ5r;`$zYgyG*8_LU)5$f_;8Z6C(64J@GQqzOsWUbNYk?x%ynzSShpQ>!@S0cgDK)h z+$g7xV~rWtO#Gtp*DNn`4b4ZqdChzZ_$*yfELzWlm=MN~aA5ohhJMbsn`_I`A{9rr zV$_d;_-jiwKUs0H3~Lp)slCgU=3>*2={2QE!H#v2AR{aBvZ}1E*3?m?H964?8IcJ~ z-2;D>5#67V49)LEY|SW{j#08cZp_Lmc_C4<9ovZjvLj>@_>DSoCxPD>;2x~t1+x?f z?1LS^mvEAGW7Z;LKqwi6z#Yy+Xp$1ycZSrBW?PlR)>XZCPalRRu~ za~L7laLwx=I0wcAxfkr~M%l)11iumdM)4cPZw$XN{KoMccN2Q`U$a*qMO+ebN&Kep zo5F7zziIqt@SBmm==(L6oRtu;KteeQ#ez_ugyN1NX?Y1Hg0y~zbrjrz^+9}xt{V%s zlNvGb8`-cBGrVEI9YHw0VQ#Rd(ID>_;?S;z=nnbD-GPNcr1!sMstG)^3;y(mIcqj7 zCyg+lM}M|^e%6*2^gEeF3s0AVr-5zy=@H}m(ZceC^d#$~q+h#H)@~;yPy9$wK4`4_ zvtc3sWaFI78wdt3B-}lqBl3@8ah%1M>1?q(MWNvZQeg5LDZqAil{04swIg>sEhFXc zMTz@R&OV47eNw`{_5B-0Q2GH`Yy&fXYPoVHz{ghxF1pbnrc>w>)|Jyo&$(za^cMZ z1I{04nP4F3`U1$aU|}GEAsBFDJ>nP}=A8Y5)n{p zS-)EIBTv@OJ>$pEs+HB1HTQ<)TgUEJcc3D0n@F>^G~0y&sZ&e!rdwTEUVZRGY!6X4 zW?076gW=LTRxZ~Y=t=q5sil=_WvTffJmE82)*U)Kv92QY!cghlqhx38Q$>k2co;qUv5g zmI&$Pr!IkB7xb762yaF`V5;Dm5~&_${#0XaIhZDX#=)4CSJlde^3t*&pR2imb>*iZ zCD zoAelF4qPi?;fudPaT8V!vG}ucR?HeUV`l1GF}o1_M$A#9G0u9;icI{m^;#^RGc)F> zHEA9*3wFZ1+J8C}XTtI9nd_Wx6cR|KCaFXTWdVHfnxzKmTd*1_i2=)Bfb>Y25n*r} zxX}%Jb6DgQJk2{A8ncxs2gmX{rw}lCubNR@3Z4 z^Qz?9G;jU1tYJ(I2U8FaG;?qaEA?8+0uk)InSbr-V7trRnc36u!v)5#^qQm(M4xjI zdjMV#dn}4Q1g;kNs{^6tV-Su!`(xHD=pb@3M5JxwA<=or1+=hc1Wb!j5lhrvs7D=x zKqU zM)d7@{HP0b?xEw*nTLZU8A`4+5$L+QmzF*R(Sn~H5Y1yr*H*E!k>oX1Qz$HuwDWe# z%-JdH>R5Y-H|L(IvY+8+5FX0XGSL4C+{-i9A%LSV*wA2_TZ@)DgpH5o*a$=L88bE@ z>53n+trs0PS~2N=)Qb&*f3@N$A&Tt~LU;~(1Z<9sg_si^F*fX00(p^xRNAngjkJ=j zR4d)ect~H2s870y^`sZ`l3vP7dzm?iL!CTsDlCt}!%OD*ds|tQoQAM+#mnOPsQ8S9 zj1yaoh~He#>Xu-t=CG}R*wo5-IXsnFG+KRL-+CXncOt&zA+5_a_OddC^O)NC#w_ym zW4ad{8-45V&9gADF1g&n^&u}2KJ7$DA?{)kK#apP1uwro;+S4Pa*cWgwn3M+KE~J$ zYXiEO#$a#By4-fJKEElo)9TNgpv1+mf~wI_;XZP8km?vt z#Aih8d4)^E*R0x87J;CQHlc6v#cbs_PfPQ_EmZwt76Pl1YbeWI(aR^x0b2t6eYPr2$Kf=~! zR=d`!A1qwGD>LhUx>;FXU8)HeZU>kO{e%mCuT#--KPl1xmi8q-rYqZmZDo+`az#1c zfqI*7g=3g0{1t@#P1?g?XJB=&Wg>&-B(~gu0&^JbaTuXP)}VF3j9H^*!RlstL5VW- z)dY(SK56_iEWj`W52hHv0TY@_*z^(vKR{g!+7C{)Bw!$}PPrCfk#LyY2sA3K6#1i$ zX{E5RV*v-0I>GmCOfg^~g)Ki)g;vKmANA81x^i=6)&-?!}$RqSc)Cph5Ct;|Be%>BO#?-wiec$i?nY zs0|KFYWDyp)C>y|Co*Czn$1giDxsfRh+<^ofL=iEESVH`w+q;FLw}V;nW;`{_&h*4 zCwq3oO?xPXJ137j-5>;@Re(x^P>S~C-Q0Q~+MDzMpRE65fI5t7cPQNx0<1130mlct zOm{qXq(%Vn{jCDPYS2w%e~kLiKiC@bhMc&-@^jeVft}2eiC`BU^ko?B>Gy`cey;$C z8}#xr2S-|YfZ-|R1n7Dr?bL{qc`=Sy)DImUsEv{>Wub?0sqe|ws7^&sM#l^%i{C8T zN;-~?fGJSAA$<1a${|;7yIg3)k6rxbFs~Eul(@AdNexQsq~htUmy>nUH{&NS)GEs& z>A#Y~9P?9iYGrLzi~i*0ejW!({H5X#bRDE`;QL8~e3=oJq zRB!fd^stIVvm+b5JOZ-OOSH^6qibWQ7xN=X-n^{2gZnvrJ6qkm+RM4TvV^^p247NZ zyDjexQ!TUDAAsXW2_DVjfNui{{M_kg6KoI~>qnGYQR<`2Cv1=m(N9otpIuv8QY9wH zri`3SOSr-~g)^ocXnD)5zo~gDnW(x(N49hXCclqO)iXdwv^chji)r|irv4}&y+P-1 z(Rqo^57QBt{EPJc2p#DX+1$B+L8|vLnh;vbKxf=$aiuIn5T~^$TIyr4JEEN)a6DHD zKod@VtK8T$CVqZ1;eQet16G#7zAR%+m?>)*`?yJS+#Uo2OPMkAF|f1@e8biR_H8NT zJ7gY(Z>mSm-87^<_&kJa+G6g+{}=cyh@1c-_X$EK1RJ4@wYnV&p;WoK|tTTva%I{XI5`i(CCT9LT#rK|j~nCWumh zpK&)uC-pgI5mCPlMk3`4K;bk?n36z}`fg+`4&5GHQlDqNgrW|VXn6W5#FgR{&T-JS z8Cf#jaPA}L&ZuIg>|tntL{zrNfhI(hQNff^;Lbkl>d>|%??IG=x<=WKA)E>zEh+w__*3G?ISS&_;?IabBmS)Tv)D36uw{P9i#j=| zJYT|k7tIQ#@(rjTJd}m>&mumMAA@)kie{V%U?YrkOh_;Z*|t-{!-+^+&dj$&D#P}X zrM&9K*AtK(^IE0~|jUohgw#({5%V>-0hL9D?M@Z8vISD!w8)Lh>AT zfl~`9%8|0Ex6h|qBi@LU7TK@SDOJ#|zg;5bwSqSqwg;G%a`G?6u@4yo9#SRXODR+X z_}&<5AEQnRy&m_**9V*kkHV#gZidB?DSv5sSD1J#=P+nqc!GbAuXN_ z%A>ERJa+`!`t3>^gW6zxgFW7baonZL^~T-I`h>S@eYaD<*^cR?MXn5M%DiN&pF~Xq zP-f8ADR*Gh&~4Qkw$=nFmlNsr6Rq9egtr^0n}*V15;f`&R5c9ETCGWMi0S%yVDcWR zfim!yy}|W8-lVr@&a9&~P(<7>X%VDdLHr<}TZ|#}P3`w4QR*IVkL1nqxiJ2e@sf4% zN>_^q+2VFSj zowcF{Xc2iEgh#Z=zW5#ZPCFy5eHhOvXB2aA-=HDmI>Nb!nP#gmL5&i*_$1cAe&mWm zfnsAWjkflK8k~!^`Z{Ktv2b9~ZXIYGgD;Knfj-oL{0H>RVV^e4LC%;1VeV<8MSpu7 zgV7!iXP<-*tRI9LO3(STx+UO8XEAESvIiKO@v{N+FW21p6~~XwDM-%GYc?f&fHXAh z&{2mg5&pw7C5=@-0*jZ1+Re5^abUChyjr9`HLF&Z z%Mt;zj3zfL$O@VxgmZQ1Z=zaLPf3%?s!K4y@bls3t9HdLUtUp;KBb|KyLfPCcKI^W z_0WESK9S|ygQ&j9IBsmz&(jehU)bdjGuSKXQ~9PxREq_|TmK>RwZ(L8Igy~^iK9(R zANbtMD+uA-NBt$__H{BMakR6BGm3`mM?{NdF8I;P(&~K0&*{E~I|@Iw)|iFbuQ`ts zrNgXIoFgnP9ta{-ukP3LNm{EoiZ@AqX{F5s3Zc(C4Sjacl1gDtpXG}Wb;!9<;@8@V&?4(wiN{^%QL8%+YHnO zO&L@^`OU1qhwC3jTdC)<@ooOETjct}wT0)mIexOu@imv%9A9&J&G9vt&s;ax!6P=z z?=`N&s^L00SQb9}dWTczT74~WVw&E)V7QS`htn#2-1xZpackDb**eeOJ1GP0l>SXA zgGhmpBQo@*DdOn+I;#~z`KJIU$W1C0bG078yPjANg41)z+=XM&ioyRw9nsN z-Y5_*3Vs~%Mp+j5?xLx~0rt&a7t4$}6X{xSH2~>h0?P5R053;FZrz#y6;j}I)WA?G z8DcH6$JtH)DQL5>@7V3dcNsK9e9;1TA(oS~S>EpTEY$mx=)s8a(EGs?2fRJt8(JzD zr2@YNeiDSI81|CjXk9VjF6PBw@=_@AN$DNSeiCKp@WmM!#%iy(cOA0M*7h%87ben1 z!5I#u4e%~-rQKczrM%oO@8#a|VE?cUHBj0RqzxQ6fN~?4Tl-uo*V~H&l~HdWjv7X~ zq>Zqq^h*&O?u@{JBr+CT`>#zAsK zP)nW%6?F~dq0^9ic6kTALtWAaYbV{lf%si~4$_9LmbBkHh*EcXhXQE>&xP?KZG5Av z#ba!7yS0f0$Uqzenwmn|*p1OX-0s7(x6p^~)_g9kHIOupc!$B^k8F`N{s5B3=bVXF z5%cb_vm25|Q6!Dsn5&bB!<-knVxo12jIRZ%&v=;03wt0}+}W7v>Q5k*B5i8X-Zmc3 zK)Se-Ij|1y6j|iXur#J1wmqUEXWX&A*BQid+<0e}%N&ke{3FbheIDdu98T@OZctK3 zPD^43H^+)hGC8A!O^HbGqkuzwdL#_%XBi~J^7ERT*WKq~`9RAL^$^p(NC)8G#m8>@ zCYt4UHql2ru-+?RY^V9#(v~=yI)HEU5FAlMPPR;d6%3X@esk7>Yq>TQ^2Up8`yDMK zMR(q6Q)zj_)(reA{8}Jrvq>>Qal3o5rCrn)O1s9xb`{vJ^`KoFR+pj_;+tY6B|5BH z{Vtu}>eg;r9R*FPZWtaRLoTdtqsQ3B)OrKff@(!Vd2DNCm%1wxYAK#q{dMP5JC@(tNSL#&Nr|JB4I+JvSF2774`MdgO zbpAP=uh97wIur=iuhOA7sD6#kzo7Fk>HIpKe+35zSZkQ%zrk?FAm=($c#v3KRNn=E z@#rnfej@h$M;7-*IzLB8#J-&*zEUof_{whO!cwi2q{2jd`@1y0ItoXUJl+ff7A$rm zSRTttL_|>}iNyGCSnVh2dGPxu8drpkmsz~hlK(v*Q zAYNzy?8L;B6Sg2PBw+#!<6oKpitwjUWY>*n+HZ_OTK8nK-deI|@ z$$6Td(uRYAAw)CQ(BG8{YPT>~SB`>yO4L?3Sk&!792hE|9|+sfju~tew!{r>iGi9x ztTkTYa6G*Q{u785JmyFX9#3_QOC1BmfN6^m!y+*>Gl13Ie`jgnJk$6UYd3o3B2kCr>&a-sR)A=8CrsGhNYOG50Bpi#pd zwObkJN3&iAd#8+-(Wy*B3St>%D2T~*Vw|YxmYENl?1kJ*W|JjB7c6B2OIpbcFji9v`#jixfJ%cx5z6f>b;ygt*bMbRKOKQ@fWC`` zU>vX~$2h<`qCE~eW~eJ>2;~hsF(>{4N+86++|QD@gv3R|IPB$Ylr;o%MjJ4c)J9`k z&KE(gEtzdWCUQNlxn{Sdh*JQAChh?anX&`GDZ9v9vXVOtti6&)Fl{?fJ1YZSivz2o2g=9hM(WsF-ylAjskF122)4 z=VnJ%{~*HQS{Y&{A@oHk zjD0Ht?O%#ILx_AZ-$GmtZU$Cyd}`1fd@W*K9oS|D70st_#MIYOqxx@jevghYiUH1s z-)FEhH9N8u8OkrPwFKRSncFQaW{WIEnTMYqS*qtU6=Vv?Qf7}Vl@A(Pfyx?o&?}7| zvvf?v&ef+cBS3k|>$&;_BQCl6gkQP(gkQP(gx?sC^YA4|6B=RRj8F=0 z8Hf(YmK7x|dKx)Zna*iCgI|*K$loSm%W-Tr( zfZQkN)e_5+f$qo^YzIw8WEi_RH1x@G#o$k3kwUj0GZbdnnn$+>vCXJwsuYLA&mcT; ze1%i5qt`TbhHlM<_7@`PIQg|Drq(tt99qIzU^onXjBCG0u)c`8j&)r~izK32q+1Qczk5P6>-duWw-pVmlqWohuDTkuj(kqh;<#=_7AZJT>X! zE4vT`0|`uJkFf~*!ZM7ugTpN~feeA^vmc!=lPeVvRZ8Lv@RHbdmHOIVy&|+tbON(h ztPD&2UFfZIlrmCMu$D?`9g^h)n=!BgSb{Ir!K7-XI5`t|O(`3Af)cFx)1}z_>Di@< zTe9B!cqxX<=BHh^l+?d8Jm+QCT7ymY&CUEm5u4w_^>{b>Y}lhb!@-FT?z!N50Ovpx zIME^QkCO|S2go^)!qXy1&!bybX25>1ZC8URdRxRNAnjtE8u2q@iZNRdpQ-#b$=KSv z=XlTVIU!ePyD>lMr*xyvzzF)$E!$_Z5q_o6UbuI~!UGqL0rj|m49Oloyx<|cgc8?q zdW78)I3?%8O=E1+r2_$>$z{}QNZJni&a>NsS%ABY88T9~wF}e;V&RA8F>O%e#Kf|K z2{}wma4qUtyA^LGu#jm?9*1d(7?W>A2~GMEy}mzVsUllVl|eDpk#{zmfQvfp1nN)+ zkh2tBm3e-Mj|zf|!_kmwbD`6hK~9E3hwjJP)8NsY&IHo(+0IPJMHORi-%NOB^(-uD zx1R}w8<`12hTfR~#RwU;?8Sn*VWWmv*u%{y5 zqvd%5e6J6saB^qoz@NpR@^bWJp9Ovvyg4Eg?6PEo#ev4G@34Yn90slhb)3$pk-nK9 z{AWIszm=7eL=mMNL7oGqw*n<_2*i}4RsQ%k{eA6`4D!V0ahlVZE5)kvE7vQwTV|^q z&jQ+G05A%`bI*WJ1R ziS7K#0(=3UofcrQoT4~n4gi^p4I&#{d?9RjCSE>(R}BbU^Hl>0F4dmcbUQtR>B#(9 zh$qO&uPo<2f%&7PL|(7ed*JU{t9K*lM=+2l#68(rt?DF`q8$CZ_0I|Yb5c#CV_NeS z0aHJDBe;OM%xJ2wN+YcWIHUD`I+9;lqbuBzP1|~wT%==3*`|40Py?0pW zeR#gjPPkdaD#WHoTRmcBa$A9$HC*|QDYsGr9MB=GG_Gzi9a-JbbMf}l2Cg7-HA^{h zS>c^>@ZOiMa)MA$4z4ZYxhQp7Sm)5%U^v4PGl+w&Ct8-mCC2WSLB-rvVh{>S1ZMEU zAK#~ecPu1;H@Ryzbun_@3VD-gAUYF$Td#%S7R3vJP!umDx2S~kS|!X&x}geKG#$|N zc5vkp7QKm4K90E0gv(7UdwWJHShd@!Uy4cfR!+iB1@|)eHac&i^L9G-!SU^@_0=1F zw*Xfqd%k=(%zx#)>h6-f2ZfepCzx;|OgPa=ILU;QVZzDH-J|&UDRwV|ej3(&5FhWu zY0-Vw)xqt$Su}4h4{*_aUq~(J0tOUgZXY0>#g)q1j)25k*N+z7lwT|k5 zaHl7x%tX$=4$$d2De8nXAfnbJ#Any|0<05IPKoEFx}OhigK;-F_7pI8!ngvA&gJC< z5&e60sC!6nf_W5(*rFzPZ9tYk>zD@}Lx83!jeXEue|snhIVmgw(h|-la7h}Mw*?t_ zgYo|{GOBlQjCX>>uY`+SLt@V$-V74cw+e~sA(@wSPSJUQ&L(Uoc>Us5@Ol@(Yr6)$ z9(Zv9eKzbDz-vM=W(#-`VfgvB6q$?JJ2stpvDRJg7JPi>gHmQ zPZ-9{YG|gYcGcQDxbmzhZbhC+idHX-=W7lUZVIPW2_9-gi-y}J^6Ijm!Tuwn>xHFP zhH?uuW!#k{aaW_GDNCZwx^+Q|MIRW4=$X`Q^HO-ePc&yZ#e<$e2#bPtL8QFOga z)peS2qT5Pp-4@-9N~< z6L9b@KtE@@AA$iV^Z=jj|u7AL7Dx3+XpG)>A0afYOLZ2sAwA@d3?)X-P5+ z9RXe;qA$+tjci9va1(72r{S-%aL#tvUL^EdU`^ehH?*u^YXMDly>XH|lr77Pvy#O& z{|cO&FE1=J%)!pl0K(X@mSu~A--Mflju=Z1FROq{I|9UFTOzU4$97^f?Y!NwS;X>{ z8}cHMGVUJ+8=n4de*OwuTrBA2`v}KWlnou;Eoz4kAyd~G{WOE8;b5P7vf4Re6W*=& z#exK~Vctol|N9)E_T~iY!N76|?3#Z$Kx!AIyH-9|did#wCxP!7e4A5fKKXQ@AK+@ z=F0@7N-kJ+5^tnzv;asWHD^&vLNxNIU6-aq^y@GXB@4c*^&IFCB(;>6m6LDi`tdw& ze+icG9vR*SfEMSWX-@;9P3ZOISM(ftt>?HA+pOp4*!f|(%ZRm4OqIiXfmpvb@bbgu>a6)x+_W`4r zXz`_THdM^ghMS`X;kSbtoUgDpFrL#|MH~gnEWsUK!2-PB0r)Qph=g9h?;_D;{mLLs zqcmBjdg;nI?z9@mNK&n3&YO`o$=KZX(z_Ae_N;ctV_G(6l;whwfMLb(&NMuPOxw{A z786?!X)~lI&NlwEQMenMJ1E>8dvg?qw)as`B?RDe?E(BY0C@fl0`Ptg%}oKA*Kyla zZcl@m>TO0~AeS8=BC!~jO#v3ewn(rz2E+`pSWhFBcL1X#>`#9}^2 zOhuW3#k%aQ8xsWXDisLaQ|9X+aO~SfVD%UWd<)b*!J*#{Z?}P3^*A#h=JFP&eZKNm z4(UF`8<3g~bpd1BRHFlm!d=59bW5nn*}0E31Y*^IV$9Af`W8?zFkF?_8gCJ%P?Z*@ zCgIr(i#&Q0{VfFS@VG6iVQ_}j5g3$%w+%eK<9_2@*dM-(I5h2(=CtR#dN<4KP+OD` zLULC0ZWa K&yl99exH_QZQ$hagis5QJ|>gp?>GDOz|Jc0iH$um`tDktcg7GThUm zTc8Q6dS4e!7=BYU5vYf}{W>H$xdTb`9O2ucNgR_(Y8wW4!7*(S$37SEWRE@8|AmfM7aWwet z)cZ)^ZAR=ud&G|PG`c<0cOJ5*Sfnf}VcQ!Vq$e47)8YE?mf;%SzT%z7ixr>m8ZU<5 z)Oe-!nc(Y;)&X>IGYV56<)L?g#V`vBMbenY;-^UvA(EJD^59TykWxDdqBV9b6hvcy ztI?+&bv^|x`XHcopq;>fr#A95^_7G)J<>Et9XDsjAt#)T#pAbh7U7mwmx z?U^GWz|F+S9aqo~fJ%A=9qe?uJq@=mAbF|vZK?SO!#;=`k|iw5btjJhy{NUl4*4%@D8z~&JFrdvdz246C~7Z5 z5<5`s;_?)(ykNn>4Ucw=4YUrhr|lqVk^(Ep#?Rx>2miE0eUs_|jB1pIA5i*g6|opO zd+|P)m+>-{sExbg@Dd{2ScKtASDfVA6vr2s_SV~5{?^jvwE;b42YI?mmORP8k`w<3 zX~V)g3Dvu_A?%qGL@dtJEw_Wj#tua4y|PXV5&S93GIY?&Lq4Zxc$y~ zYz(<1LYu9t@at1EIWXI_`S6hXcSG9YebRX8aaTJwzdZg%d|Oav2le|;cT~c1314SD zl&Oi#iCv0uRRnuJ$RnROg3a-BPrME=yC|Ddjs7n|TC?rqKkVpDdD}pnmyG4$85ef( z<{r623mX>126nRWrRBL0im{nTTt?#J0?#;mmAI_%RotHAp!BQ-bYBK_0KalSTLzVP z25~Ngnqje~EpeF<|M@}TN8Yulg%8rbp=G?*FGiLJi#e0(#sGwQn!ecsZ>5 z_HlzRghdu}T0)xzIW#xuY;&jC`RC|#gtH2RWCXIhg)vI2x+F27o)C`Q(kAN!h~yDAKf-=TQW@8315?st0)Ey8l&xK7bCT`^ji=eX|(tMG< z2zIW#gOmwdP9VH>P$4sMj5eq!&Rpft*#<#=iFFEI5kW@78aKm-s`WnKQDJgSKNmI} z@Q_Z#6yu7=`N9PK!f3qVfZRhNo1s{lej&$K=o3DXkpEbBtkDozy)+O8kA?rX5dS40 z)$!}Anz&?x0!oE;X8DhX$M8><ZH>b$u)qCg^>D)=@9y<5J@nb+9+zZ#Y z%!k-d(fU%;;Ku`jQO`!%TgwLWZe|ljn`|0HebdX{{{*lESxT+ f3;U2~vT(Str*KC;RVe0T`Di|lznJ_L@<;zK{bwH9 literal 0 HcmV?d00001 diff --git a/captioning/models/__pycache__/BertCapModel.cpython-38.pyc b/captioning/models/__pycache__/BertCapModel.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4dd920f7988c321bd8d94747af49238bfa8b762a GIT binary patch literal 3941 zcmZ`+TXQ2v6`q@<(O8!4wY_WaZo-6XM2HkZxP>GnSzwok7^<=qWC%5kXIj#%=c07C zHfz2(7#pra+F#bWE<%n!?_^nZFnb_Mj*p0+HHcjIrqp*HQJFH|o`EIbA?*u#f z&ETeK+%=L;vijIaR$1%8tz$d5jgfA$hLJV4gZ>Li<3ppjKKqmRKI8IEH2I)RSmvj~ zpNg~?`Qzzmgy%4dnJ;-%h+)Zd#{GTF_H#Wq624%cypCEA^gZwXnE9NEX(nl>qLe-+ zjmmOF&yuIuGRl_s>w9Ogy3sS$TC^|9Y${op=97|347o}AFcoQ8pxvGDw2@0e=S`*X#T6r1fkvLH6_b(h#Yt+8W3dSQ@@|euHTBsD2&qsOB z#|euh;)y?h|LS~#AMtpcN*2p0XMO*DIz6!C%P|mvC4L?ihrZsVlw%+&CoqY=9*Lc$ zBT$=8K~;PmLK@PnjAQG>Izv1hH(}N5r$8V|NeTAL^2uxj1MEp z2Kl4I0c|<>Wgd;#9Eu2={^XE+G9JV=Ix#e;&pB8QH%#N3_==AzTnNMLnoay>H!fSf zzc@qOpjq=mZDOxGsA31AGES(PFpYg`!6;VcoH}r$IeX)Nbin+y&@PzpGG;=+72<#; zzTmMQmJuH@NkOdxM)Z4@YSyMw&DwUo2ETz1DI05)Ek{zhXw~p^(bX+}6H{t&izc_= z|4#vxZ=zEl(^@wz!+Jiue&v|+iSYuwSfJfcP|&BmhDK#PAm;yW2r|h;aD6fS#YZBvE*#U$ zSzhwP`cz-R3;q&`E9bk7u>q>+LKt*D`aN@O>)GedSOcKfra&s9oFS0BFa(;X8MQlM z_{lWN=5JbIn3QoCYFP8HVsmXvP9IdO^GJ$WmSM_;PJ~@fP|*=f4Q+VWbevApZMcpe zk1>9!5ca8vKEyf2bu^hRh>i%2h|Wt^i0Cx2HG!B_)me_jGurFSut*>f&nOuG`PaWt z`P)z9HhPPKD}>QO)o?7(khmmN4Fp@OYF2LLB5i|TpR|wK(m((<;Q7TH8*jk4PrM@! zOx8X&k5+2tJ?c~|f3zyd(83!e_oG#5S1Z-3bg+YUY#rN2-Kq=4-PzzC%fN1L`gcm? z89t4XW#=h*$jjUxAL8vK;_#JZT=Y5d1pMbcEaPZD6zPnql?A}4vr$a*1>&;k5Np&X z4PBsZ%pye81~M?iEg(sO%6gy8`929RC*>50mzVqM_QgW#i}2Fh8+r67oRr9i6q-6% z6Y!%@YZ8iw7fPOOK=1oKkAD|W{yK^8k$8i|_euPK#1BbOlH)&y2)q>EtLgm`VBlJE zuMwhZ5~|=QO&-=B{~8809zi1xSWu|u0T^o}mO!ehY?U+QwYl2jSF{&xYM8VBGMxH3 z*p_01W&O7@m^~Bk^e-(%yb_A|3WV}5te}z331q#snrhobu-YoBO$=K%8+##0LE(YJ zBB9kEQ%({Nhsqpdr1nVQVOkjLOH1Hk?z~;+xwlCLq7T*3%3eLLcX1A~uy_~3ut9~F zfB$K=KR0Gt|89D&=~|A_G~M}p_nG6&ZhVQQVr@zK0t&V5c@>nCFOq*{p5pFH$(|EJ zK<1PcIPtZnot`z%QRPH(O)Z*fG`~=`{Rnm8we3r~oM+hSGx^WEf z99Vp(GEWffr{+7!hL~PAhA#f)mg0N&s z7lDUq%zEwGxK=9F$}kRxEJDhij8z9gf8OKYpv4WXq;mA-)jAZ)21Q(jm=l+reT?ba z%ZAf2JC<*H08H0(o?HI|U^Xr}iRSm3{5zzh7Ozr^QUIrv#$SWzwK!ci__s*BOyV91 z@;(npe4E4hj!ssbg!5^o4-&n#pIC8CC91(0!GIwCY(|er;7+DSaM2DoB$bg zmj5tlCefUt8v4Hy@6tr_qNWqo{6MGdnqYLWT%7+rK$QDdo!Q@^gX>$j(}YXl6c_Ru VB*k>c+<}9(aB=9GTh_H({|CLn@pu3L literal 0 HcmV?d00001 diff --git a/captioning/models/__pycache__/CaptionModel.cpython-38.pyc b/captioning/models/__pycache__/CaptionModel.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dfbc99f34879c2ab76cade88013eaeacfc15baa5 GIT binary patch literal 14779 zcmds8dypJQd7ti?-JN~j-rn8bQ?e|}veyVn7(+}rWch&!U=9b{glwz~dpmP?d$+Ia znUkeswlGLWt5oD1&WjMo*~=3pgulp(O65^S3jRnXsR~6UQ~4tWsUrF}`7165^831H zAG0S3VUqljQ_Z(M-96o3fBp5>-%HPBvndUK|CBsj`NyBqv~SbP=%0a?vv}MtfUViA zt~FShF)DMEd6~1>1N=D`jo~8?D5GiOQVCp5d-!4?}3|azRx?l&3wKU7YjbO*?Z)YZQa*rv2Y(PFEfzMSP2ZmCP=Kc#0wQ+2h1w* zKwo#fAOTSEJW(>l1V#>widc7p_!hdYyJ8wuf{3ZW6qRNG4&iZ40F7ms$>MBxI3sOMokw0N4vJr>inD`uY+Ix)oxowyHXnAbW9KYHCsN|v*B4&}5gPsnmcmh-Zl^;nO~_es03&-Ym;=O??d1??*D#y534H?Lh~C>dAv zE9?rtqOa-pls$df^kUuECH91NiFGDa3otkAnX$I~;H%Frw zrSW>K%G^wMW3!r_n8VmlXnXoOyEENiE#5xTnenInnXMScf6bq!{{D>Wzr|$Tls|*r zD*EX?E|#e8oMz{?4Ea{VUktGyXR zOrvifhO`1MF|4FWYxPrIekuMu?-V<;ejIhPU9M=L*6Qbpn*OAO1%j8%=M8GZC};yj z8z9=2HSDD1wLNxjx-%D!FgLmjv;)9rX^VH}RXfhXACm#-F=o%@OR`-RP&**zboL z`{=7iXVK3Maq}1Ldq4qBzsp+J@aHfxXwp0CFA`3B`t~;l?``ERxeUELw zhG!45#ehW6?R&#BIgkkUanxWXQq~;u4?%X{w^EHmVD|q=fYQxHlGyz^fX{t>PJ`fN zJ#C)^CoYD0Z4-Aa4yNhjde}0=U!s-!rq581jUYkl;gXOW_AG>_=BLIEX`G1ANu!q( zedkVeGJdjW;A%;S43v>u8p9>T&=U7eTgM^8(cNC#KI_LZFBf0L6_oUiX$=^%=Clv; z^%7uri1L0ddL-7kyrcwO;WxgAKS5VZ$qZuEdaLQ+#@?v3ogmqCUa)o@(Q<>Bs5IA| zAmdhXcR9kU*tQiUcbe|@j^pe)fk|EQsawtpr|ncc%X6A;OF)CjtvZ#4Gf>ga#t<){A)^LK{PP4MJZVkRp ztyaA1hUM0FopQXoA@NJMg>&AjZnSDud>~(Jrz#aJH&BiB$||%3k;8(AcWjWQ1r4w` zjz)I@K*L>YvK%*A0r&0#o)akN_%z>*>KWl0D^N*JE=#1{GbbZsR^{zYW8#ml`6Qla z@VFm?V&ca)B_Dx5@~lS*?@uj9$!7VYHN*g%lZ^+s3{Q(U%6Ng*;}1yVVGOraxxChJ!oN6I{8V zb+YtZ)Nr9^F*bM+M>pYT0mE4_4{)5u>^)MOaFMH%flKV;zR?5k`dQVt2Y%gW#GlmD zH`MC!J^{g!qP8vGOYduY%zZ$v_NHHV3_lhbRpNoMV>EUr1}rna;pfzx>sUQ%4Nys9 zg4se^<6@*5s5qb-sOYB$vwuO(&Lm#LeaYjb58fgeWIO-@e~3 zKq7$_z&x==#)r8gPq|nGr^!?PtRV3gQFlo(CdE;5V;+Td&DYME9oP?_<=YrVq){nP z%XJ>`lcxpg(_?CCkXo+ST(8<{w9nMg?2-Z<*SNO(NJLY;=}A$Qp3@#4p5bVvydd>A zNC-#CrQD;g>j<(+-YY~)h`XsZChY+j6)>!Vcty#pgmlJUq5kbI*B&-DyCoLh*ORTj<-=c?h;uCA3X69D_SyrP!LX| zb>6X7E4Jl;#!XmS8CavZ7+`KbxAMUthfQAhHmtVOtkk`W!NG%(z$EhWKZ1fh7;_!w zL0ojTE)K}P-s4>eICW4;OioE;&HlPaWd6a~@aK z@FEfqqWY6~hR5f`_b21rxqjLi^0T43DA;Y$`Vt{C5ewwiA>J1MZJis!J-bJNis>INk*$gvp7T(LV1M7 za@;p${?Y&ZmFU+P|GQgA6QV`7Cyzs)=c)rY7Bb zz}0krNV*)os=;faUE#1sh_WU*&B$uJ4WC6uI?*5L*iA>malY+Xbkz^`nRY{BHQkZY z42lGdm@X8GQNqZr3;CDTAin1DS->^}wlzkXJv0n9aQCq{$;P7`tt*axgl3E@{~oxA zj#7w+SVvik8`Q5+d2pyA*hqG|&WFQ^O&WyD#q__IW8M7wTW{@9uBr`zOXyBFHnJIW zz(;OIo;E{n1U6%QBSdusZMoj)QFsN-gDp^JsDUcATBcT8r;5QcqhFAk_`31dT5~Ok zNfjl~aZTR%H^{l`Mxeur3u1IG+;T#hW9@*o#YKDtN3L$y8V3k9kgvzlNiOg(2t}u0 zs|KJfQL)Q0gQ<3Qd*H-q0IL*Dz5Vt zxe&?JSPeU%Dw)g5Iz97u`0zjQXuD_MJd;a9_Y9G3%elCM-#DqRjxDzsZgaM(bcDPO zyIQJvuCk!VP8}gLw&mwgCAS>cB=M*C3<0_jN=&>&<-ztxzK)QJ$V8hbNkt03+`Jr- z-Dm4?1C8)z39EpLdrQ&&_xOB(W9hh<#k(NYB$x;_FKczDR<{u*AXQ%}5mE~`=z0s% z^6XnO>=9($b{!!EF8tsb#Z?*#io`NP!f%Z-iQ?@4awoEFIb=EDl%%0gl6#RkP8< zX}MGo52@K|^}40*J-Vha1$124CvB|T4hjec43vIL-faPUGDzcyS#E31Yg8_j4l8Zy z)s2$1|0bVrL*1U^9Eg9M%;aF)P*1fC}F;{-lJfc8<`0}#v}Fjqp?jB4oLdt|hP z4QXX?C5W9Iv^gb_e=-P%gDrgNiNr7?zC}q=sxQ8M>5I z3melU2b`6%CMZL(2 z2KC9Y1zzNH_&v-`KF!X+h!Ms6;hMx-%q6z!#@pE4v3DLX)7d`q%LNu=^S9BPn34;k? zh=wEBk?MzN`fl|(TwuL4IrZV8DsIAu=H((mnU%1o3iGU}Rs>}xdU)_s zT#x}6E`t6DR2K~fL;^7EQhX46LC}LjTH%^*7JQ-54Y~dg1YbIH*d^s7`)WV95{`yY z1k_q%b5RCY>SMb`ft80xLleI2-&^VjPoP@ByL#K&L!kKKK@epgJBl|MKv{?cP!_@f z3Vsnl;WBt~m%l*46Z=>gI+=$jPu~tPQY*lCA#ZdRv5q7D;vEG>?%q5)5*WGHK0Xo{ zc?E%y%TfvB_I+dLfP06!oNoBelD`BW;{DN(2nneK9=cFmQlrO1B1i*Jp^m4>o$Z^@ z(BkCKM!gDdJP4O`zkCewV>qVKC+Vl>Nt-jk03$Wa+V=AH>Fqf`5jrAaOiMeG^n+4H zU1`bwowPP`9m!iz>OUoP4JM)igf&6BqrbS<<%rkQk~SPC7@eIUbEjs!Oa7x zkU;-tInH+99+B>?x~<%Cu5+is08)lX7n1b+60AM)E|FdeCp9_f%}#QiocJ7cHsQc# z`wUx?_Bq_~Na;Y>=bbj|WFR)vGQhObfm7bk$nsgZ=uNO-mX1I#hJe)sT*HXG`x9_* z=MfZ$gYIEFe&NohR@-c#r$j;KYCS zCx;GpKeZOa@yM-&j&(BBOGdz|0o#_QM8HHurKS$Ius(#3HpO_IL21Z>pG4goRR1Et z?xT0ca;q}%Hs)1VW{3C`#);t8H%28Fj=D+?_;H4=lJSG`AO`&|o`I^r`}hxmY2U9U z{s2RYFB13?0kSqrDXGDWccL8VjapNDj+#g$h*t^x8iC6Mq^x)ZrP8eUJ*xP90%r-_ zOW;ugj}dsBz!L3<84`!Il(CAWJXkBJBJi0APfgYzBX zN|f;?HQpoa2+#8qvL=U~V}Z+C%BunMjMx??rd5>t5S7TFRHnvQ5c%%6r1QrB;5lJam+# z3DZ9zAb0yFh@T!s{PlPLpu9TbIqh#dxk}41N-ea+_nR<6Bw+IY$9*kbYQiDXM|tvX zqXgf5@`oV!DD5|BB2tXS7J)heI;Jw#6PZ$cg{m6_WdA0WS_E$1Ke9!1WB+e);@d|F z6PY!_7(@R5KXZPIHjyW~jSr1tJ5JMW>L1;MXn#Q+Tml|}9RlYGkfKqWdjsiFGJ9ed zuMtwv1`OoZkQCA}4RJ8$l!gu2_vhcaQ~D0n!X_3kUIZ$;?|KvbJE9J*h)Wnnxu3)P z^fOcur8P}AxKB&>3|+aROP!?#b`_;S->tR9pHdxpXvNPGpnF@qOyF(;A0_ZH0zXGU z-cfs0`YeH02>d*OUm);{1b&Ht+^b)v(&q^L3W3iPkgDDlDh+kO*XZ@v3G@I;hyJI% zsbZhT{tW`ZN#M5#+)IEIgrUY+o){>GE@hfEq^c)=hdTW(fiDmsT0>8mr!sNW&~rcH z^_&<}+Wr9@UD_&5H#6o#H}QPxb8L#Z5U=N%B|1vqvVXN%>od zr0Fo06Vjwk48}7%Na^k&mlC7KF$d*{85r|P>c6O61s=VGo!Xa4*Pe-)kReCs7yihk zjodc_C~s-vD>mOkW}0}uX)vw%nuqjo=qudLlA-KrNZ(G`IlNC?8)VucNt^QFvVPW2 zZXtsXm}T+g{G2jz0mB?T-X1(<{il5{_eEROvhJuidv}%OX#bBmA&!SQL;lwbxpRaqjDk({%3NV-xQoplO_zy2B+i|NZ@k)FGbqEtCwj8I8II*Bx7pVTV`uPj`S&}yZ%=yX%BnIQZ z0R;x-=miFnlGnw%&_$*Cr4Wr0p8G=0+fWw=Q6or%0W;+jh*x(S6p?%nH6~wFcI-g;7-qG058RM~K=gVvxu(3(}7xUvIr7 zYS?i~0GVT1P5=oC zPzEuXGF{H77%_YWlt7&e-)hKht0MP0VX0N)e;-xct#S-0kJc78r_mPQBwP|(jpSqq%tXM9u9a!jPvalC z8W0X+yN7PeSdqg)a)1uU<9OU0fI9m)5RlLPv#p15W0~~lP+s|t(?klLi~g{4r3c^z zl+}?Q>Ms9^y3f;kq_i2oq{c>rdx`XwAZ1yIJnujpSV6|Jws$IZ6=s(It4#jSEODJC z`!fRHCP4E9+3>%#RUJfsM^nDYc2>H53Z)r5E>(e+2{UefQhOVoBj)|`Kg*S-X#TYw z%7(-sgtw4qjA*2EBB>{l+7y^F{qWjF>HiHIJL7Z! literal 0 HcmV?d00001 diff --git a/captioning/models/__pycache__/FCModel.cpython-38.pyc b/captioning/models/__pycache__/FCModel.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c421418672e8186ebe2b64b32b6243371e27ad98 GIT binary patch literal 6365 zcma)=%a0>R9ml(>A8xlF<9Y1t%x1H(qA0`=c0vddpkK=}Tu?VU#w#Ee{B^{eXYdi;K$ z->>TJTFo=?_m}dO;O`F`#^0!L`p-h)S^UxukqASup)q1@#wgE`=WVX@Ch}(6)Ol-Y zwJp?JLwn@3orSD4a=5{ak=rh7i9Ph*GK3>a!^)`Iu0rMrcUT+M+BIgpY>2Y(?i#`i zok9JM*_<(;!Xz1E=26B_gL|+J;|bB3Q_g>U`*AoCn6knsOQsv=Aw{3f=p<%nK0V?eWS|+ z6a22*HiapyyGGl3->{6fjnUc4z47K-uRov2u)|MvMR^-P>M<2aG9zQTafi?N94nNw zftee=iKm69g{O_DojJ2oZVXI;wgYR!m~%+oh15DDEko*|l|4sWzak8bhVR~^kG87H zPAB6~$}&pI_5CQ0GT&G0Q9Pbxei(NW5yd?}jgCX*$~Z1^RUwnHKbFa!KO89@$I7`8 z#bF?o89muoW*RB%)y405{L+6S$&DE{ftGH@axU05RDIi+vx~;m z%*`3cMl*q}8JO4vb82PQ%+9Sjn_9V5)KVR4Af+;u782(~BJ2%(Nw+JZ2WQdqqTixs zA#!J4y;)h%jLOBc%cnfA zPg{7Hv$v^w`lOYf2Sw>-Iqhg?UZoH&p&Ig+%@%GMjBN@>&^C6Q3( zD7dApEQtH^0@ar%ae6Qb!{e}3l^03EZ)H%w4Ex=#2g4zBk901bjM8>((Tm^dPvZS{ zDT@6b+Nnx6k-ac03NDmN=h{`&XEKOWRFB$D9NxQB-@gK|FVs-8zf84mQIg1xt4p~ebj6zt9=lf8^$&m6@ z-#?fH!=grOqkNpcSX*d(C&X<1Q{$l_NE(#3!x}OvEw;<56)owYaOQ8RvS}OQ|HU6+ z3kMvsMMJove3!Ky;fV?)B~gluE*Di%gVbG=>S6_@vZj^?by!uUgem);XbYhl{nk1(e}hn}!DsINW4Vx8y;~P#1p@gEO33(?cbIg7JA<`;aS1Nm$^rB2bvBBe`FzWTQRBlo?T356xYlWdPe;vOnk8FMUM>bAcr_uK;GJ;L9*aKt027Cq&R3Q+;!_$HfY{(qJ=kWfz ze6G!6v3no=j{g4ijHXJfB(I@g2+xpU!F0 zm}*XoV9(9-W#Z*|*_(xy@m*yN(`?jo^qk}~_<(#0i7Ln8k$)V@B$eMHA*oc$EM{r$ zrN>3nbEK;1Y4jr@!uVypkkY0b*j-{(<1@=VUOzM8qUCOfT>SEnw2z#;p75vG_nghJ z!*B?8ZVnK)Y{PY2GNv$|8HW>b;Lg~z)HBGz2)=L0>zO&T;B24+Qza zx6B%Vhb!!DV*uwkUCC>+H6W)B`Fg%W-!A%?ZIE=%=ZIwEt1{6 zkYR-mevfszkk@EkklW0?yfo+U@M&XO$}SGJAluFxw5G!OHKe&ayF_JX9K3s{OS{mopxVjiORb+jMA)=jKX1<47~D91WT~6} z!f#%^HUwga>2+k5ue2USa7fIw(|!QdzJj-9iTbU}a7;wa!Z=MN$WULj{xBN~rzU`~ZXfn*A zI2lF3kl0H#2wB5Urokn@ORuq)R0VYQ)1;dr#wn=f1mkfSBkJh{nesYGoJGA!GD%yj zg>AbF$B?RWkm2xw8KoFtcuaE9s0$LPDIB8*8Di{w{G zP)UQ)7y$>lnEM6lNpZLiZ&8a`sIm^#$1*&WU&EW{Ac`nvu`@qFWlEeffR;M`Eshw3 zZvi!ReuZs8T4k4bon0~;>`}hOA7c*lcmvd}o^~&b?#@KkMBDrB^ibW98Iw` z_^M68n(~ZOj11TyjwTqu6l~z3h)~VlLcd={7cKi_U*t^)Zm&o@3pTypy4l{9gOL|luAtJ!A5 z#F#L_0S6N!mt$Qt_F4LPW(xZbdza~Bgn@9$0gX!F8wK+SmPoQ}k3)q^XXd*`R?Zz_ zunvyAW$ET$!DC}q$%%)M3YXg4yhUUMhq?SCYO8Q>L~VSZh%#pgx@bh;-n1G&ik22y zDuiY$vj+7vTLoijLc7N6mg1h@jI)StLg>ItiOH3B*;jiko|3Q-r}VYaT^ezZu2N?j)PYuo-om zaTtnFe2&8=2BwN!A}+#^=>3_?=4MdNp`Jl0xe+r z(l80KMa2)P!V^HTPh&|7z-x#*z;F-?9c*ia%eSbp1#b7g)Vx)t*j(H>bo#2S4}aq% z4O@-E?Dx=x>`)R?qNgS)*QG>Hk_dZa0RqML3&^sp;DpA8X62Y(IJ@FF;GJ66pP@lY zgfRd|k8jfH(F71REM4|6;E7WuAi0UUE3C;+!qd~;oMFcXdL!?r*|CYp{0uvYBU|<&fi30ez-8q`72GEcO0ZKC zaR~!>wG48^XgEfh(+Ze^R>Hd<>*W-c2^Vbm82#Zb^j8Bfs+t#nM$Z+n1z#an=t!xK zc6B0xCI_M}Tul%!R;J~wF~D&SG+EKSsOZ=SRKXXCiyZNji};Bj)G;@%%TM7}Ydvoa zz$e|yDk8o*BDQMr#c4g>14$g+_6@YXKy6XC339E)TO`5nB1tI*Th>6fm9J5s6p0Nk zVYKy}A}@;~!RhMRZ|K?D@`Ga5808=GI@ZLRuE*7Uy~sanOp(jiXP0U2xQf9zb8kU0 ztQN(v;M{64cJLOV7oyvfn)TU(h<$0LKPcAV-~&*!vZP#Xp^Bq0>nB1vJqg0xmRi}# z=c-B;ju1!$M4%o7`1{v0I9c6AMouY#Z=!osTj*1+4jc&yI&f@Vk>mp9yOd}Wyh6E` zC?RXt43IhH2oaJDU6Ng&n6kV@MLPNE0P38hmXA=yk0|*u5=2nr3B{_1@+VZ}S)v>f z9!8xIa|?*hszl@TgH*na7OjmhxA;5Mt6qusD5q^-#{vWZ`EyEsNeP8hMWk@w` z*Ciojj5f-f#9aiOX&>kJb4(~LjQJz9OX);OIH3QAzCfZYq)q%No>~H715p&>Cjg^? zE2cGEH4!%I+(pUb$f0}*5|9xS(r7xdTe{=KY3p zPf&6)@@*>WgEU=$JdU5fGsPe`T*r02N8NJ$vc4R_RS>SY3O|OsDNtX0vAXE|cOA4HngBxAksOg)M;)dpA grRm?R+iSg%GVzHFP~-mEdT%j literal 0 HcmV?d00001 diff --git a/captioning/models/__pycache__/M2Transformer.cpython-38.pyc b/captioning/models/__pycache__/M2Transformer.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24df91b5d42ed8644b0b8ec5a28139abc1dc07bb GIT binary patch literal 4100 zcmb_f%aa>N8SkElG(_eikg4!UL=wq1MMxamNpM1v0&hV%Az=tLnN_#6vge`n zjO?sxbbz%hr*J0+YWJ8b{sI0J`a~56YR}v_V18eZh zhVqxjx1z7s8T%VGmP-wdH&OGyf(RBc!G#s`#0o49!5U<0!i(3FQ^BcZBiKkz2d9(u zV4bt+B}g~oGs)TDY;rC*mz)pIC!4`0XCJZj496av$qS;n&pG=QORbM-rB=L^TnsKI zTfr7*?=sOAD-W4ikz4zAa7nm#SZ6gl%^7RHpXP;{^ovoJ`bFkXa_J{B-<2XvWRj`9 zuuxH&4>FZV<-f376yy9_uNSG$NB6s#8up@{ym#fbD_6UhU%h(ul}@uc9*zAG7LDTA zAC8I_7gNCRPIkKeEa|C9R*a%Fo@Bjhua~R6US8U(3;AYqJ1gY1=BBr5Rlx3;Ov)N&ms_-W&d%Vg%t)dSB+n zXc!e(qT4)z_F0GPHC!!=Cxr}0$v9I5x;!zuKgt0W_|>r*rA0WHqz1h0BcMss+P9)H zp?NzKGVVC~LXNrNIO^X65dy&G^0SFd`%-VABb=m91eV9zX2t^*jd4Bgy;T&81zR6{ zqY-`Xw$ajClVJjcqREpRAaV#SkiJGMTr~gM?f7-G1T0YsSz$wBc%J1p*rK_6j_enR)g?yGu{!0+a z0uH~m#98vIhdi*qU;{2Z(RkwmW`XgtQ8xGOS*vW}2=!_2JsHCiuKBmJxySmEpOB6D z1C=HI?jARU#`yS|FXac5)dHbR$>&@P5{hc66f{7IW6!4q@CHe?s%E^uLpgUca@g@+ zk*WS}uV48cx_V}vymw4%kkid1x+kjxJ%=rFx*fAUUVB`#S9zOHpE)+=Hl4it&73~8 z{iG0Y&udN~7p&kV+qVv^k{==#7j}uPu}`@JihjsS7CM4Kna;Lu0Q*oTJ5uyXPm4l4 z)jO!U15$$hnr|CG5L@`?hN}qeslDrQln$XqCmG|EN_`*w9ZOxLVLKZaUvu3E!+sp)c^H0k6m=N_FQMkf zC;2*W^6AC@ahCr;lWuOD#N3?UVd_QnRCllTSG4{~2;4YBuJ{4%Z1lsYn@`!*2=!^@ zMX7^vQO%a`fhpMmE3MK#v_5fW*36mNGq+@SITUIg@=vT;ZDyAo$ruWKWDTqfkUrL` zmG;1ag6-|f{W`YZ497~sEoC^6QGvu0B2~f7g!&$KRNn#V)GKVgLEi}Fm35H(oTXTp zYa5|admvS8tW`PlBpT}{HzaSaRAQas>(=z!PXXuTob5gVvna=a3ea2HcVXH0EOn{0 z4-m%ChJ{~XhZX>02R1--0HbYSv;m|GIp>hgY?3YA16MeQe2rZv$ZHkkvszgLbnf;M zIhEQ1IU;o){fh!oT|225CAzMV)}pb5&C;Ku%W1bCXQ|Y7lOI*QsgJG^A^gv0jZp#9)l5Y@Nn;Eh`~m=lJc*2<+BE@I+x@7h%=Wu9sZp7&H7ay#VY=aXSGOs)8S6z$U56;NMW@k@N$S=f zY#ACjiN4d%lq5gQ%`@F;192a?PtQNaVoCOxk&?DNRO8Km0#iB%z!W%JFhAmL7#)1r z@UH2vmv!OXv+_5AuUpo(81cxrSSG_ zC)XZyP%Tf+rrrSQ@X7?{f1b97ve2t<<+)V!SbRsROc|#$cA#6;navq=lR^_LXISj2 zAJdAYiq5LBk|kd>Zb;7{^&yduh z(^*X7r0XzbbNXFE&|Bpe-f7n3wzbJO-RZMWF{kAGL#7 z`AkuYH+k6ylW&4}^i5fn$IWKD?%7_w+45ROs`HGlhoQ*&)u{#(lW;;saa5VGO8HOy z6hwQ*r@LvYtuzfzFIwHj_e>P)<`U0;qT!!-LcxcNUG1im1Pv=4E6VnYELKqfROEw- z!e4OwW35}g$+}BlW9Btky)ez=l3ZB3=CexS`<$sYBJ?a-(zyYLaKY0DOX4AWlK{z| c192Kg>4?^Cz5$A#)#jU)#V_$1%wXOAH!SW;djJ3c literal 0 HcmV?d00001 diff --git a/captioning/models/__pycache__/ShowTellModel.cpython-38.pyc b/captioning/models/__pycache__/ShowTellModel.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..561a02bc89ee5bf524020c22f500f56807ce8857 GIT binary patch literal 5631 zcma)ATaVku73T0Jin^@!VtaSvIBHV3k&$}SqAgm~b*@etz}Y5n(u*2{AwWe7Zr4ZF$^C#5&cEW?GM?IhZ8H-BVMIq3-jB8!}W3A6PZ_m zaT~eCTioH*2dryzm)9Vv@JgulbzbKUNS#Hk$(K;8eyo{V*M&{XvTwjj)l0SAur!v zUGzu#szvp=2I&QQcTQto%(dbYS->?cfwlc_n5KSjbaZL%gnlY3TqKiZnh6^XopouY z(@7vi6*cL2UKod&=Sg=z>G?Ze8Xg65HH;_I%nRaP!ozq_3LO#0UUqL1NL5mbZaf`( zk$*1`sjSm5-b5rjUNn|&8ti#d5D&6px8Co0{lL%4eky6Iw4@%#(!L$WfiGk=91px; zyc2L)83dW1WkOoh3FaxyULpcny+w^Q8)=n#R+J3FOjcfBSXU=|>dKmfARG*{lvW@O zqP{eeN%kp|RnO~1K8E)`)jofBi~E_sh4a1dXTjF^@ZJ`z4YuAI`-5O{o-x|>$-OKQ zz2R1`JmL^-jp>-DTW1a%^h6UgpeJo3(O8o?XaBmbudwIYMf{uk(Z#dxA8$mDoa^{U zrB2W-%p8Cj6oT%Q9{RQ;TeIw8jP;7@;~-(Ox5 z%8e@mSfxq3Tr+9Xsz_%Vhy6s1J@E`0MU6D9ei%hwM;Fhc)~(V)3qKwN;wq`AdckI%z=P`g-Mk1?maNr#UB1y$630IJG3}ve{cG4rERL`0! zYs!pa$axTV@Ip#wP{XFXFuC=GX&h~QiP@$8cF!W^#gFDd4ynxk7d9qm1vY!mW>#*D zfRr0rZf$BaKtZABR&K9pXsZmgnUiD7cTI6KGYT^YP|fuj%kUQ7$5G(=CG9?YUHgRH zoy@9vRV`tJUPzP8+)r2x2pLsAVd6!!8oQP%5j2^*rYZyK3io)-qDH8VF=}}Y?>W2L zte)44`W&;x=KL);5UsS|>gY>f2WD?*MPshb8u16vYRyE)LcW?W(b#1l#TrTH`W%rmUVpN`vbV=J9X_dn5f}4@orer-wD(u6 z%!Rx`D}&O-+$}EAF65QD{((Mg%_`aYXag;qc%I5z#bs)xKi9ea5E{8tJe{}51^3>; zjIZztcOGc4N2gWU&|b%GC>x%c6JS@jl@}^#enfsCh(kR30zUwj!Nc(rctFSI4xaQP zk^%hVZWEY5#|1~?5BeU3cUEFOAa5F{d($8|3O?1Pe*HzsymS64-5XJuX1!!Qx%obI zOL${zx^{H!?Ai4I2nW94*Kb6K6j6E;h2s}HZ6Ga04l5n{0PQXGAt>B-n8{o<><1C*#nMC>Br)B0$i~6v~OqG|qRZclL&?A%J*k($B{JVF?E+ zgj5y_CT$<_C+5=V`I&TkNt}g)X);Ya%Oxy0i?Ei;D&m!jaH%v9_saU%Pj^%4?)X`6 zSVq1Y#YC(Qs1~AFiYN!m5SS5bQ^`W9A7W?Q;pd#es3Uwv*FzdRDR#+yhD%4Iy zurKYhlk+?49B6as21m-M^JtIBgiJaSNQc~6yP5en%E3bT`-^#z#D!9?}Zl#nKat7R6c%R%_EzUlHyGh`#U=$3qVBEk$ z*)W-692np(OJB$gZrx|UVVOyb1kPv!_$q+X3MAm5rDWz1?SG(U)!ZhSzfYXdJ+9sT zFCbqlYB@nJq}(B$yT2pw2Gr}~&uFb9j#6O2xCBsh1`bOj0$OMF_$o9_1?+6JRJ5q4 zVi_<_GXd8HYbCG*#|1do0m-l=chx30G2(@h;-_d`&zr>twJNYQc@>+ywD%##+05O1 zDL1+MkfOSqS8q_vN4}wP15XX1uW}qz`ue%nI$E96t$V|~QanXh3b1A8xTKC!V_Q~t z0)MQ$@e%FF3HToUItsG14R$yk2kmSarfr<_cAOl%&`z>pAP&NGAqO9g1JUmJ5cK?M z8sN2dKR9R)1E2f7A?iFih|x*V7KzWVLo06Y1Z^JpQPw^Pvtc`o+x=-a6+t`fx8oq- z0e=GM$-flP)A>iIYIO5XOFD6KoVMyFGF^8{y z>1F;E6jGv*8qk)2WQ8@2H7etA*t2-nl>Aw=U14qgJZV1N*%^RYL!Xb?V;eN$48RcB zD0cwFP#L#uxON4+5e!efnp?cGYa=I|214iN+yH@QKx|ww!7nw?Z7R`a*USletjxv* zw))TjOJZQ1N=BS8Qzh;`=A6GLcE}w7NP&LP1NvzIRO$*;KBMb1;0e7Gpb#4aGX!G< zuY(>+O|ZWunnzw$pt3ZpX06dOT5w$^J_yJ`>jG42ih-#Mw}Md}ylw@&PTyOHmHIB# zZ)P?aUTfqMKp++^W5i~0k=M&P%$o5I;Ko+EUFcpSU9_!ZZY%LRN$_ltlvv?$3uK%5 z3iV06aC8~-Z-O5_%}wH{v*oj6DElq(QMnG7)n|DVt7Ohr<9fbYmS5IpDCMig6RBRf9_)v`0M_~h>as#J_xx1cghprWt1%&>Ue(&fR8r`mCH)>Py{R9S zE$SU1V{r+x?&@*%`tjFF+yUH^vN|^1W~t>* tvt%HAF3W3*aHvC=GP4h8IZ{FiNWvxXv|9;k9r5#RTp z?wP$haY#?~nbUutK7G!A{`>j+qobom1IM=u57uVhYZ(8@i`|EViwiiTFPerC7=hU| zT4u#GS+`KPDwfo3)a{Bbb*Jf69Nc%Bxt3dTQ&_&`T83$~yh;I(T;Mi~t&z$|dbd<5 z$rGcMQTZRMjH6AyInkP^Oh|jLIoX=3Otp?xj*rV$iFclG#| zU6}!7Bq#w=N+BnL(O~SRQJKY^@n8aXCek}60htV@0GUc5bATKRrU98wA*TR29?SqT zlR{1daw3=oWHyC70LaN;4v@JN@?dZ(IE{Bc6qJJpf(LJ!6(4OL3VgKj(>4zS@^J77 zAdjSwM*w*=cnpxoQplr#JRY0@c%u5jv@MYxOl0K@bMjD7>~Fw(DUs2S9bbJ!r7|UP+#~u4>(Gs1omD97~6Fj&k>5 zrx($)sDwtv42+5um=*h$v1A5TVBhpWNluUhEHVkD27XP|R~m6xkJnW=@4rxsLVu~F{CK4i`FO-{ z)!G}rbkd1eLglxJk7!;Xb$8bkSm!KwMo*)Njo9oPTh_L<18VQ1UbXucN~dq5%=I0V zZa;@I9~f7SyXHdKOLEaVFr`Z9An~fzM!OMLt4SV+=CqQ0peL3TB2}-3t;H}%3h{EB zRhPqdsA_RXmEFXN!se2iKu>Ct#T1KUET&Q1HIqWMT5r~(s9L>iy!G0-pcdE8fiB*t z#o@Wujg51B<+%^HYRh3t42|~ke0KxA*H_NfHGKedPUGa}RAK&6lmj)hJC|jpK4o9mk6j8_y&*P@8x} zH9JZ?VwyF@vFPF?8|N|Y>maWua1nVZ3~R!)4C{VxX5Tc^hV_i#*_L(ARA$Ll8qRXZ z>{~n5RYQ#eT(B?77D}c9p5()J=>$PJ%gtP!oV-+Ip>--pN<1L8o4=3ZW^Vuj9lOUy_Wb^j!~|O*9!Ih;vh1IN>w4ji~%Q({W0k zm-B3a*8$pdWNQ|k1^D|3DG^F8DAP>}AIiA0cm8s?)M$r8dDO&JP5DpyQD-S`)o%DH z2SfCtQP`b7Kq*wj97*J(*ehgWd)whUfioarC{*-XxoQk;6{%AYm00a|;=U}eP8{&c z>RnAvkT1k^*i3S7bn3N5C72S#3A)J6@4*>y5M<}x)L?Ey4i<`I101ZgvCuZ)OLL}qW7ks3uzcKo z7H39Mlfyx!{B}359zg^3D2vBfJkEmi(NuGWwZkZcNP7lnL^%WsF-wjhvc7+oBOu8v99JIJx38* z#pjxhD6V%}-RD2R?zv1Z7OXc{Z+6;YH2+)^G|-HmMUErZ4kC}oJ7uqYD%pCEUh1gO*Md#?RN&;Z z`kLQa0-`l~53m#8$}=K~?Pn)4)0XJ$IM+8<-7RWy(bk%=Jk3}a@u;Sy3#cWoepOO< znJc;gjR;-b=!J>XMm=M-oU~FO%^o>>H zGAYG|l9X9&n%7*dI)w<$-g{q#?uVsArP*DN{j=KMiPp*hs?PfIiXYbMD;jZBtgw;V z^Wef4(7$>SMcL6xh$10zL@|Dd_lA5lV9zUn>*-iUPG>xuPRgJPKF96{ha#GYAd-d* zm{Ww+aqc-de;%@uyTx+=3lUbN9XK}&up(hU(uy1iXir_jQ@v*|b=py^*6Xof({tu( z(clB2`)99)p7$4Ep#?fkO6ILv|T2oMyF|5O!m}ZGX?oI4k^_e8s3snd8 zMbz!E8>MDL;w^x;)b7+u20(4 zUdloz);cTbj6sM4XDupFuC0h%d25X`(et{<7+HJL@q_5nV}1R$XS3kFhv}T7Mq! z!Lk<3-l>Bq91`vR+2rvsN1;6oS|g*527t2ntI_?w<|BBnI_{ zpyKiYkAxzHV~gI0VlS%Pdds+FE?GJnvSn?Wa>wBv)G%pGG}n9LibwMKMBjUCO#(x){JbLUN5&!fG}zR8T>O$oJ+nsrG@7Hu1??wBSPqyGq<-Om|A&bh zppT)6s!g~{go#+COj0z_4wJ3d0UyX#TB7N6hV;ud%MK!uCp(WGLy=mdQ=2)&x}fpF zuFjUT*BTYdQZJ!(*&Ujc>R|wMG(iFhN7_{ma$;pYds>{k zoFc4_bI(Ea^LWR?F&&B)jxVKG1L3$^GQOKpDv^=zmRQx>3Q3u*9+F_&L7iia4qBQb z1`mlm#`zewWpnO*fZWBkSvc7-gmwfERdUAS$y395h$tt+yAI zTWE1GLU?>P?hYq55*|;PI&HGP^_krQ)4XLLE-)!dZ&^z=oB@QE2g20dBTVzXvj>E! zYAmn(kkZr`NKtisYJmMJ;fpAGXZE1v?S<$MBHM2p_*sL| z!1R^HlJOu2)6zp4h8IQPa!9|H4M}~RwK|GSJ&K^-$GMyjZ1fejm03!cGSz_${}tR! zW2cTeXP$-}KWw%~%W;zGFS8RehcL6J2-|}QQVN;n63&Qmod8xs-&nF4QdwnK1#U?e zncB7kL_RmI*xAk@s%8exnpUz8z5-e8!%C&ar=!WS55u`ZTOGQUGzr(mT*HP6_F0}4 zWs69l)$S%vtQza;6|_iPSlAl~_mgsy;*#pLs_pd_;x@L3#I<)vDL+}g;{YVY>@*V3(52=>&>|FQV26U z7@+Mtzxz8pzWTv>)-AhcokU6Uk7(Q+Fr>{H5W<=jJs8`cqLw!2Kpfbsq7j{o-8hfP zgvDhHbVRHW(CjY8`PC5;#2vOP1-7<9v#l73C5!tbDh;7;y!YcqG- zYzKoSx{bxJMc6r5_&$1w_{v1g9mJSHi2xJGsJv){TN<)gx<1yVm)daSY zoi`fcbtTKerBswbX!k7s60@1Q6cw+%Sc~f`h)#*Jm}N`R6SQu=ic`m`P4|B9=|k7& z5Y0GqdHQ{`kHI6wc!k3ESVs+Ld zb&Jr!oH7IG2B8})dMreqh;EeWe1@=r?)wU^A|Hied7!l zQFsOJ_7%Nt+MC@?msb{iq!vh^bzbIkK$1Clu`t>DQQ0fW_HKd66%{&T_%5#U*ft)7 z1#}EQjR}xnwV`{KH4!WJ_3HTq#WM#b3WZ7HK&vbrDN??HCW8p-7&puOBj!JlCkHl_ z3h3h!`#Ft50#gV}#nv`tFSq&_JLWIj=*y{JM>{XIjU#Rvk%q>Byk28hmb@@1av_`3d4A0UaOoh;IZNICxyYQ5*t}GT6a9jHZv@S z7$jT$GtSiilx03PcsM$GWjWwWe2^W=%3>Y05w;2%ogQr7I_fvkLPacM7NWh7BcwnCe3K7zV4jI6At7~~dk)^^@s2lHlBtZgYE6mHU>SRJ z1HPAF>8mgLSw_W_Am3Q(}(FQmIy2s@;W(x(?Q(}5O;v0 zgw?WRZWnh97`*QGNO}i`{f0xH`@n=wQ#TwXz;L*c#27#y5)3dxkCDZHTQu!!y*s>x-55PbXEg~aNHuEStpGX#wPA+zYR#Wv_S)_ zfKMrVburl)bq_#yFIIzs-)`%i=H|;v#*Uy$z(_m?G=|7x6w1F~u|K zhh#z~wq;3gOm1V+TQL~nc4bfsM)B_kW5GE7^TDj7ESgx6lai`vRth>nu~HNtVFb9C zOD6VbW!>Yu?=dgq9?iRZH1V{SrLM^iQrbZG{3R~v9B&qo$dCuEdt3HqF}6wdQXhel zN~pD2ip5vJRiR(RHc+8Ig3{YU&fAt1dvRgco!$454aYbXk|m6|2)-<#9sP_#zeFc9 z9@W=dxy{l3==RtS_qI=Oj`hd3$9D`koR0c@TwI;lp6rk9ASd&E={jSNTA<9gWBONK4*=xR1wICm_Kl)o;Y(+f%3= z+cB5SwiOg{J``8ru!%g<6 zV<&&WH@WLkSY2! zSM_Hs{+#Xd_4vj!cs?nL@X$T{DR2J?3g~5V>Zw%JAF`NXA?j2lD5swTCZo%!7}(Eo z?Fp3p!Ki?D?7*yGB60D15D(7W|62FmJbdQ#=U7FX!Qa8}t=`EaNn(${pbtc0aS5k{ zfYa-|83`q7n(kX1(Ve3J$wK+`=40YgCRaTz|gMo@#ZfGFWV z_3c$i+10`tG8&{-T(d72oA~Tu;CAHbaKNF^PXZnz__|#RT;{24kHWPu`2?OvT##o=?Dn|)*L=~S>8tu>T785EGgg- zR3KBVF$Cr3%%bJlvl8u{1yy+bG~s^HL6oiheW@d(-aP?e+R_}hNRW@9f`yQ%x;l+z zf^~~)1oTfD@JboM5p=^ZwS~!Suo;*PaUHb7_DOEX0h=Ks4x3L{U~H}AY11yKCG@OA ztFlD+(xZ+1xFnDL32qfBMs#G#Ls?b!wAB4u-V*Ey;sk|)JzG+)P+(&w+J`=5;-?9J%ABz#EF=URB45+?bvkj`L4RW4Q{2EsEP-k6I;dH3=@=rNk4gp!G!_5z5^r51m3*9hp&Ykcf~(;#LDV<+kZ{>L|>`9TEu?9=5>3|j1R&(C4hIx`X98rgJ@KMRN^aJ!C; zC6Q%wgmqX@g{g0$;8!MZH0mKTkf{9};Q$L&7&CDOY70cMZgiOK`SnM2{hIY<3a#G@0jbFZax;UcP4$k~S`3%4RSXyVgD!$^d09G~&DoKrcPx4;p+ifvxD#xkaEbeh`vWqZu=)S-k;?;4YyD%j<6D8(72OkQ_$Inx?M(;yGAE*S; oRRZ(zA3reQtTi<|Rq&jtQT)d6D@{#$Zm~4=@YKjuVQTvS0Gh3t@Bjb+ literal 0 HcmV?d00001 diff --git a/captioning/models/__pycache__/__init__.cpython-38.pyc b/captioning/models/__pycache__/__init__.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cdaf18885cb6aefe708ba5821d29030b44ad716f GIT binary patch literal 1905 zcmZvc&u`pB6vx-zU-quO*-g?EB$WCBVtdF+D^8_C4Jj2!NsyWZs1H~Z&un5c{-N>P z<_D>S8qTOE#1SMeNEK(!9Qgx~@PP}ba^%Q~H_zV9M#9OA-}}5bo;Tl}nFH7D)bRPk zUT1Hk^_O-Q9}}Gi*z%Xy@ftT|EjE0^(6))T>6>KN(XRV-vKzADH!$9iO}|O+mfxng zW^Hfej1W}x&3J_U5dK%?XnxNgwRFpt4nI<71 z*?nb{!`!{&^mJcHxs!4sF|qwc>tJqWjr%p7zt2b z6Z1ZS`ka1Fw1recV@^LL+D7^unsYiPY9aj&T66jaQ5)&c(4JF_W3?TmzhSjqT@_em zH2Iri4#xfjyJEi+bCLZ6or?WRtas9SVJHi_6@5u`>BL034o*eCCE6$Y4!9Nlis*po zA@nY5e3`FY)p6u!uvFz((66k=l)XyXXE3PfeWJIBegn%DeT(QC(H~)@qJ3E9x4*xo zV?P*I#&hG+z^(V%?$9jENm$r9?x6s16J2b7%2X01NB6upb1&9g%8Rs-gFN@R$dm{f z2>z}Yv*g@M#HlwC45mut9#g_gQt_`MOQPHF%)x$x4ivf(I#BZ1Hv2SUENjbiFGA|L1u{QkIvobGC6;=lx9v>-U zkpR;NMb8Dt!mEbQ)|3!TRdf_3OnMp1gxbJaIm1MpjNWmI0<|H{czT)?ttq{ID+d;y zP#PCWj?0Il3)hFtUk+J#EO@ZM9{F{avZ8T_=dN($@j~e04jx3Rw3PdXg!jBJpAFXv zH!M?TLeX8I+FGdg_70sr5NM^Cp`M8Q5 zWDjxmRcuc${W|rbUjNK3+D8I42UQG80T!+lWn)Fk(Vf1?RTsO${@^SWwDh8-?_&DDsS1xrl~{;I z^qqVZ1W^(}5MWBt%mqxdRs3-N7`vL$F?!f6Y|dgc(buhvq1B;=zIoknq)Gm4tl2k} zhhH786rBKXYkf3=Kl=oZ#}RAMT{d-^8Nb%ui#JEmn=)U;=5QvR9$0nU&R)c^nh literal 0 HcmV?d00001 diff --git a/captioning/models/__pycache__/cachedTransformer.cpython-38.pyc b/captioning/models/__pycache__/cachedTransformer.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25b043327618d426f920e96f728ec092ad795ee1 GIT binary patch literal 16425 zcmb_jYm6k+NJcYa7Qg#yGZjy|IZsn5+$<<)q!)RWs8w z{qn6E?{3p046~6pz$Q+JOb8;HA)v5yp({j2q1*`NAU_mDB2$&BoH7g%JO~Z zR`p|g#+HF<)$LoSZr!@~-1EMtUYMLL7&!hqf4utEV}|ihycj+lTs(#|{G4eRzTuk< zqiL2+lXVMqt87W#M%^ykQg<3o*};9Mk!fbj*#wqrW-Y@snr=A{NXE}L3eAb~L~^%S zF3J;=+Lt})Y`DbGlIx3RA|Tb^z1FYj;8mFJrK%KJ>?gNC2?3s(%k5M);m zY})06fK2#BK#B?EkU!~9T`|gsacA0}!JV1p&JjTN`Llq`CXjhR_WN^y%q5VcfE@4- z0&*~c90TN#e;AO%3FH<)j`;I{%qNiJ{!#xJ-g&EE@^A5vUop!b+T7}UXyYYqZUf{t z|8_ubPawAga)*B>Aa^E^I{>-MKLN;z1ac=Jcl##+IT`H3eDCs|^G4}CH@IdcGoG$q z2-}VID5%t%owkYqa{c-%^|0PizFv_NHlr7&Z+pif*rf>Q76&IA`_!;056yxXcpT+ue&CPP|HN!E=F6Lj1 zr=D!pFlKc&kiYJW1rNvLUKmwdepUHia{r#BfmctaBbb!+TLgMr8JkO>4{A2EF%Cuh=~crPH%f zW_k|FY%hZ{=NspZ>*iv~jWgjoaHWdqAa*O2daE8)Dsc{|=CtCRuO}AgLshE;%?p7a z=cDB)tu6r&n20f{LEM{5kXEBH3x*6vym0F`3hLy^7lSzQhia;UeK7dn^Gdu`=(O;ZR!r!`IhsjUS5^Qzhkfx>2> z&{j?3Rkz{{IU~cIGPB9AduVj27ShSqGHjU~tP`_maE6Cb^o%W@Ch%!*Tfn5%bGAwQ zX7_>Tt80N*Z^^oKRJ#_0A=WQk3w#gQ5^z*i%R%I&)Eh38tT?N28fP_=M`PD0Ycz))dF0iz95D!;CM2zV60F=%O&J8PhVXo85yurkOOXr8LjBtaGL^ zOD@x3mfL2}+P2OcY7*d*rI(XUO12DB9Yb9mXE8b?bri5uI717CVRjGf7>J(j;usO< z&{_sEk4n#2)#$b{5DW!GoAJ$S86ezR1PM8vY8YWyLSUq=QdVOx+YFj*by<(>7Ti#z z0(Co!9RuZiPUGBbpn7>qV2?amqE&4MV2MJd0=ipouwJZGUS6*@k~_Id#c$Ut6`?)l z@r}fV;(OxBgyw5Oqk$%ap&VoruBGrE+lEw$uIV^M*Uh-Lz-s`F99%jc;Yk$lB)UXU zWr4eFod00Tsond|221r;FpyCVTve6#UN3AfMa}9ZFX3tkWHbso3uDAX^~{lk9*R-H z6WLo1R}Rbqp+iBVNhzzQ&{maLkV#{!)rzyv$g8oes=az$Qy63pF&#AG%q#6$^@0*i z3E~7@eE0#JAqPPY@6Pt;HehNYLN>s`IvWd>Q5HqUD#YWK)ibwE9|Hdx2)<-5-nfYa zL}vT-X337tbMZu@y z42dWshj?>p?apPW{~+kF28gwAZ){e8*sSItJx7sag-06oFsikiokzcy-E*0;Sg`JV zt4&u~Q-p&y~2>)jxBTBxV2mXTKK zaXz~Xn-MNFM>Z>(-I-*T12*HR(}Qc*XEP5M6y|41DK_+`%wofQIjgm)5TV(9-*d1B zVEs^!cGe^Bq_%;=wGx1;y}rES1yJ)Ean!Z2n_5t!>MZ(KA4E}dv|f4;_1F2x{?gR1wBJ~&`NBKHI7)53rmMOY2z$ieyZkd+)3&pxb1*p{~M zT*gjGq~vM-=r^(3C^K6Iwt3L~{%i@d6yGgpg7AQX>x@_Z0@wR%*gsVrurA4FAJ z&Mx+O>G>l zrZ!>4{BuyVPnpqjl60f zGLDjJokLJtC1+95u718`#DCY*_V*{1H8Yi1+#l}9EAg-9i9z|mTY>49Z-6c z>9#<++nB|M4Y$tTG$U)xRS$q%jV*{%-#h}vdkAFfTStr~*e~fFLPfuS@8g|LQyB!NX@sI*YB! zuC`GPeI%X=b;-Qho!3}7MHssuIZ_0_Tf#3UNCP2Q}6 zcGo-9dl*YVEE;h#&K8RwJY(>^mR|?@qkjp)1 z6LAUA{y8pi#U&BL6}my8SITQ)Ou`d7hL?_kRXDJ?$Wc{Ne3GgJPwGY1NXLoascHHp zLaQwHqV5K6iieUq)sV)iizm3+owhkuQjtuV7W7pS$VWi0uNkkIOO_6|Y+4(p+;MmZ zHB1^4&2;ZRCxXR0S6_yz@b2?YHiD(7B-(EvenL&JBW5cJnD(-mt&B=sHNjKzgV9QZ zl`igeQ@y1BxE?U?zYl-fqUgIEvHM7)+PvUbA6>`If5&NbPYjdELW1UDYgIoW9E4s4 z>%?y=N%%JXE_sd5%f6u!;!3+W7Wiatw^!}08b}SR?0}OOTP3FmtKl3uxFkHrY?YBH zSWavebJ$jib4fI-dzUyl^;&A$dP%rQOB5Po4B?E~Bz&_F#Q(wGT|plMdso|oPZB0F zT4ecCyxc`lcLDDUYMQmlbovxL8l)(oc@F0pK6Wq23Lyz&Ew4Ju@PwAF4e+Utxb(-t zsVtWukT<}&(&jP#kKSupzOms5=VvIqS7Dog37l&d`ppbqZ(0!lnbwSNESnMS6KB($ zGd40&hLOHighto`7WPM8v^H`$-Hm(?wj1P#jlfBMGlOyDWgM^fvV0e9J-&nWZA8-n z#Z@1Qxv0zS;7^?EKm)?Pjm^5EV-Od_tU+8EhW0MT*^i=4csmNibn%-rJ@XV| zYsbujY29=z*F1<{0rqIY>Q1MGlkh)cMtk!m3HA~@@AG9w5mv)Fa*#84ynDVZCgah% z>#5|bue*j7*C~c5MTMP`K;_%&FpfAqSnunwai2o8)Gl%7C>N13L9vNG16^uO#_3`e zB|1dGrh)=#6FED8S&DKU!Zx7TEEpv8E2_`opk_K*aW0-QgG9DWT& z&-w%+AifFH?D?w}JPmLQ6r1&$1I0#P&lSbSJKzf3;XQlF#-76Py__gLODjD{5tCA7 zs6Rt`w0&1U1x`1~b&nX*M)wS5Dt$ZHrI5apR+*&L&^vlr;Y@~s3f+@q3RcyZS9Z~` zi^9I|pkj@_iZ%NxR$Tz56v-TZ`vKO4JNxQY1%Ukm;Y%pGCq@?H?KSH=j7AjxRRce( zKN=`}ArTo5f_zOqB#|1UL;7MmBt?-m_9_=Zu`Q@!hHMLq1OY*+W7b;c4KQFU#cuaP z!|!RxMDiy~9iE9^xk=;+s7?7wrDdH!9>`klnxUs|Ut40}_Qml>OD*u})==0md32((+R4qPQwUssG?Kx>8co3z3zEU9+0 z0*!{Ss4X%pE>N;n!j&p~tLed;cH8t(ucE_W!KrN*#IWW~fsObWBumOEnLXEsuB4Co z!fr4zb4Ve~8B%zg8k@7uY@0_7STVo^teUgEEbIZut~D!y?YD_`8`;QRrJgtaETSkm z*feZ;0X9Ys7D`U~%}K~VN6Avkbx)i@NEiFT-reUBzJ+^qwjFso;2e36M-k)A?bh8A zV7c)fgx$4t={on+TT5acKqZ{R=r!W9&1%$KZ(!?@yM2rfFw_^VAg1}UisA`cn)G(+ z4eS~$HQLokeHR{%b4l!;8=crcR+nRYEx3%0g=zy=V(Qc1LBzkZ7H7j6Vz*(O?T81p z!#IfZSSda1U&kZi90~(#Fl)ME7R+?U$js|VLFzXC>!Mv__Mf-z79C&eY&EBD7QL6g1E+ZMc>B2;FqPFpR!dWF$1w9 zK-?FwZ4Fan1%pofvKt&5x9`kOX;N$Y*8$N!>(t|67$}m%lS;Lf`YGIr%@v8yOE^Zo zmyn#^r3zz*9HTzM=ec%i6ygm67EB;9!l`5W@DYHAh`) zIi}IN7cMXOk9Gh!3U-pz#rn344GTdakxxB)neWK9Zr2gyD9h)x#lK{4THM}w5ps@C z=(odeFYfXFH~Ie8&9K-)F3pZN<~H2OSo6+iJ=|^>~Gi`yUpU)y?0kq+r`F7_qs$Bj%YUvef3R` zlemy!dY~C|3zm-001E>suy`Qn;mK`VdVdZYhv95(A<&{ho{jTLJ-DcZb;x#VlVD3G zB{RBi#cu0DHA13AsLg_biNx9cGfq7_I7fJ}FHz-HrI|R7OdSZ`09yc*=5X++>D14l zt|fa$cE{8z0c{o(pE?NjIo3F$xPVAcgaA-`Jyf43v_In1{rd!=Kf&TBS;#oVIH9#t zcIoU@Ao)ShAnvGkufg8@Q6pAj5JYB?{S7vimUO%-?jK1=)v~HK*(%1^-Pq zQ!t&IMMttPi~>{vRD#t#_I3-6=>Dhi4utU*uV>YreED50POz9};j!T24w2MBUdkXT zJAWUiVBR&SCKMqxoFfORj>niE`A&OL@uBAuJLh6O49+2rcur@VbPo?R)wx9vNm;TP zwICB3Q%4*D*-EfEs*Xm}1mucV8TUD}xXK~hhU!M*oqC@$33iI3AV_)-de*wZzF4_0 zMsfeRTA)UV9he47dqy_@lOrTcF|x^D*V|+MeWftA@t()iXV}j%6tuW-B4m$Hy}@*d z?c|pxLuAx1qMaU*7%Gkm8tdwd1ip`ju-SmuJixwp0|te1Iz|KV`&mF0lX;FQb< zSy`;5wxm}EdhSUt&E;IUadkglpT3T-r1!y- z%N>3V#m;3$WWya?|d8<%OeKvPmkTV2lrvYZdUe?aOCaqyAjR3D5tJ9*eC`P}-;I^UDBf87|`KBfO-UD%wsxhZCZk$8DR)yixUu8#b zClTC|ep|tV&D|KGK;k1wiNu2@I|xg7>SosR5bfm72&NrCm5}Ly%;kdP@CurJ>Uk-D~LS-0YcYc=v4n!Y)6HVayPNh#0ow*(38_y+*f2oN+9+ZJ0p1?2^1T9#=zMPirMrnl;Y|ebBN*HL zJHz*pLd@{3Z(T8>0+@{D zsIWS-wXZkDZHAXW)0@Qfo$l$Uf(DGjhrKgH*)Q6!WWdB*VF0C*W|-{APQyx4u4}WM z1ky_-vKF6Q)*(SE22=o82@mOj<2^gEztBWtug*c-V@Izvs~%#%-_3%?mncsmBT3to z*x^!z@kmXF|n@RqQYkJD$E&g@LY+?p_CVH!G!S zF~W5+wj@XDWNcl{ELMV6EykZ8)U@{bB@XQk7G$p2tf|lO--0zndWkcFW%XIMINUFY zqqs0Y!f|p7tx?0yV(Y%htq{bXW=&-3-Lzo&TNYhu_bsQuyD*@Rz70x8Y$xi70z1YT zp2K?vx`I|yn#T^(g&;xpPG*xKiL6}1R7kFKat$rA3ZlpTyjdXXUCW zS2_?lxAvObz#oq_n;r#>4^Z_T2%f7@I#v0qof^&GEp=0WJwTL++nrne5_ z`ViV6ZwR%0{sg@Gzf7LZr_UZTK8d@P!}6XY-=n__0qlstrUf>`uSEcx7uY_+dd5pf z(eE*T7NQfD@#-!9e&z_QB8LvXImUu>0%sBDq(67f#q-ByU**7}`Z`dm{*Z-e3z9-S ziSP`ATNMe;sNdmTQijMDiE!z}eoYKia&ac0F*o9ZVwwI|EcY}X)GYVjG5Sce@y8hs zd1A?z2mddF_)t2CI0Ic%_$<&IasLx>c+AudoqiZ)~?a$Pq$jJnF-8Iu(Sb2t9$4QQT`=< z$zm_+>=~rI9|pMu&_#ie6%uax2}Gm{`K%!^L4_+xd!ZH^LsbUjk?x5-6YSjvFYB$0 zJNWz#6*IH*Z7&foVs=@uN3HH{W4PJ*#Nr5s_-G$Pf1hFqp}TEMhw&tDLI?8@B?Uf^ z*`OYSo}lhvrUp#aBSw^2#qO%)i&|Sby+4EiICDP!qGzu{%C6?ukf$ZJ!kYaU@;Q8` z?{j46ZNQt*PXZnxe7;rmvrOmPnuK3r@(DbTJYYtA_5VD49Yi@Dm`Q&X!DNClg%Kj& zn~c4M@iDp#Zwa5p$jZSlkw6~4r1NE$>Bop=VnDkp_o9X%oLg(uWluv<)rurQSTDF( zVUDi)0%4L*;UMLHS=(qhill zorI&0%?7=VfVO#Y{~8Zo@6k4I1qNm?j*80`RxIO9zQ|B zex>grN_<-4!wa4m(%vP0y+;=twdBjBelN^xe%SNZ1Li8TCsF^B_ItR8_y#j_`wQ4J@kLuu3}G zKBXG0GprEj$=;H8ugw{uH?afB={nLbB7L5hud*Ols~afzf#NImT9CxnV+)?h1Z*%7 z@s-X>Tukkb5Lc3Pg}6TPDRKAMx*RGpvHDjQJ7hk-4pPALht&NlX7P_K zgc1LZwf|;8S7&J4V)}^H^A+4U#<3G?ER$IZgQ;tAKrFIPBLm{4rT=zi%0KO0YE$vRTk{&@$JR&m|UGEiv}W`_s#XG{9DE#Dzg!Z)+POd9@3A1zp7%$th4j z!fwgWYJvqhSusE$azD-@w1Ce@w1ZkA_z4!|b@_Hc=Ldg`mn$qn7O%4S6pMkI`z2m} znFS|0geh5TVhS0v@GSANM=4Rl2T{;7EJCCgA(Z%!9|Y53YxeML-gRat@!K~$HG6FK Rkee+OXWu_NF`J*A`ycm=0J;DG literal 0 HcmV?d00001 diff --git a/captioning/models/__pycache__/utils.cpython-38.pyc b/captioning/models/__pycache__/utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3b72448afaa977bd3bc854950f0b3c9c79acc68 GIT binary patch literal 1269 zcmbW0&2AGh5P)qbn`{CJP(o=dgcc4+LDen~P(@IFq+WWdxKu)ASv%>H&FU3(V{$DHWV>N(tSw&P#rJHP9MT5)%1%fYRmwcpGC3v<(#C!BxI8AlugfEP zNVVOy)nKYuy0-lI0pfXymyhfbeF9lAAJl9Qb=86KERl}yXb~$=lD4D5p>Wm*uIsKk zZ-pa8DtzsP+KC+<_Jmq>{FpmEzbnwV9%JM@QIJHD2y~dl5XTcI7D@{~O_D`TD70eo z3&sRHv?*~NpqLFbTQIM~REr=wWJF%n52%LA!xh;VwpZLDW74Fio|m_im#@amF!zC> zJ7a2SEHTT^4UhZU_fi?Y@wM=JgMBYZxbQZ5en&j^Q!F-&J8rtK6B+D!!QABaATCl~ zUx$$bzp1W8q0&LpOItN8t_Y$;7#ZHZScbb6C4nEQ7AUg~&8o4o1571U(|S#%AFI8- z5JO=~Vvs^(O(7DovDB`g3R9LMSBiQcmPJekS^8-t#!Oy<(+wSDx0Fb-yo{m*B4e%& z3bCrRO7FpMi7wF0_ZOttrEJ4zQ09~SWbFY` zYATIFJp)DJmzg&Wub+1g-xPT}f7d$~HvRKiD-!!Gub@C;P5qHlLLya9rA(x3pt01C zUxhK3H!uv(kT+2*p~wn2f4>#5U4yBvf*|mEi`0eJYfu|>iQZ-*KNDX%Gvq00#l187akM%BU1ZkTzQ48$$(w??PfEWl0xCRmR z%FY_2lU*b^7;$bn_@F)dl0T-m^4gPu{slo0_dSlZUc0@80;$JG^1a93``~=q?M4LJ zzgll4ahH&P5fPziJSL(c z$!U1nc;v-RLGO@h^94vCw_i{gr__d7skPKE7*3gOFN%fXIxUq{_Fyi7mf4J_YGD>> zl5r&;BubCJ;a&wC?pN$I5cJGfe>8AMH)Bt5rCCHK+_WwT z(m+LyY6I1&*ahzB3Ib+$x78Mimd?x4+*=g$Y0n0F1y-pBD7Q_{9Vpy-<F;A+V%clttYC#G!e~unmb5zk4{k?(5TulJg2tBd733!bN-zC@bR%oOme(bi~EU@$N3lM z$0=w&{&NmCe3&c^D2ICY2x5}hG#;D*AYUjW{h0z&8*8>;`?a>f^ z5goD!#)so8t~}bom7;08whu%LI-;cSL$Fokl&sqevhGxo8VXu4!PecX^^~gHRr{0* z?~JHZ*ahC+pto5CAq0^h{DaNdA*q53-*G%awj3YUU(LEAeB!Tr#lZNRKr}AsCamZM zUH6>y2E@#NKzc1X0A2%n6MAd;J&ciPp0VZkqJ{JsuC+z<%olCZc@q4Mt^4bN3-7Mz zJ#JQwP3MB#B#-FVc&~0XsQO=#shFFwTEpU>h{>jhc?Nd&FUU9VL&ja= zB-&uz0Q#O7KVxFzuo$h!pOHV1kI5(GuWs*V=<@XP6zt!j+ZOxJy#GP*+M6hvQ8fYt z`=FUT^4AlvWn4|bhm&dmKAZr5A5lKNR!z1z533182gHJCV0>&2AmbfA)e!y%fLRCf z{ouCsi?Wc`&t+05ta?W;UP6OX*iuqJ(m`^#nh^JNIWu|k1Lg!g zlIPOgD}`;{l}XO^;-R#A<;pBqhG*q`d)hX)n?XQLO_|GDFaeS>#qzUjXbx8T5p+F< zVGelyy|bo(1wAC+vd{fHMB#!Po-@mies3v<0^rS!u2&|oY@?HGp-o!m%Ny^aFq9e+ zMjpO*HC1P&!g79el9f>0^$j3)r{FE{u?X0(F8S&`6x+yhxEgS`^$2P8KCHltCe~1AGJhZ!^Ls}1 z@Hm!E8@TFcLn8}aD%&vUOUV-*kBmwd1%J)Qald9>dO9B06?vsm{Lb0&=nbVPGw!`6 z5aPg;h_k|4YJGV>$!w!2`CKI;9_aK9%^l!ess*FbIEA&?Ml)6ByjbPSbIaf|eJ2lV zzq$tGbYJZur#VacOeRL#R$|PS4Bh*s;W=T|3gab?<_2Cpu}IH_lG{mohBFQD^ zHsZW~HG%GM{`o4&>J|5Bst>?Y+uc6TsmwCY>#Mmm9>S`^`?+t800~Si$E7Jl^iy6%|g8gRps08+rq4j>Zsjn6Uo>K!3dS literal 0 HcmV?d00001 diff --git a/captioning/modules/__pycache__/losses.cpython-38.pyc b/captioning/modules/__pycache__/losses.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..260eb6f6ea8918f13a48c0639e8fa0b26a84854f GIT binary patch literal 5806 zcmcIoON`sb8J;02KJ?mMt-LEcj>9~fMs2#Aq=y8B(U+SPL0qGVQv?-Uu(XD|%e6$w zL&~ui6bjf0ax&m^PSP>=w8%M!9$NHjpmQjS6lmdFPd(()e*aL?O51h%Kudw6;XgC~ z%>45|zJGpFtCbm^-+LFrcfZ2epXg)xvGDO-yy*~yV1oBpr}*c6qhoL;3}N=nzSXh% zrA~>nA2MNy()&!5BID4$XLcM=Y~g_7%qT7>uJAzdW)u&UvZ#PknNiAuUuEs;Bh1RK zJTg$W4OQNcvLNjaWR$8ix*qlRf^IA#IU{$e|MFwbKGNq=iCBjV)-gV0dt7i~yzh2Q ztc}*oJJyHHVx1CZb=2D1(R-m3mt>qpG9DydzQmL2#FpN)fg)oW&)Gd=VoVtX!mhH9 z`IWY*O6h1ANm)V{uEg?wkE^Ke`kSGb|%c_IHMNvVrU#>~DHJqK9-br9;aj!7+Z+ zIn3(iG z=Ac``&^2>bGjr(24Qo=$sfUx<6DM~jE?!TVQ!XrwE~DhRBQVPc21?@tV-Iuf8@jdN z=;XY@aDr^&`{jx)Pq$j@?jQproX4KCG!%j)+N=$o8JZyUY;fZ40LJLUS{nR05) zT{`hAk|g!$?|3Ag?;|Rdl?;+duA$nl;WQ7zVWjN+Fgu8(@;1d%Zwew_=dYbkuKql1xyQ;$%2N*EG7RY^n86c`_2C zE*-t%R}{aD;|kLo${vg|^p!P?D~dXCaP|h0)Rx$#EfkD5c*`jBGOzP*@w(yi=ef(9 z#?kgkeVC1GbG_S-ZPM2y>_r;LY*bS6C@!y>n$pF`%0os)Px>_!b9)uMV``g)F2QCk;fRv3VY4M|vo=o9Rclu> z30aL~bXnWM(cQ9-=beK%^~o}P{KrW;j3HD1`W^ognO@utd;Vge@PCZPSPR@QxcHEa ze>K8MC*8=;2FLZwQ7FkG{TE&rqhxT)k1{{(?Y!u}^3uz%wRiO7U2T7QLyS?KwnuO3 zqgeGDIebW9!q_bQm&l7ql2_Z7WQ}r%3bJcu4u+Z9Oy|1|Xmde!2fFDks%=9nmE5IK z`Vh3P=#wBOKHikrGpL3Q2eFVrvF(L}(E70AADS3+>>qHNF&o3WCx{KOa67l*49$D| z92>*Zb9QJPnm3GeC$ny2$kWHJb4OmyN7`xR!H@M)wWW*XQ*n zn`{Vs$`2bAHZJGo+|?9#GOTNZJQ_SjIolL&UY<0kY;tA_ch4DSq5aiBoz2{2+l*!h zp9y-am^EwAL2!)?UPim2X)G|sXX^(Ja-nTk2FtI1hm!_%Il^})+M zLI3cy#Ji^X{RWDI^=j{{*SfT3^2l?U@Vqj?tnj(;t>Y^8e;y;N@QwBR=D7C2Osb0& z8}gZxTs8j%udF_SBqvW!8DvzMON(SPu7TqkR%WFSGwUvYU?iVtzU!bF@>eJMK7O^w zGD6g;kJs}07$*jK82K8`cmr#=0I99FSiXUmZ(}VJNDw=OOoa{k)$-b7*5vCoZD38N zy!iiJ(_>e`laKT&NH0wJ(Z9V4NE}-Bug;Sv7d7{W=Ki<;E%)S)n)fDnTk>J<9YXU8 zea=0cg?IQAX_+uu45!e5oiw21UPe+kL{sp4Kn$=+OWw_%Ub5#+ZO@z9p2@1AZ{!We zoB8JCncO3O(EiPt-94LcHrcCWZ-fnC1GzUJ0ec`lJfqk5DYW3EWz>=jb3K(0G3!QN z)3{)R@Ij$D@E}|8AcP3Fk6zXOoC3<56s9SdL@@yY#FqUyOMQe1)cka?m-WNji{2Vd z$PM_+D{>vBe1-}N1#PZO5%-l7C7B!y?{q8~BevIP9D;r*_v7RpbaQS*(U7{@zy?XL zhba9laUo#RDPy)--%hnZ>Sgi#Yg0yHFIZ*Q*#Hs=2!94$fLjWGs#Sby6kXts;Jql` zKgcA3TV+vDX+I~=(}&rM5*<`jJ(FRa1ZmhG_M#wB_3r_EN`fk{OF58|@P|B0#W@=2 z(!6`4UQbf^Q{_E5=m*KDKfI$Hid{jN$_r@eoSBIuSXqkVZvbW$vQ-rbdcH6D6tS}i zJF61FGh|6(Q5KMUU%pILvmYlY!rPKUmpnto7B$qml0xTfLS;s`hsu&sZzMOV*+GU8 zW|@?PSd@p>FcGxQSMl9$76H~rT|P}6Tv|i0pQZY65Tya;%GblJdk|>gSHa;0$OnSB zzb`2p!Ex$TMt+k%O5H)4rOM85>`^AaN;DoQBM3TW-4vu~>XbD~)OF=v4{>yH0t7ls zf0mG!4)p}ikkzqtf`VBVuwd0_8g>2(-WIT69f+`{QKF6iExu*9j4E(q6aDJCbb%hL z#(CawXLR(oc$Ghg(T&iFC>ny9z%u4Bfsew&!{X!|4x zIe|(YNuW@Eor)JxVB3KhbOAUC(r6?qzd^NQTzrf_2|tkvYt-X%%dD0(rs{&W@{oSg z^PgpmTbILRe}q)+M@ZUw|8?S6Kt8j8d_To1img#jb{dTPC9P9qOn&s#=_~jWBu7#s z40wsS{24`VAk*W>ly-nPfD`Zy1a_9e0~@)iDdvJgP}M-T29&nQ3+Y?uaEQ}DZc|=h zHyV9<%f!-oA$Uzn0=RBsQ&tF^ z+U}B9c$)^2p%>ozIjWH{YTI7Y(S@acipE*qR)&?~XPo1{ap5>`yYQGZUNscirddui z0@A2YXcls5Ite4nKbZiv(vy;vEhk~Z(`gu8n0QFT%1FfAl{BoP>X*aoQSWMhFvt#& z9UYx}lLAB^PS)>Hi=WP$c64m~i=GKZcRni-s7^1PQ~DZ;)m$kv4=r5aKt=B(6+pIR z<|beV3v7T17T_uYrAs+f+q&`wZjP$tf`jl@Bw?bIUL&oJ_Q4m=U%!JCoH81DjS8{~ zWh9C6-n#sQ_!i|u%A;VZZ-Nw@ak7V4iF8nTi@hih=~xtqIL+Etp#wTzR)(Pv5JM1) z+foqwQg`kf}5Oji6Hthh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6vTKP9mwQNN%l zvn;VBRX?|)Qa?FAB~{-uH!(fcIkBK5Ge0jgFI}&ovLwGKIYU1gE~H;tl9^MiA0MBY bmst`Y4^(lB!zMRBr8Fni4rIw^AZ7pnfp#g9 literal 0 HcmV?d00001 diff --git a/captioning/utils/__pycache__/config.cpython-38.pyc b/captioning/utils/__pycache__/config.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d5690ae173b481729b5f787e1f7b10ea974d22ef GIT binary patch literal 4669 zcmcIoUvt~W5x)aK@Q)~2u@fhmW-^G9X~d~bHgS{bXlzey#p$?-WjA&vF((}e0*(|U z5MT~RDWs`(CQO_5p_6_DDsTA?{UXI{Uz(SG06)b2?E#=^CvBce8AsrF*xUW>?r-<} z;p%FG;q#CB%ie3xG4@Y-SpHae_!X}707Jk69fi0>tvulf$pn7P?p0MZ5pmxf;t71*8&N$xUcUi0cE%swO zEuNXz;xWFe+r9oy5{RztUcYwthU?Ss%OBE|KIt6{f{}JzBj83gKALwqSmeI`+Cs8eNEr51Wnx6zhoBcHlWWLX+o^GgR$Y@b9*G6FNZ?v6zShY0}YJY2YAY@;-J(&#LUKoj8ho+OjNk_wBB2_MN zBS&e&ITYgyS_qUr-VgnK$D_USRk1jB3e6mqIAOdmWT-o)nIfuoKvM zDYj09BrzA0I2xC`Z1~Awi2VbsU+AL27(2cfJA0yd5kjQsdoqYbnvxHW_W_EKr62;i zG&I!KOay!-oj6gs&X?NGM_NY>k2{J}uP3hDlOwTl!2u=@V zlzWNKhHxDP1DRET)OPL)0q=T)AymYCxq4rzVY=1n2?01gC!?1H_eE-_5_uonU z(Lf+l2%1-EuA}yuuBnq$+>}nr&4!p4w2`?b$dU7vBY)K%8 zG=eI^ACc58XPk_j!6;P(Mky#t@RnC7uU(smz$jJ{OM!QqjHC}!_Czm%F9%*c0(4_? z$WZtS!CcBB@G0D32f2nGAIHkOzCl@Ae5&|f#YM@Mu5rc`oAW6s;neu*=){<;DPwL< zjZ=PNe#GKkiz(y4I;b2rv*;ec!gQJcKBnG8j|NdU** zK(t!B7aXK8+*0KBB9C&1hH4Q!#B0uOP8=Yr#V*%mA55fwErwFMvzaQ{+FH2Op)E@v z`y(mIXkrG4as%cFw`eQ{JUFm#zOmfQsX%0N4ydUloSa3$TbBu#lW_If>pcbjSg4 z9-QPXs6Dl`S=+gfR6+vz`(aFpG$xdg!ZJ(`o$ztO=p@oLW8g)hA10%;%&YbjSwy9< zg6kY*(I*(DY|c*EJM4EmpBN{`iHYbpl`%8q&eWLlLtE}(hEI(bkxw%ty{gQB&n?P{ z8ULK?IMlyVEPh%;RO0C}?gECGGkMUh1P1O5J{xz4-8wf1USDi`Ln1hYK^t6|NPoZMFQz&mzbM6OU7m+yc?LLL zC%~S?z>IZ%mK)q=#v|LPeP@~L#^h2_L3EnA##7Z_)(IIY-K4Fv)Xm)t=_wFKWpf^ zosxl>k))Y5gmPH}#SOIwDJ(P7i^rKc@P?Unkc4qo4MIs9=}sv-M``gCNUdsCE2%@G z+hiHft^`{tJdH!3*QZl zmSKEnJhJQ>Z}QiS=1POtFyF+l>yIqU<_);jGO?Q6YuX0AGx^E?aIa4Vsr*oS(Y1{$ z-Nc~Sfli-*gS?SgQ#M0_%}}I}c;?ik!V?%s6?5|M1x4JVHm)GYqQCN-et3kwgq&V5 z-(vaqNw`FbF#j(UtOXSzzK}#jx*U1`5B4m{T2eCYv@9JH+H09b4NLZHo+i4BO+eC; zoQ{OIQW5pK&1Ktkq;wlO&C;?w7<5EXFl1+zWKno|v@x8P-1>{qAU42Y(-5gGxRkE0eoF0%;VY^Vy^kaOwwi zmOlckMC-(Mz9k<^QmtiYW-8QE$jP_x(-SeJcQJDeYlI<9WER|#%C||_A{6Bo9v|S! z!>@AISU~tg;YV`5$1FBp*XZ5D&6Lln`Q`FDN-jg8{^b;NXgR?uK+%vtr4o$bH0CJe zGY}9MF-LX9bpbdX z!*vHqFp8)iHeL5q99qii=(<7T(=!mB!$w1*`rl&uySSJ=&J>Aj1DV zkk#@v44Dn?r24Hx_yx>#SC;;hL5u9x+yaBl2orgp-qqAV*5}5wNmK_{{zgZ2(kbG literal 0 HcmV?d00001 diff --git a/captioning/utils/__pycache__/eval_utils.cpython-38.pyc b/captioning/utils/__pycache__/eval_utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..43c8e9246a0672ff8314905da8b53b15a469a0e2 GIT binary patch literal 9279 zcmbtaU2GdycAh^De?*ZKMMDdSYf{z(bET~`?y+XN zHEB%_+cH*$+p<=c?>Q^S_bF?N?|CaPOc8HRw+hw__enHoTXWVNmy^w6t7Mf>PKgPT z7Ly_)vLYv@L|#mbf|wDrVonrANz89eS@X`kNN=gnDb~DL5R11JYe6iDW!x9VF;T{S zsiD+Wu_9J)tJX4?kBc>wj|t7gni8aEnlqtiw? z)0K_3SFQKjH4imcP!)IDwT^J)C(tg9V6+-;?LO`oUb*lJo+s#j>4x3BV0S~+4mGfUuH)EX$%I}$ZssCDY0@m9mT9-5mDDxEN9*F6V~opxySx zjiw#_sI23r;!+@b?8>KV`IDdGCr$K_EQS`w z+-|esg%g{$s5))YXm7a{)zV$3eP5AfFn;>-hOj+*0~)zud(KAd=B*7<#Kz^8z2!hy zQV_bWcW-$eS-ZYb8&_@gyhhXAaG;bb7r_%IYn>ici}@BYx|;w|)RdZ54Na0LM+DVS zK^F`97z#r56^ND*TA;QILZ>HpSr}ZqDq?}^DZaX|LSXYsXw*8j&fYrFCx*+~u-om9 zXVaqH(Nf(iR`}GJC!bm0c42^Iz1gVk$fwWjghm~7Z;=&1#p-Yr_Zbq3*(FI((ZV<} zxsGSWF~f_Ero-Bc$O>4(_?F|*9$CYK%dfJUP)#+b8mc^nx>^im^dqJuF0!)w5vZ>V zwWI_wUx%eNp%1jcY$!q8Q+;h;75a`L|7oa(r=W%elw-1TUJ!9s&Wj=uBs@(dMe4TV z>3zeC`H6k(88J~(E-L#_T98D0n%aGBSd;QoezK?p6P}5gCWtGVaY6CpV)C4V*OmBF z#2=)$lwi_NUsSH8gN#4vXP(3J?uVZ0Y5pYjMW5_B#fxtz`bm1p{n(r$@^gxB;66R4 zK&q^l@@cG`pX*Nyly}s(RlLvz%#!d`o@L5Q!!6LPQ_(81qN#m#K?(9`nSE%>v^Oc{ zB0f;8q^`|k7L_!S@u#4RtSIhl1Gq@coAtAJmkJ)%;1~9_V8$=dTKt*qDN#Cydp|#b zgT(W6zc7IIuI1m^bGGy8C@U4{7JYJ&pSU_1c~Zir%bW?9UA_x6n(9 zSS%_1B99>6QSK;WX-B`S${%~B{`}kI$d*}d?m@B+NQ1}jD02$N8peL<#*$qFloJ70d?<{(@ih=dq$PRVA+rNkNL}Dt)vW~ZM2pTwI1WvW8+p6bEUwua)_tQ zJmnEj8^={p2G8mto)zX<8S%iHOJvRW7K2r`@3Mc)FZ(Ov#681bB^&>W@~rZz@|yC7 zf*o`6$c~|Xj@?B23A#IVuv`7u13UFN<~h#u6o>P~{h~O1PkV3|%y4q)HAeaxUZt1aXnhn>hbz0r?U!;YR12nPs(n<<+ zZ>4Fsw|Y2Gs_a7HLd=oOj`I&j=0veMaL;9^+|D6p*Qk*x1)BnuAd_|R|j zs_tuY9jtPLNO?@iNy{fep2y|VsI)#S+wR3FHKNJ$G?-dHYJ?jwg1Ym7{;S{|tHd&S z{;H%mosk{7q(2%=o2>5Cw|uW}LMBIE84_mq4Zjo9z2xIJ>e-=I-OcC|8qgYbF;3?)Ej zMUxlN9wr;Dt=Bu3unQ~4Lml`j)SC{_G+-874q-K;*|9}vbh@aH)$u?O+YODb?Om5o z&;kfrSxNN0+}gS<0I1zwD~vI|4-L_4b*(s!@fw^O`J~Y7+R|>hVS;qn!(u{%N@1+w zIW1RSCLV))8NVGp2b1kMx7<)`cY)=lv~M9aZ~*=ld4Z;mUf;38&0ZId+_5rjqOta6 zg?O`Ed7%HwnmkGKVY!ZQtvKJR?cpJ^^Oh~QTy`sAq6J9aq?ecm3iVFUL#P%Z(rO)O z)XIC(ZV*sEaAnr?meamVW1oZ12JJzFu?V3 zrU{W85O4H^xEkdIx1_mWlFNv+QJ>;+mdBXja*oSsE>AK4Bp?ysyQgjIB7ZJg4Px_w z2FR-R4Ti#6AK?T+JYfVGTv=Q>F%wK7B*^2M_O-rA96`a)cyYfVX31mBDF_q_Vh&)m zAc{;0m=*+}DCj~-*@^`-422iP5<-9(4gqHTnKxquxPxL)3g!X3l$|7i^9R03)I)eN zP~N);z^UF*{aL{BBA~P)lHbzmz&-x_zQHx7P=+-nw2z($0uWgkJ4S!XUmDfXj0kR){iS|htN{8>6aPT#7d}_NV~3=e5GDY< zCco>)hr?Di}nzkP&O*UkiGZ%!QV7dhlwBYoddw@W^PgedsnfN4&gpgV$ux;Nin z0LP-gh@G_JE%_@v&c8s~)!;bn2OImVIOtB|ejFeiWzwv_JSvG(_l#hT_u6UNYgnI# zAizA-w}QTF;xX*IJVJ{#4kLhA#m^n$-xiN!cOL5(5hgzY{)3s(d!Qk>T%SWy1p6*e z&>qhlu;K}C&SzOX4PnU%9+|ZBA+&G?b3Vy4KUGpj!6Ib-&`+}tb@^xB3amUmv~yt# zZ|1&%{ZJ53L*i!;te->xkwieJ%YXJ(eLNwQJlQtwH8m@rkC#j6EF3~@QkafDQMVb^}66c^zSPt6y zGD7qO?&m4U;~+W7mNKA)(?}7ZpRXgG5DQMhBUIQU;9WromG@88k%p@L{FN{c{IS__ zo!7(kuvFc2>=s8s2UX3E>oSfBQ|(?0={W-ZRt6`=uw3oAj$DF^i~u!dh1iAiQOVI% zn51zTJ%`QNSsStRPUYTY5cl(ao$2)SrB?M!0wsru|k$RK_wirmE+@o@t$QY0R9-hWfYHvBh5R}zJ z^z@l9YCSX~joe3Q*XaUB5pwv1f(8IbT{K#un)=MLDaXajrlX0MT_)VuSMMm>SS+Ex zJ`z;ql@5biLTbBFrqKvvoB|B9!lCpb0LN~RV#zSZu}+xmwO!OWd(Kc)Xp=C_1H%6K z5n&5YJme>7krm?*ju@kLD@llt;Q`WofJKLkZM)?l&rJ-`uG=$e-SVoHiUK6r>3DmY z)iRRA8^EPZ_KfxI?$)Pi<|vhLJn|quJ1*W9f0voykfQCFI z1sF&Lw`^!*eSLixb(}s?uW*=g#26|>BU@c1HIP-1??n`W7fibn1tn!v-bd7MGg2Hq zM!+X=u4%V6g?;`u87BJt<$M5`wQA4cTeSH|QyYQ5>%dBn{Asj5xt^ypLY(x}=~ojBy4zjJG@0 zElIeZFgIm!5xv>OU=XAY)fdCK+o^j{sU$`43lf@J!2FJscZe}Q+)H6hBC&u?=3ch} zPG2W(3Jb!N=(OPr;3r)9Evo%ZB5zYu(t+b)SR9&m7my59j&0GR2oFuX@kfZ2LN zO95Js{{YQ%0N8ng-Y9>YhNI!Pe2~M zX7x1c(hS+tsLO-X#M2@}_>{Vi)|{SD<=VSUJDU6tKix;N&Nqc2lJ|5zx{=KX+$}QfO!!G0$Y~K5 zsdr;XGMxbE+a_iE8KiyV9b|I*1n`o`k*3~O0QxBVKY2TWyrxM!`|Qw>(Vyh3amJqz znG#MYz?&?fQAXtMAzzPEYl8CSs?gw|ZHo=}aV4Ug#sEDai3Gg@%(4op7kp#X; za7IiYkpS6cK&m4W07~Y9sRIZ*TC<;no+kaN{-h`%W1m4dLXbF1cR*=_p>$46FqEDe z)d_rO;O_)^3TPnJ**6iyAOsmo=9}|l>HIuN+MgaMV&U2Ji--wmjK+`c>knAw`BRwt6k#P-Jd8-Jx{h ziY;4Ka2G8?`iChiEv`~AE;C&G8Q?Z9ugU= z`A2H~hzPCm$mHOBH0WU|%GB>YB3xp^Grf;G{L_IVKS1FyHa@D@2=U0@0h9bTk+E(n zMTRu;9@W}JzDMMDiTnYPKP2)1f{5W@!x!W~K@-xco-I9>zWK{O@e)3f10q+66p8#X z5yJ8EFF?W+W1?s`{wdWGZmCQ&ZjgV;9F2Cj=T!*~9fA&QU4{jJN4;(m;S-(C@#v`k zE0ipg(|xY|YpVVmkkBAJZe^>{Zh2{VtXmU@GJ26}*^;cI{3as@1xX-&|Aq?2@{MT*V}DmW82pB=xKfQe;mjCQ=Jd5n6l5~1{T6uuIQU_lS<;WE{cw!_3ZL_@u%b6&-h iOW&^;{2=d`hI#>Sa3>o5r2$>i`bjmVC+PQTy8b_`49Nrl literal 0 HcmV?d00001 diff --git a/captioning/utils/__pycache__/misc.cpython-38.pyc b/captioning/utils/__pycache__/misc.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..90eed6bc851640038ee089266da25aab95082480 GIT binary patch literal 8382 zcmcIpdyHJyS-r>)nmluQ+2TaXcyBO@KP7>pHF5Qb3hh!JH4lNrKZOF7sQ3d>qzY9Cj)a8DAHjfx5FjB7 zVSeAaGqVqGii*S?o$tKQ`ObH~*ZJnveftU;zQ4&I^ZxX>ru{utwtxFjxquWu&oqr| zoP}D8)fl6)jXC`(ZGBct^bOQD*{sG59K!7VNcqLc z<7KbhQ7BCp%J{B%!fW}7FJe_aJ^WO)akktDLjQCaEc)eWZo#i7s=b#By(DO#ewrxY zx9brP+RbXkltvufl*YA>|B$eNmNY}`h%u~jM-vaB<=)j9?j_y~D7EY*{!HuU>I`Q4 zGcUBfrhm!n5aF=abZ0e*M16jyK5UxlCP5g_w1T*fURlge;nEJYa%=-=w#`O#F@;iH zr+WXRG9jc0b`k|eumc&vQl>}-B4s2?*(S)>C)!;3a<|p_zK8}oo$b?@ZztOO5%Vr# zM}+K%D2eJ(2ttbeXcOZEGD31-QPHJ=#l<8w)0s1H0Hq;0XdLuHpBy=xjp^bj>O1&{ zmNbwQInJPvXd58HCR<|zu36JTB6Gt^b+j7jw^L)2E#yEf{We=OQxXiLDvf%l3*2q) zF9&tM;z&!~n>dD5Gt@0YDVD{HtBF60PTaFBHbm5N=emssn7lweulVga5~7G{6-%1L zBx((5_ebU8{#-lBhVCN3MC^;e3xl5TCXt(snZatY(wD5D7g&jnGjR%SJBUHl9}?p* zDyg=iZGs?3`lbew7#k*d0)ljhRV$Ww7-OW>#;I0iE{W!VecV4PF=JfO#S<7J9w)F1 z%OdxOcs`5%n8a9PV@#YzzJq6p6gebyN&kpK$_TI5^)-FXXliR_!cu*mUDn?C)|$oH znvJ@l>KE5=O{taU8u}q^%}E`uFPfr}=D4wFiEF9N%_(ioPVE#NpE?`Hrnb%wY1FoK zifK#X(y7EutPPu6+}^|<;xD+OY)03 z@h;WZ@>uI@OYd@XqmUN3p3O<~Q`#qi7uK;;bEAk=ORy1;xlLH9rGtvGoT}6WX}P-u z?y+)h{Q8xbUjDHw?!}K@cCS<~zfiq8E7?;$$gdD#nN1epqp*4(?N^O3BMD1;OAnk5Zok& zT8RY5%KnmMRmm%oQoB@P?-LCoGX43 zKm%Kv{ZwL(`W?_Aw`eT@jx!s-!)%g?7cg2(1JsqSqsjR4{iOH=#u&n!TI&#J-1#Ke zQw#TMC2T>Ng9Y4QJe@EIHpH7Ku6n=;3nn*G>wvbYgOhR_4!AA9sinAoj49v_aZ{VP z@f|a){*0WOVW=57HKVj`U0QwLl*V&3dI~#mXI+11Y|TkVQt;n8c6d$up!TEMb?swX zYKs3#w$ILgkJ;w#*<6EKDpVEk++(ME#TJMjJ|0hZlKI|#RVl~bvd_y&RQ}1e-e?4i z(*pB|Y>c#nb|Z@2o=x?|Q0gmlvH*JwlU4kNwT z0T(5%YIj=RqK}?foTC|!s2Q}kEW`lp zAcZ(~BHUzth_lQOF%uu5O|AjdMuzth&a90{MQfWR&dxB${pZMy}>j{U?9MSu}~Wm#aNj@tjG#1$eVYmChMA&uNr74n%ky0I$fLq2EU_%xx7*A_9 z8)abBWsHJ!^hZ%Iyb#pji=%fAy}3PmE%e^n(Mvnfe3)f5-^Pfq?lA)1_z;6U#(W23 zoZ%j{O1?oD@5Z*UsDFkDataFHdvsFd#f=i-K-%I>rX;ddg-UJ3aBws~+mu6{Oxi9oRL9V4MAbho=>UO=bGw;>*Wfi{@2VvCCw#hnR{FTa8bF+CL_ptGwG;ns=r**aXavo}_{Kls$sXPRo_jqELY*8*yH$+uDG z$8*xoww6v9H6g`wvG^(UN(c5ZXata-omyL@=zRw%b^tWH!0h{`u3C4Hg2+IetVlsa zacmqP$7a2Y>O|N5q$}ES887d0|9;DafHxPwe_Sm`4LA^sc!-pPcG)Wryt?ch8TN+w zX{;=W;}zDsas323zK9en`7#C7>qt93!t?@Cd=bFcYEY*&nAMt*$vMI{Cbtmoun<;( zYDdVzreF(Naujeu?oiMLd7j%i-ffPB$|6s63<^J)=@H)m-QuK(GxU*a(oANOMiqj&Kro)iZ~zVk>(qEf<{7k@kuw zT3zG~uMYQ07L=FcX0E)_()1FH{jee4#%@B%(Wj}TtoF{UmyYWO?I3a8U&aWf8Z_lO zllN@1-dO+G(}Q6<)F~Z}D4fyO=|1BD=rFqvNay}Nq<92C(;Xbm#32vd z*Y9r~k#>IIgd|RvXy(9lz(-^2i1sMc0qd26+vg>TP`A*>L7HuZ$aWzii>})r>$+d3 zB}p_i)|(j6cw7GtUZZhCJO)0q;xD2nd@u5v$*jU>R^%Bt$+MWK8V0mc4Lij^P}?6P zW#KSfdx7=F2Pkd5JKmo%B-jvLwrGokgv%jtKO_L9kD#=`Hh>|$8%eOmCI$mNWl2dW z7m(r?0d^?nLloBaXx|EL2UaWegGr$e2O1LOMy>9 zSss!5ulu}P_g}pJa{C&Lr|)&e&tezxC4l$7^hNr9|GB#b)SiFu^rfT!^xZRupOc30 zL-p)^@9d|a`tIASSI*Zdf;9N3OiX_XDgI#qc=GBlp|YXL5G}IPTXEGTmx-^Wu*bnecQDv&x?5A z&yo6qP$vC%_yNN1Z4L~~qom9M1!2MXqyPuX9l{A;?ibKeMXruY+y+S?EiFDz;9UZ{ zgp&q+8!6Kjdhfu4_>g9Pf8nGTI^nVbTvYG|_I3e5KV@%8AjuPlvG^TinE=ogWElP~ zb!A#r8p~d|3pqJ5ygX%|zk}iZmv4#n%6nisxO+8xhl-;Y0-00AY===GoRPqA2q^($ z$~pTo{V$7Q?hd^$)-!Fu#En@s80QVjnr2+WGbBh z?j7CZL_@XpzzqyfLvJmm9&_R`I@OMwP-mlXR{P1J>LjPZq)}W2J(;3~sv8sp%@k?H ztY!!LI+hoI2!KmeeDxJ-s1w*a@cZf1zs0b==i}%lHfB)#uy_2ygYMF>|1X|^_yQqK z4zBn`0=q7S6!TrAy=?d%V)1{l=Hkn=?Jw=I_XgVj6=`6yAAXQR+l+T5uN1GiLo1PA z^}P5^D*hG$;`9vsHcDiwisDzO?N8i8LYuw{|{fT~-@i0F_Rs4W7il2T%%RsRf0 zz4gc}6)eZZ&vHryN9kO#KnsvBdH37!-|EDugOAE&C6pLGi$bEJy$c4#X5qQ9+py*M zhk!xLQyhvc0aVFG4#J5r0zEBmo9pyPk4<_`?X-wx)PXB{^Dn+d-#QpVuOE4#zgD-s1GY#;)RIMt*~N*9ulze2vKO)e|M0%abi4!$Z2Z3XCVJ>aLV zjP-4qu|-YsN7Vipc^i!`-h%Kp1bJ@4AV8FrMf_D4;vZ2wZsRPx>);tvpPz0cg4=G3 zCuxx0?#N=+tQ1*xauq(F7>u4C{A29l?AyxhdG$9S%xZ9Rx&;?H`)}>glHfP(=BRk%%taq;$xH(m9v;Tjz<{W8$L{ r9uy_`-Sn_fNgqe8q#t7i!znoMXwp5ZvY12oXDt#wSTRBZ1ZqxTGY14Q5rzh>n-{MzK+S4AjIc@&*v`44sq~Eqa!wA&!H;K_2jq~<<9ARF%L0Cf))}!zoxxj~m=(ZY1s|jbjGwF=q%HG-W zX69Cw@6c1-JqYCfROF=GQ5)(~n(9)TO8fTHO)1ITwM=p9E;x`WW;+wj(Dh@h+VU&0 z8~96=EIrVF1NhD1a{-^|S$wU7XDj(|O6Fe8-OS$1`7=NUz==oYphorq8A6+rG7ki; z{DK@-&(?G6*(!}*SSk*7MxuJKX2p)@Ef3qCXSt19Cs(a35BbhowNlJIdwqGrv;A7j zt~pjCkd6oJ*7y0bdkz*RSLA>|2@QKxe#SH)$cNM`ySdIhGyAojhGs zDf8Bd-PTH|OhxM=8u`}1W(VnYdl(`og81RCk;`+5xvmFOvr9P(#1Db2at{2S( z%{ao-&NjDdF?Gj|hlo>?_{!u8+0m^2;5ys}tU?dcKbpxfKFr*I!Bw2OlYKLD6HI7L>fnplSHXvhi+(0IPY=^4!dhL)?a;Mz&#(<{#09s_^ zZllE>AP>sD2C^?6l>6lY19=E-`sa5LNdNo}0qLKg0Mb93hk+cHj~J96jfdnBdDK9T z0hyLX19=R+9+xu)GRwV61~SLUyn!qLIWC_tkS7^AVIU`g^pEu}AgAP0Mw`=&oH3B6 z898ep=NMTukY|9Lm+v-^3qUT)O9t{R*7clx-auZ6^YT07dko|6cP`w!-!~UU&jb9%kMPWz7OO*WGWmz%W zNFa`^8b}RDUAhMH8jw?R#Xvkp8V2G+V;hFX2AE9~NKdC%;Q7xp-7gq@eu0r+ zG>~5c-HXzk%`b!QTYYqYh1Y4$=2y|?i+yc=ZBv_H$5`k4+WZEOuIa=DV<-J4=w;+@ z8T0yW`8)D=YZ>`_^7r2uqWyMf1ok|;o~7VmW#sly^etR(IvG8v6pUKhoG9jeQB&S2XsZ#=ZvZk2SUz*wF2K_>Z8oS5B{B%;=kV{wMnR zKJGIdWq|#u#`bG$7qCCm*a3}A1N(E0J*2Uxf&GQX4g$;F9*$nccn2{aU*q{->gR{_ z^L0Ewq@S-F&xL;eIXoBo`R9!158obceo=lEPY>^gysU?D^$4)9;1}1{Bk`lSI&v%f zD#9G=C_cwR{3rYt<<&jZtGJ#;0Q6T&_r_*tUBB6it=FQ!PbN4PsRWgv#P_G0 zc3c;6AmX|s2oY7+b3`TZW83u+g9uwnmjWHz6RuwkgzZbQ8dU7^TLYcJxZQL*!_ukR zEidj2)*Y{TZz3JATCt=XCX?MEW7}InY?Sm3iY874kx$zklxEWmon{zRoG5ans8&Q3 zVNY~#drBR_7L(u6Pabe`#$41pqmOPI2Pg+O?l%D75mZHw_+&bJ&n`Qj zRX>qTb*FDJR;r)4chV$ZKe16Fo0vl+A}3Bjoh~gGQK{1aOFN?xIO|w(*m9E6b$koA zD%KpaX8SQoJ+fCF7CkYy6=J%sFI3-Ru9LSpRu3u8OrS?mY=;Qp!=RD;2Wg1wgJ2wL zMIoFCO2JVymJQNwBxp4y@e(OoqZP%eoLSQ%-J{$ld9S)bE9z0y;y*1$#TI|ZNF@;* zuXxA0El$$_Rkzj(ZPME_TX*JZmYQ+h?7ZfWL&Giu zO36+wQM*-ZIM|Z3pl6fEQb{5k5H-hluu|+1lD)7aVK$~2_qM+!?PNy^sO?G6R0`=m zMHog_{ZdT@NSm)O?FlJyOFrC9ux12bf{f#XMM$}3gig6;u!V6G$H!ei{XMQO(1Na{ zji>2vBbX5?{Q}L{uB`Ag36?`q(Fz>*BQUn%gvsdaZ0P$|7YP3=!C9Pc*A z5MEGoV=afIFgr_WZ`lu7BJm4Vu>cLILx?gEWjn6aEg!02dv3xz1lulzox$2@SO}Tlj%Ztfr z>I{Xqr3)Q{@zF+~o6XTQiNp+as^MVEf}o%tO(+)WuMC5x1v_81yv9Fczv1ui7ZZ4z z(D7>!5cVrI30f>pQs&FLR3olXsfM>9b{H8@D>a~&U5&Au>*wi5tkp3mW{PGiyD{hp z6t%@dC$blomKMeI?1DHeq#M~~&p9J}XBGW*Lx3z zYXlP(3Fk%=meE~xO5#$fR)Pbp1fmXCZ~~qMQkmmXlHu7|)RNprXAs7=kvzqmft7{# zps7Ja7e&~JAdcpGRFm{+1dVkxCi&7Y-!#@pcRj}riF+0~pW5t&1ONJslcyJx@e~(? zC;K-vw4_t9+ZKx;c>#jqIk=FM+PTCb;;Gih1$3SPO|33st_&0Wj(nsvMOH(#T_ zZ!RVie9sbRwqs+m6s)on+qK6(vHS1-_CNo9F`1%Uk^zEUCO2iWB{!agPwH3HZqqDUhkqmliay(Q3M1MJDA#R~-WMmdksO9_na$c$KX zJx>I%8fyr~9ZpU*O^I}z=ZfTp5?2coR^FS_sUB(ehKf8t;bqZQ>ou_`IZBT9l2lqA zH4J?ug+b4@#W_(S(jn-E(eSZ$Tx%Q4$^00U?@_ z(A~Y{?yizbDha93CGzbivVoXIP17K&hRxz3sDuLO$P0>A!>|!NV513zAQ!9sZ;_2s zfcT{BM2I#iUrJ71au`7UowH#4Vf)TA@Sk+)@@fIdC0&ML`fHWur<4+?FgNf|J%e z>_bhT1Nx^zi$mkrW5ef3m&;$5>$PXN4Yt3;SCCD3S>`*B(r_PEuWS$hq zRZM01H1R&9;l%YocJitL>63a~cI<{qnaDy^lxM)O2@JOZNz3k&SwjD%z9&_m_ z+ztcN8S-f~oForlsMeNXn3DbPQxWKCVv0CIxrIvlRJ#%1ZYU-s52y73N_JbG#P~>A zkaMNWnNJ?+CH`ACno%W@G;3!S+Pd{4z5HwIMn_qF$loFZGMXV8Tq7pUvjtj69yJmS zquGX49PVoC)@@4cy9^hkJ=9j5DQ-0QTa!MRdLeiQ*v=la=P8$5ND8GnavMjY;$xix z#M!jN$mvX+*P8MY-G-epl$@QBs1$rf?fQNw0%s3%Sv3?%Vk-zYD#mL)>!kZQH;7)2&8Py*BsVjBH7vFz~kX z+Nz)KWvo)S+yWV7e5E$9VqOmw^xa@Km&s*w zgZRFepD1J>%#Pu|sa&3aC)4kNU0L;?(7(H~g8lglfF)nk-Hdqz(960vd2Ik+=T4igzVdd>}RC+B27lpi!=x2A%il#Br_!+ zHju-N^j?z5$VYj7M~pT{d40zeA~QH-oL&kKfWfLddjf&)Ie0COdnpzOk$a;UPVvH; zi^!Klcg`t##-7-%k>rL=1PEOM%3o2JEwsi&9-pQ4f*p#%k)Fu8#q zO-*hKGMo)wd>`kYsxn9j@<=NFAUWNND2=&+B%)WQV5#|skJ8egLy4KsPyLcUqA>4pV{lhhkg`R()l8a^!hkgr%|*H zTToYSM=dBbqiTZYPq>4`^#Fa5KL~fy*Dm@(1tODth$cbZXqwFMjfbRL9$S}$S~W_! z4o(hKYNZY=4TYpyY1Bm72WQ-fLz6|QHL^r57t=Y)wF@s^cu_M5RfD<JL&J4zE&OSROojU*lF@4_h1106@;geH z)ikQz8ib1e&WZ)zG%9=MTsu(dUIW{LG8L6LLs4NS$^k_Y$jg!vm_tV&$iax29ApVP z06?7xiew{7M(G*+rt64L!P5S;>64St@$|Tr%ngbQ>Kv;U1}!??30u$|eeg;VBvq&a zzc?74ApJT;U#IEoB);CthF9pi`#PtJRwldACju7Q-}~TLA|EX?+9eteaWgX57Z7aa zus8SH)H1w|5w~`Cw2Up?UGDAN9UJyhKH*HWYh@@WFLHEEr7;|2K&r}gQz3y*HY|Gg zfU@rJF$|uZRp^W;qXH!93rQhfeZcu$&_qE*s$3fvX+xy;)VBfmQiPDrW!KUewS>FJt z@TEd`eX110^cctUysN1uU~F|VcP*|cAE!j}34s#KimG##a1IC?$Nnrv9~^auB|!xf zHm5Kk$_iEOh_Zdya2$ZCa)vF+E)Jkj&4G7vLO6IF454I_p09d*1bLUdPI53UmMbtd zAG|OsU7;4Gp5PD=uS&&jKaYgw<*l;h)gEGUZH5hLI-r(dTL?;;X$K=~=F$|3k2MUBzi%xj4Fe@V zh4&D&K~WwVMc_DeQFY?zY1&00%V?I?>Yz(Xx8M;kb9L&hI2_qkCp>IMUnQyx;zUus zeu2y%wDHzbtU}f!a2VhvRga)!!5XMCP+V?pA@4~0>UvDHM}g_`WkOYcdh zmMA9DaUM!)v8E%wVs0;K^h%1hUGsDMxGaUZXXoLjX@4@Xv`Gov| zgX_(K!MD)m2@pmY%}G@K)1ehvwBqf^0iBiGLpO4VUgS~o17S9E&Ixn4b67j{4}&v1 z3fMaHSnZrfA^U*&EC8;~Hdx5&z^$+k*$Qg_*I>8UDq90?mC>I`>!TU$PM)?b9X?KE zmOjqQ^a%J;shk%)JxWo3z2UjB$=g6Ui6|qH#hf?ffHI5O=Rst@Ar6TgSl!j$GcKQU zHT%;hB%M#i=`M5+I>?E92}BV^r{v7Ku%>isP3=pY5@jDc%q+_s=KkhPN$e^Qw(-ua z@5!&-*AATJSXW^a*0KMq*%4`~<&ulkI@p~9y#+;nCE5fsStmZR%OTogQy zWj5j3lS*Wa>)I$`EGq^&NO_jW!;^oM zB+n1D`@)3CGuujCFUG?-%@ZlP)HJ)II*6-r9AA;Y_B$+5NoORo<3#b!@Z_|Uf*YOv zVKU%*$w*~o0UowTr>Yd`%TD@E)ETQRmmR@n!4>Gm8XN{amL{+1E%;XHCY`KRA#Tr| zY*ArAa?$T18FKJDj1j$Yj{Hg-zZxfbr7^4O#@i#OJkR4;U}jb1Q{+BFfSQ+i=3`@&H!v$n`3y`+>(@haGp)4j5Q z(2r9kn@O{BYrFTLiGk1CVc3NBofyc6hQrZFa3zwg;H-J~1q8D7u(^NGd@jcPt0t(% zqQI5Re6-!38<$1XL;$XaS<9r7_A9V^53W4ppFo%NT3#kBmXH`cW?8CqVAhK#WvP9m z8;e|rl@M5Y1(*5#!L(~M*FZIRehy2oB2rMB0Jjb5Ks*IX{}ZTi)fl}IRA&KH_hwLc z?k@vW%!U~z&J4LZygT>B9T12(zwZgzW1MQrvlIHrTQ*RJH7rTYE`Hm+MM literal 0 HcmV?d00001 diff --git a/captioning/utils/__pycache__/resnet_utils.cpython-38.pyc b/captioning/utils/__pycache__/resnet_utils.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cd133081fd55ed84811ccaf2ae4cbf16c7e10d58 GIT binary patch literal 1117 zcmYjQOK;Oa5Z;GhNt5y@2gIQg7bIU|6*yK1sS=0-MFMg0Wwm%W!IdBB+73=SJjGMOqH-1k@y7>g2K6pY=Yzn=oZ zOCHQsyx^q_(yL_<%Xjd=up1EP@oxv`Cf~au=JzC>yTB8fxkeco>?P05j?H+b| z*zIAjhX);|%}|gv1_?QugEhLg%26|}X-#S}BWq&LnnUOv1xEDRuC3ah5~#g>t}t-j zm@nCw;%|)UKhT3JD=sT3SF$a-wxc{-oM`$jJJFVsNu}MCU(NG8(N4mbQk``AWv5?t z`n9%F$usSC;WU1ykGL>RSV+bfGiHLk6xxOUr&TF^N10k`8mBYu@v>w^ypkGUX)=vM z3I}Qc{;e>7jCQBF`oWbjMmuT<56K9(&t5;l$7J~ z7gb{Lh_TCxCtFWD8S1%7IqoJfl1ai?kB|#qWr(z+YzCqoBM*Qn+sUu{@kI6r6i5G(a literal 0 HcmV?d00001 diff --git a/captioning/utils/__pycache__/rewards.cpython-38.pyc b/captioning/utils/__pycache__/rewards.cpython-38.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f12b33d95bdebd43e7b6177eca75afb28483a1f GIT binary patch literal 4770 zcmb_g&2JmW6`z^?;BxsPT9zeCb_^v2+%Qcfx9F#<8jkI>ja#^fAZS7)AZX5tTA5#( z9XgS)JhXBl>QJCXuRTyb)z|(v-9s-u`PQ>9?eER3DB6~bqVN*?c0S(Bym|BH_nYCT zl}gC){JnUs#TOa-CuQazADP>DlYc-+CV7|jc!P6F3#3Iun6!hm({N1M?Ya#Q^={X1 z`1pF=LN90pd0nwlG<9JkJh&Z&yQOJa`X{!>LbuYZHmXK5=q~hXjT&dUPBGUByBB(k zjYZCW&H786(J#6vONZ#;bJiE1Vcgad-b;9wWm#69u*Qn4$_0Ec%bL7^@2Xss7x7(_ zOY#!FSETz1t1myp99f-b3$2IApgYo0GwuxsN~0){@#8p&2YsZgLlyURb9>ZpYm_`d zQR!DoMk zpSfsR_p^$%ubH$5DpHEpIxCy9)1m?umB`A|l7O1>tkQ0^cOu!;gGW(6dCJrR%Aeog zlr7!bgnu5lbhO#~dVjM$kkRJHy;di>Yp0DVZ4CGIK(%)^b5EdXbEM;LvZNb=sY3wn5g2XE+F-SU&tpOUgMWLNJu{BnvrJ?`60i@zI=BqQm#eHQn89M zDOT7I*ckPcm(C-h5Ndlj_i8DhIw#~U_d&*<^L0f&ue+J2TK!IxIo+tAIf+)82N4hJ zPUf_f%ACPSXFN83u%d=`Ss8V$eYmL!?YB{qlpz>*d5tgeGWWTw)HQBfGxcn)o==<=2Au;liqHjXiFNX^bHt9E zZ6Sqpo&>hewx#+*KEX(6``06Pn@e}5gRyM=*fTYr^lj_Zo7MP7;>bhK1zTtBhgS+| zVOl)mM*?+$t+Tctazc#M*$pLsbSx>$?b;B6O>{ z?qBqR(e-oPP}imx(olx0?1Ya)v@E3o-8Bdq6!Ct50ndJ{6hRxRLQ)Bm&yW&5lmg z8hV(E=y0nWC%Qf84R8I32B66GnP3O+cOvZ9sJq>4t60bFR<~)l^Tt+pfLwA5iPNU9 zH<*Sq1}|cF@`*84vOf!J?o(ceZ870VGnQE)M-V#;p{|jw)WcJrIVwud>{jof^v8H- z2EX2fWvIpEv+9`%e>(G)?esd7)Hg{G^eMC3!z~E{FuvJBp@A&&&Hw0~S>snsBRDY2 zNpmx%&Ac-K2&}Sm&BHP}Si!86M0?F)q(En#Giy~Hef{J`GhZ2)K=s^#R}e`QN9PWV zHCpIIeXM^n>LMuOgGhY~J-syet@*)y_T2K(x}l@w<}<>-=Vcr3v(Gay!cG1vo9;3~#qPVl8 zRf$?xXa$M>jU}09_&!^FNHl0e`c$2uIure2<{6yN3Yf96s?Z0MeH|y6ucBep()CIs z?5HS``{WeE1X+0IBGAWBXW_$^Ztoa(WiHg3ARWkN!^J8#Gv3U~)@JLWtYj?8Grs*c zV1nULG)v7_VVj?E=5Rk8-nqGhoGxP;qxkfWkX(KM9NJ_h{&BLJR;yqt*uxqQd8ftKwhCnf=6B8srYWa@0Y zoS4uH+I|7@Iwhuj6e3>Kn6x&w*QGM{C}W*TdKU3+V5UTG|>U$)W7pi zv{S_5>Lv-IM@5jPh`khXLp`(r@*x%6A#s<)JrW<0_$i5>k@z`@k4aFZ)IHnqwV(s^ z%a8ZTC(uu5DvUrz_`(284lc`l8Q>DW3@u_ws9&PRE1*Ib6tXe-8^nvCLYILxHg-E~ z>;X~)9rOuS04d%nq>TO4oB9MRsZX#%XUM%9Xi?DK+;!sq*w?|dn0l$7y4Zp~T_`9; z=eiETSRftgK4Caj7N&k$)aIs(4Gqqe-W8l8^S(dR;k1$#&;ln}Y+m1PUiEF79P!=h zLum>^PPI49tW`K|tXf1I={tbFCpuoTp2iMPN zh3>(YP3Q0-Z}g0)pk7tKz{d=9UeIQKYdDPh(ondfJ5;06+s}`i)VsCb@5cRBx040A zp*#OAhz$W6Dm0`>p_@6nH&o=|m#28?+{tKYCe+$SuA4_L-R{*iCVl@=q?i(vmz`qiwUC@zYgM^}uWO2d31o_;81Kqyu zIuNUyU1%mb=d!cV2)23yIqF8YD16N(FXMbr76DEaWgg%wC~tl>T!*VTHQ?t2^t>0m F{U6K31>yhz literal 0 HcmV?d00001 diff --git a/captioning/utils/eval_utils.py b/captioning/utils/eval_utils.py index c4bc7f44..78abffc8 100644 --- a/captioning/utils/eval_utils.py +++ b/captioning/utils/eval_utils.py @@ -17,12 +17,9 @@ from . import misc as utils # load coco-caption if available -try: - sys.path.append("coco-caption") - from pycocotools.coco import COCO - from pycocoevalcap.eval import COCOEvalCap -except: - print('Warning: coco-caption not available') +from pycocotools.coco import COCO +from pycocoevalcap.eval import COCOEvalCap + bad_endings = ['a','an','the','in','for','at','of','with','before','after','on','upon','near','to','is','are','am'] bad_endings += ['the'] diff --git a/captioning/utils/opts.py b/captioning/utils/opts.py index 282bd72c..562760cb 100644 --- a/captioning/utils/opts.py +++ b/captioning/utils/opts.py @@ -38,6 +38,18 @@ def parse_opt(): parser.add_argument('--cached_tokens', type=str, default='coco-train-idxs', help='Cached token file for calculating cider score during self critical training.') + # Data Augmentation + parser.add_argument('--data_augmentation', type=bool, default=False, + help='True if use generated data as augmentation') + parser.add_argument('--input_json_da', type=str, default='data/coco.json', + help='path to the json file containing additional info and vocab') + parser.add_argument('--input_fc_dir_da', type=str, default='data/cocotalk_fc', + help='path to the directory containing the preprocessed fc feats') + parser.add_argument('--input_att_dir_da', type=str, default='data/cocotalk_att', + help='path to the directory containing the preprocessed att feats') + parser.add_argument('--input_label_h5_da', type=str, default='data/coco_label.h5', + help='path to the h5file containing the preprocessed dataset') + # Model settings parser.add_argument('--caption_model', type=str, default="show_tell", help='show_tell, show_attend_tell, all_img, fc, att2in, att2in2, att2all2, adaatt, adaattmo, updown, stackatt, denseatt, transformer') diff --git a/configs/fc.yml b/configs/fc.yml index 979b69ee..744459f4 100644 --- a/configs/fc.yml +++ b/configs/fc.yml @@ -1,16 +1,23 @@ caption_model: newfc -input_json: data/cocotalk.json -input_fc_dir: data/cocotalk_fc -input_att_dir: data/cocotalk_att -input_label_h5: data/cocotalk_label.h5 +input_json: /data/share/image-caption/cocotalk.json +input_fc_dir: /data/share/image-caption/cocotalk_fc +input_att_dir: /data/share/image-caption/cocotalk_att +input_label_h5: /data/share/image-caption/cocotalk_label.h5 learning_rate: 0.0005 learning_rate_decay_start: 0 scheduled_sampling_start: 0 -# checkpoint_path: $ckpt_path -# $start_from +checkpoint_path: /data/private/mxy/exp/image-caption/da +# start_from: /data/share/image-caption/fc_nsc language_eval: 1 -save_checkpoint_every: 3000 +save_checkpoint_every: 1000 val_images_use: 5000 -batch_size: 10 -max_epochs: 30 \ No newline at end of file +batch_size: 100 +max_epochs: 30 + +# configs for da +data_augmentation: True +input_json_da: /data/share/image-caption/f30ktalk_da.json +input_fc_dir_da: /data/share/image-caption/f30ktalk_fc_da +input_att_dir_da: /data/share/image-caption/f30ktalk_att_da +input_label_h5_da: /data/share/image-caption/f30ktalk_da_label.h5 \ No newline at end of file diff --git a/prepro.sh b/prepro.sh new file mode 100644 index 00000000..1fbaabad --- /dev/null +++ b/prepro.sh @@ -0,0 +1,44 @@ +set -ex + +DATA_ROOT=/data/share/image-caption +MODEL=/data/private/mxy/data/image-caption + +# original +JSON=$DATA_ROOT/dataset_flickr30k.json +OUT_DIR=$DATA_ROOT/f30ktalk +IMG_ROOT=/data/share/UNITER/origin_imgs/flickr30k/flickr30k-images + +# da +JSON_DA=$DATA_ROOT/dataset_flickr30k_da.json +OUT_DIR_DA=$DATA_ROOT/f30ktalk_da +IMG_ROOT_DA=/data/private/mxy/code/T2I_CL/DM-GAN+CL/output/coco_DMGAN_2021_08_15_14_19_42/Model/netG_epoch_120/f30k +LBL_JSON_DA=$DATA_ROOT/f30ktalk_da.json +LBL_H5_DA=$DATA_ROOT/f30ktalk_da + +# pre labels for da +# python scripts/prepro_labels.py \ +# --input_json $JSON_DA \ +# --output_json $LBL_JSON_DA \ +# --output_h5 $LBL_H5_DA + +# pre ngram for da +# python scripts/prepro_ngrams.py \ +# --input_json $JSON_DA \ +# --dict_json $LBL_JSON_DA \ +# --output_pkl $DATA_ROOT/f30k_da-train \ +# --split train + + +# pre feats for da +# python scripts/prepro_feats_da.py \ +# --input_json $JSON_DA \ +# --output_dir $OUT_DIR_DA \ +# --images_root $IMG_ROOT_DA \ +# --model_root $MODEL + +# pre feats +python scripts/prepro_feats.py \ + --input_json $JSON \ + --output_dir $OUT_DIR \ + --images_root $IMG_ROOT \ + --model_root $MODEL \ No newline at end of file diff --git a/scripts/prepro_feats.py b/scripts/prepro_feats.py index a59ccb43..d39e2314 100644 --- a/scripts/prepro_feats.py +++ b/scripts/prepro_feats.py @@ -34,7 +34,8 @@ #trn.ToTensor(), trn.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) ]) - +import sys +sys.path.append('/data/private/mxy/code/ImageCaptioning.pytorch') from captioning.utils.resnet_utils import myResnet import captioning.utils.resnet as resnet diff --git a/scripts/prepro_feats_da.py b/scripts/prepro_feats_da.py new file mode 100644 index 00000000..63346bf3 --- /dev/null +++ b/scripts/prepro_feats_da.py @@ -0,0 +1,112 @@ +""" +Preprocess a raw json dataset into features files for use in data_loader.py + +Input: json file that has the form +[{ file_path: 'path/img.jpg', captions: ['a caption', ...] }, ...] +example element in this list would look like +{'captions': [u'A man with a red helmet on a small moped on a dirt road. ', u'Man riding a motor bike on a dirt road on the countryside.', u'A man riding on the back of a motorcycle.', u'A dirt path with a young person on a motor bike rests to the foreground of a verdant area with a bridge and a background of cloud-wreathed mountains. ', u'A man in a red shirt and a red hat is on a motorcycle on a hill side.'], 'file_path': u'val2014/COCO_val2014_000000391895.jpg', 'id': 391895} + +This script reads this json, does some basic preprocessing on the captions +(e.g. lowercase, etc.), creates a special UNK token, and encodes everything to arrays + +Output: two folders of features +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import json +import argparse +from random import shuffle, seed +import string +# non-standard dependencies: +import h5py +from six.moves import cPickle +import numpy as np +import torch +import torchvision.models as models +import skimage.io + +from torchvision import transforms as trn +preprocess = trn.Compose([ + #trn.ToTensor(), + trn.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) +]) +import sys +sys.path.append('/data/private/mxy/code/ImageCaptioning.pytorch') +from captioning.utils.resnet_utils import myResnet +import captioning.utils.resnet as resnet + + +def main(params): + net = getattr(resnet, params['model'])() + net.load_state_dict(torch.load(os.path.join(params['model_root'],params['model']+'.pth'))) + my_resnet = myResnet(net) + my_resnet.cuda() + my_resnet.eval() + + imgs = json.load(open(params['input_json'], 'r')) + imgs = imgs['images'] + N = len(imgs) + + seed(123) # make reproducible + + dir_fc = params['output_dir']+'_fc' + dir_att = params['output_dir']+'_att' + if not os.path.isdir(dir_fc): + os.mkdir(dir_fc) + if not os.path.isdir(dir_att): + os.mkdir(dir_att) + + total_cnt = 0 + for i,img in enumerate(imgs): + if i != img['imgid']: + print(f"Error: id {img['imgid']} != index {i}") + continue + for j in range(len(img['sentences'])): + img_name = f"{params['images_root']}/{i}/0_s_{j}_g2.png" + if not os.path.isfile(img_name): + print(f'{img_name} not exist!') + break + # load the image + I = skimage.io.imread(img_name) + # handle grayscale input images + if len(I.shape) == 2: + I = I[:,:,np.newaxis] + I = np.concatenate((I,I,I), axis=2) + + I = I.astype('float32')/255.0 + I = torch.from_numpy(I.transpose([2,0,1])).cuda() + I = preprocess(I) + with torch.no_grad(): + tmp_fc, tmp_att = my_resnet(I, params['att_size']) + # write to pkl + np.save(f"{dir_fc}/{i}_{j}", tmp_fc.data.cpu().float().numpy()) + np.savez_compressed(f"{dir_att}/{i}_{j}", feat=tmp_att.data.cpu().float().numpy()) + total_cnt += 1 + + if i % 1000 == 0: + print('processing %d/%d (%.2f%% done) total images: %d' % (i, N, i*100.0/N, total_cnt)) + print('wrote ', params['output_dir']) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + + # input json + parser.add_argument('--input_json', required=True, help='input json file to process into hdf5') + parser.add_argument('--output_dir', default='data', help='output h5 file') + + # options + parser.add_argument('--images_root', default='', help='root location in which images are stored, to be prepended to file_path in input json') + parser.add_argument('--att_size', default=14, type=int, help='14x14 or 7x7') + parser.add_argument('--model', default='resnet101', type=str, help='resnet101, resnet152') + parser.add_argument('--model_root', default='./data/imagenet_weights', type=str, help='model root') + + args = parser.parse_args() + params = vars(args) # convert to ordinary dict + print('parsed input parameters:') + print(json.dumps(params, indent = 2)) + main(params) diff --git a/scripts/prepro_ngrams.py b/scripts/prepro_ngrams.py index f7cdce47..d1328193 100644 --- a/scripts/prepro_ngrams.py +++ b/scripts/prepro_ngrams.py @@ -6,6 +6,8 @@ import json import argparse from six.moves import cPickle +import sys +sys.path.append('/data/private/mxy/code/ImageCaptioning.pytorch') import captioning.utils.misc as utils from collections import defaultdict diff --git a/scripts/write_f30k_da.py b/scripts/write_f30k_da.py new file mode 100644 index 00000000..acd0b9ea --- /dev/null +++ b/scripts/write_f30k_da.py @@ -0,0 +1,26 @@ +import json + +def write_json(root, img_num): + ''' + split original json file for da + param: img_num # of original images used for generation. + # of generated images is supposed to be 5*img_num since 5 captions were given per image + ''' + with open(f'{root}/dataset_flickr30k.json', 'r') as fin: + dataset = json.load(fin) + images = dataset['images'] + + dataset_da = { + 'dataset': 'flickr30k_da', + 'images': images[:img_num] + } + + with open(f'{root}/dataset_flickr30k_da.json', 'w') as fout: + json.dump(dataset_da, fout) + +if __name__ == '__main__': + # root = '/data/private/mxy/code/T2I_CL/DM-GAN+CL/output/coco_DMGAN_2021_08_15_14_19_42/Model/netG_epoch_120/f30k' + # out_path = '/data/private/mxy/data' + root = '/data/share/image-caption' + img_num = 22685 + write_json(root, img_num) \ No newline at end of file diff --git a/tools/train.py b/tools/train.py index 4d015a81..49e1944d 100644 --- a/tools/train.py +++ b/tools/train.py @@ -14,7 +14,9 @@ from six.moves import cPickle import traceback from collections import defaultdict - +# for import error +import sys +sys.path.append('/data/private/mxy/code/ImageCaptioning.pytorch') import captioning.utils.opts as opts import captioning.models as models from captioning.data.dataloader import * From c75667229ef6580198e77afa2ddae70ee32ca8c4 Mon Sep 17 00:00:00 2001 From: Maxy Date: Thu, 26 Aug 2021 06:17:04 +0000 Subject: [PATCH 2/3] add data augmentation for flickr30k --- .../__pycache__/dataloader.cpython-38.pyc | Bin 15996 -> 14756 bytes captioning/data/dataloader.py | 206 +++++++----------- configs/fc.yml | 4 +- prepro.sh | 18 +- scripts/prepro_labels_da.py | 161 ++++++++++++++ scripts/write_f30k_da.py | 17 +- 6 files changed, 270 insertions(+), 136 deletions(-) create mode 100644 scripts/prepro_labels_da.py diff --git a/captioning/data/__pycache__/dataloader.cpython-38.pyc b/captioning/data/__pycache__/dataloader.cpython-38.pyc index 2a25f1cfe16200a8dfab3f13fdd31c0058386d5b..5a0b1098cf39366b79a84f5ab6721d2e383e2622 100644 GIT binary patch delta 5158 zcmZu!eQX@X72nyt-8_ZbL0)}a z;nmstd}F?8TI+A-u0ACd`e_df`qb-P;wJxnQ?`;1@T$0CE9oHj(rCmm&q%6QQmd^H zM%B{{5;c~G`!K4RR+$3aZ?k+DZ!O+By!F#MgQmQaujhdgZI+o|vLg9tg#vSUXhwsc z7$`Ab6Ib$6@sheQ;0jkxsB+N;TjgPiHDZ}sNHo%tQnD@%xl58wl2*Tn)?6@n7|S$T zs?B(WN2h7g7ONGqNN013b~dyTqdWJ6>8m;!s#}77D0IyOJGxhk=&kS*l7FA%SM*QKlY%DrZYkHnX6A*f z)z7jc7r?)6Rx|%Ck2QszCqY>zqrf}ov;3H&V9e$zN|s}^yb@C@AlnAnF1+1%S5CVY zMaYz94qpn9l@zg+5b1%E9vLD|gh>1fzpuI!A*->(5=gJEL`V-`93g9@ro|C*eI-Ix zPgVTfwaCwFE7a~qF|kg1e}(k^dMw%w#p@mKOMim{e|J0nzQSdi`6?tY`5gF?s6yvH zs~x%HRVeC^ilx9bt0?sF&I%QgaRqg0g$n;IP`QGvzD_KwP6x|IZY(pDEtY`+_NEt{{f1yD%|1DRzd|`1WCJ?k8}!FgrTaldZRAtjwL`8J;y0 zJd+p&mMaV;hSw$rbEDZ!iOJ+x=2)VbJ#sW#IGjz4W(z~sa3WVAVUDSZTyZjPW-~m| z!HFgfN(^Q*)=@KC6iD<^*nrHe0 zxx&OzE1fTn7ceZlbA`ciLaLlI0xG`fTsPGp8eW?o%^b>(rfG*H;chr%6hKMR5$7H7ioWD%95yv(oG0XNi$qy*J4cn#vM z@T$A=OmXTmGkb01XkH!Bfhl>-8O0Qo7Zy`Ws9t&3RWujsgOb;6c7==~0=3OkmgVN! zG<$^Ex~+~Vs2-~2$yeD_(!E0}cX542Eg6>2R%LGRlH2mn1#HcB&8sC3q?9}}FJ7|f zA}B%I!^u|f33ZSz1H*PZX+Nm;Do4HzPW6Js)U`zld|?q=jbU3zwo8lAm9(&&#N~pv z$5x}bd#+r#WXNq}v1@Z}qt8%M#!hp$c%$}EGGz9XBS#4g5ttxwgn$#-qPwoWr>vWq z!Vn@Fe|&tb?56)*!7|55@GwBC%IqfGB5*)FP`A13H$aP>0E!xhpCj;Ukoj3i^}^qN zJYm+Rwv+E!iz+^-Tg|>Dy!9QfS$6NRSXE!oZW7z;H?mpraQ%t_Y?|(KBkwjpwS&Ms zh=fn=syT{^D!g8YDkH)jBTx7)Q|Nk@H%5D9237um9Yka}vO#gKe(4Tq$2$8K^h=!l zCfAdispAOqs$QkA0vbY6!zCIhHJrK+=$#9T>PTF#wd$-|(G}_1UXR#eb}bL}DprKo z@bGcP)^9~IoMxpuOU-Lm)Yeg>>ujCZqQi)BcU&3Lcpb$e=G9|f+=`0%$dbkwNl*~= zDzoa55X~kDnk7Dp)N3U7d8GBW#d3p?iNLakdBwr#`1Wu53+7ZQ$LVLWI)`(*D zB=(6t+Zu0}C#$g%@g0u%INGRF-_<5qYzbQ;I%3IahC*sL0Sd0Nn&YOd!S}}Ys?AKi z6nm5%620-Io+PS8(S_>lZ|oE&K|54M{JNO_(>fWRby+W^Y$;Y<;Q(7ZZHWY}NM6iR?fx)4?%Kqi`LG11sIFhJ-b z0uK;)5}@oV<|dI(_n{;o9x;=WG@2_~Wo;}oVdiM;#=SF~!No94BRywj$BL=AOtZ3n zC}SNSHtzuk+4i5tPTwg)pC<5>NHr~ILhNm_ZvQ6;MH>M1S#>cyDjqDylI%sFh@A2rp6P!+&$xJR3tkDGfarpTvMfC1?IxG!9DZpL08FL@|3Dd^ugdPEI%PC zI8s+tO1_dGGiq(O?Qc|2Lw$f%go|A*9aD8!tJj83X^XMQ7FwwmSBf#D(pE64w<56D z$Nk3@SR9zqh+YU{cC^wnIvb2ZD>bkVK zS*}L5sA?%-q5J3IXq?P%T6AS3q?ilmbS9wtEPwQj2I$UIi+MYduDV`|MB7AQj6fEktQC=A za@7}zc1S#vNDd4V#~3b7jOHx*mwEDK;?EKI9s!xC+X*uXa01^U@DKsY>Qr?lUvdSW zJ?7j_f|#d7Jh`4dD0U}X*KdLC#jT(!nim02*HQygz+f?q8gN0iunoXsEC_m#dC;hI zs!h6hKDp@t%8f!zYmsh093(Xc34FhyMVV&u-$h9z0A;t+(H3R-G>he`9ryQT#wJFyrmU*wX_7oE9_w!ONgv39dQKcMYQ;dlqRqLx7%35hzuD>DP zUU@7>0Ge00gErB`$&jFkk&tKoW zOm6ZAM7x;)iC|w+Ty9Q^e+cUH!ngLgpTx_qk?j3N^PG5gZIeMPoB;fwmw}!U@2}lv zTp;3mBDrp2|mhk5Ef9*@hTskq2Ic;tT-d_1$xV&{drzAyPY&G;pO z-vgvv&Uyct2o7t(|Lhg<#QFkzRs47TPi~|Vm@>+K@>crr_|XD+Z2(00QC62U-QQBB zqx@Z(P>F|A;$Iuu*jZ7%ae#d+9rrtM-w`t#Uv7~LKZfDuvvQ8}o1ncf1~zSBuZy`& zkFVyC~k;)S=TBE{M-I-D5axQf|Bk;-8opyWy~-{UXLMs-`my_(F_*NINy? zgfBAT?j7?qqhiaA8}uoqHF(Sa0gzkp&j0`b delta 5875 zcmbVQ3v650dA{df^78U2ilq3EC{xrkvSnLw>{|9z$8{cF`cRxGjuY1z&G^s?DT$H~ zuhT@k1l=h?*M~dB2W^2S?c{AJU}R{vWp%ft4O*b7n>2~jtch1P0~$1K(smn)4IKt| zDE9s5QZKjIu!-Q$|NQ@XpY#9Uck~yv|8gRS!{LC2KlaZ9#pe${7Rl-wdu-&5ZKcej zakTTG#v{D#5skMk>PNd!it=`p+DjdkY^57DHgV6SHXM7phed+=PjxvOda%L$j#ddz zo8r&tm-{oCqZJ}{8%mKy20dz9Jb+SkQD++Tpu;Nd_{Q*!`q>8%s>= zvpXuumL*J6I%}Yx6f7wo&1jW$U0>08L*v?`x*9a&=)7HNofsyDDxEZ>sv6Ir-fGP* zWlL7$2U~YtGx*G z&Wxt^qZgB-3G^mWX4;hc@az*bT%X;)zV;%!S-bD0Iy~4}w0De*R`>93$209)(nSyK zLw$TxMq3_mjPvLWhCZIVrrbkixj)ho+^_tqzC4cd@wNK$Or)c6MwL~4x4&u@2QjjjwYAo+cZr>P4U2IXsOGWA+(p6Ks}`bWv(DPrvSN%CE7x$R`X`#^X*m=9uqkfB z{k$q)kM2ufU`>;Y@`EMOG-rzUmu6}|W6HUueKL80i4iJzwpHgBnr3;nI8~}$m7i+A zA#x0MaUY34fRMMxrr41DYV4Ry$M>?PoQ#hL?;`JE5_ikb#}D**MX@?nYMRCQ`BIfP z1GXrZtA)Ah%)=MFVjI;vF0UrGr>H}XnanaGto1QMEFlkc4EUog0&9{*WVItBpY9lK zd<&JaX_~`u)mU0nC&^Yz9>?PtX?Rjv-RpR{KJBTE**Z5C*&i{->*&*(Sh9_Z$D#Y~ zozl67d(Y~1(>5Jl-Hp+@&-N`_j^TLD>vapY{Eo$GOz)%mN%{R`Z2VYtbft=;yMwJ& zbu33uA%0eyAd>2)W8y^232~>qJ9WPm5bkElAEf$+#XOeOG_hNJZno*8-*VL!6c~fSjQ+&tqFC(E;OniM#1Y7D$(^4&YIWu&lU!@+h1`sGQ z9vCOzs&zTQgFJLzhbIV+g`Q#*F{gU%pdGS!xU>MuToH&n;Yj%XU^VlPrxTqC+(E{ zldc?#%BQ+gEG;i}7J0rJj>eGLv$wQmI$v!S)xqhb0-MKwhRurk{Q}Ur)HcFSQbr(r1 z$(M4y(UYW>Nzly{pO(ML<&#Z+sd{?0B#L$kceGS{pg3O=b8>r6#w<{+qMYa%3`$Zz zL*g;n=-I-~$tQd4&WkYAXekJD-NW$F@qx7Do_tp@pbzQ+!^13=lvDZH#;eFk4m)Pl z^}12_;Kt#!#eV#Gig?ZcW=-4m&2d&g;iKbzj}8`w>&?C!nx--GmS* zx<(O)mJ#`l-rbEh43l=ylbVLPVH(tb1X;6=4OEVDj);Kv3OVji7am zP@6}Qt7!6Z^m=Zg7h2+)>95T`d_=1z$C*YS>`w=8M`z)CwW@xP*6Omo)n(WQIpDEH z!lbV`+Hy*z$FzKh^A+@RIG|8sN5hy*ZV@7N}Nq^xynlqxnXpUoKjvM*v;zFH<;UHVs^Ft zuxdsCmv56E0}7>;mSBTy3|g8+^iCvNi|MQvmp7)%`v&)p7-&P7T8lu~loT0V$wh`F zkBy+c8rFr1qtdypsYKIPD9n_qg~C=EQioe(GCwpHRTKU)isE?^-O1vG7@bZ9uUW=9}0PKKM9oRr zBP(VBMfE~PLVmC#oikPmGLlq6Pl75h#MknJ?6H#aQa;>x8(sbuq`n__2#^YCtOwM` zL8TB^o}~7;_63DM01tpCz}3sWtKe#i3c%G@0j>eD0IVFjVHI2h0Fryqa}ekhawgXg z0fFxgN_tr< zgFiyUyR+cu^uG!R|C!P8j@5;*iF^7gK}CHHZg9_8qi$4_t9F7#uv~Cma^RU8q6YZV zo5bLz9abISf`JF?sArueFJ7#8=Mbboqv(X@C)=qB1r`1YH(ejY>h2oWo4(our%#@o zDK$MG5ykmi)n&u~4hovwGCa92>gMEEQ2A&(2UjT+PfyL3s&>&X&sD{fus6du%~ekn z?bhat8(~v>DqfaL!#Pu>#uwx>!(GAS1<^G~iS?e&M#E&OqX} zQI `mC8D4*MM#C(|y2hMFxdk;W+MlNmbl9MBc+1Dl8mXhBX*~?y*zaGi6Z_4hi zhoD!t=GnJUYo1*~jc4)xMZcz+lK39=^@e<9Tc=6eC0>&Mx~-qRB;(sZHm0T(sA1Pi zz@8H}=Kpc3qxOOpKg`6B$Pg!yAn`cF_vCBa$Jy($ZL}ltBQmN5DEC_^epil-J`$pJ zH$Bs(hil@p{LX0BBo|hR`11Z3>i4C7{jKH?$@oul;`$-i|6^jO*NGQk{tTS}{TBjv zKW$l-$1-#lFf4r3uV1`?c<7c_u0O)A$n85O^Dk0o-zD)T#IVPm*H6gcx>in?y(ypF zQDtw-%{Tn@K1yq#ydgT4!il-lRXQHx)i^Lk?F%zyJQ=79y#iqxy- z?^4TmdK<=kx{W%F*io0wYM^JO|7M;(4xB~r$bUsliI3EkA z62aGR839vyhja4bu`J{AvtzfJe@IP#A%8M= self.num_images: + ix1 = self.label_start_ix_da[ix - self.num_images] - 1 #label_start_ix starts from 1 + ix2 = self.label_end_ix_da[ix - self.num_images] - 1 + else: + ix1 = self.label_start_ix[ix] - 1 #label_start_ix starts from 1 + ix2 = self.label_end_ix[ix] - 1 ncap = ix2 - ix1 + 1 # number of captions available for this image assert ncap > 0, 'an image does not have any label. this can be handled but right now isn\'t' @@ -172,10 +216,16 @@ def get_captions(self, ix, seq_per_img): seq = np.zeros([seq_per_img, self.seq_length], dtype = 'int') for q in range(seq_per_img): ixl = random.randint(ix1,ix2) - seq[q, :] = self.label[ixl, :self.seq_length] + if self.opt.data_augmentation and ix >= self.num_images: + seq[q, :] = self.label_da[ixl, :self.seq_length] + else: + seq[q, :] = self.label[ixl, :self.seq_length] else: ixl = random.randint(ix1, ix2 - seq_per_img + 1) - seq = self.label[ixl: ixl + seq_per_img, :self.seq_length] + if self.opt.data_augmentation and ix >= self.num_images: + seq = self.label_da[ixl: ixl + seq_per_img, :self.seq_length] + else: + seq = self.label[ixl: ixl + seq_per_img, :self.seq_length] return seq @@ -210,15 +260,22 @@ def collate_func(self, batch, split): # Used for reward evaluation if hasattr(self, 'h5_label_file'): # if there is ground truth - gts.append(self.label[self.label_start_ix[ix] - 1: self.label_end_ix[ix]]) + if self.opt.data_augmentation and ix >= self.num_images: + gts.append(self.label_da[self.label_start_ix_da[ix - self.num_images] - 1: self.label_end_ix_da[ix - self.num_images]]) + else: + gts.append(self.label[self.label_start_ix[ix] - 1: self.label_end_ix[ix]]) else: gts.append([]) # record associated info as well info_dict = {} info_dict['ix'] = ix - info_dict['id'] = self.info['images'][ix]['id'] - info_dict['file_path'] = self.info['images'][ix].get('file_path', '') + if self.opt.data_augmentation and ix >= self.num_images: + info_dict['id'] = self.images_da[ix - self.num_images]['id'] + # info_dict['file_path'] = self.images_da[ix - self.num_images].get('file_path', '') + else: + info_dict['id'] = self.images[ix]['id'] + info_dict['file_path'] = self.info['images'][ix].get('file_path', '') infos.append(info_dict) # #sort by att_feat length @@ -264,16 +321,19 @@ def __getitem__(self, index): """ ix, it_pos_now, wrapped = index #self.split_ix[index] if self.use_att: - att_feat = self.att_loader.get(str(self.info['images'][ix]['id'])) + if self.opt.data_augmentation and ix >= self.num_images: + att_feat = self.att_loader_da.get(self.info_da['images'][ix - self.num_images]['id']) + else: + att_feat = self.att_loader.get(str(self.images[ix]['id'])) # Reshape to K x C att_feat = att_feat.reshape(-1, att_feat.shape[-1]) if self.norm_att_feat: att_feat = att_feat / np.linalg.norm(att_feat, 2, 1, keepdims=True) if self.use_box: - box_feat = self.box_loader.get(str(self.info['images'][ix]['id'])) + box_feat = self.box_loader.get(str(self.images[ix]['id'])) # devided by image width and height x1,y1,x2,y2 = np.hsplit(box_feat, 4) - h,w = self.info['images'][ix]['height'], self.info['images'][ix]['width'] + h,w = self.images[ix]['height'], self.images[ix]['width'] box_feat = np.hstack((x1/w, y1/h, x2/w, y2/h, (x2-x1)*(y2-y1)/(w*h))) # question? x2-x1+1?? if self.norm_box_feat: box_feat = box_feat / np.linalg.norm(box_feat, 2, 1, keepdims=True) @@ -283,11 +343,14 @@ def __getitem__(self, index): else: att_feat = np.zeros((0,0), dtype='float32') if self.use_fc: - try: - fc_feat = self.fc_loader.get(str(self.info['images'][ix]['id'])) - except: + # try: + if self.opt.data_augmentation and ix >= self.num_images: + fc_feat = self.fc_loader_da.get(str(self.info_da['images'][ix - self.num_images]['id'])) + else: + fc_feat = self.fc_loader.get(str(self.images[ix]['id'])) + # except: # Use average of attention when there is no fc provided (For bottomup feature) - fc_feat = att_feat.mean(0) + # fc_feat = att_feat.mean(0) else: fc_feat = np.zeros((0), dtype='float32') if hasattr(self, 'h5_label_file'): @@ -299,117 +362,16 @@ def __getitem__(self, index): ix, it_pos_now, wrapped) def __len__(self): - return len(self.info['images']) - -class Dataset_DA(Dataset): - def __init__(self, opt): - self.opt = opt - self.seq_per_img = 1 - - # feature related options - self.use_fc = getattr(opt, 'use_fc', True) - self.use_att = getattr(opt, 'use_att', True) - self.use_box = getattr(opt, 'use_box', 0) - self.norm_att_feat = getattr(opt, 'norm_att_feat', 0) - self.norm_box_feat = getattr(opt, 'norm_box_feat', 0) - - # load the json file which contains additional information about the dataset - print('DataLoader da loading json file: ', opt.input_json_da) - self.info = json.load(open(self.opt.input_json_da)) - if 'ix_to_word' in self.info: - self.ix_to_word = self.info['ix_to_word'] - self.vocab_size = len(self.ix_to_word) - print('vocab size is ', self.vocab_size) - - # open the hdf5 file - print('DataLoader da loading h5 file: ', opt.input_fc_dir_da, opt.input_att_dir_da, opt.input_label_h5_da) - """ - Setting input_label_h5 to none is used when only doing generation. - For example, when you need to test on coco test set. - """ - if self.opt.input_label_h5 != 'none': - self.h5_label_file = h5py.File(self.opt.input_label_h5_da, 'r', driver='core') - # load in the sequence data - seq_size = self.h5_label_file['labels'].shape - self.label = self.h5_label_file['labels'][:] - self.seq_length = seq_size[1] - print('max sequence length in data is', self.seq_length) - # load the pointers in full to RAM (should be small enough) - self.label_start_ix = self.h5_label_file['label_start_ix'][:] - self.label_end_ix = self.h5_label_file['label_end_ix'][:] - else: - self.seq_length = 1 - - self.data_in_memory = getattr(opt, 'data_in_memory', False) - self.fc_loader = HybridLoader(self.opt.input_fc_dir_da, '.npy', in_memory=self.data_in_memory) - self.att_loader = HybridLoader(self.opt.input_att_dir_da, '.npz', in_memory=self.data_in_memory) - # self.box_loader = HybridLoader(self.opt.input_box_dir, '.npy', in_memory=self.data_in_memory) - - self.num_images = len(self.info['images']) # self.label_start_ix.shape[0] - print('read %d image features' %(self.num_images)) - - # separate out indexes for each of the provided splits - self.split_ix = {'train': [], 'val': [], 'test': []} - for ix in range(len(self.info['images'])): - img = self.info['images'][ix] - # data augmentation only for training - if img['split'] == 'train': - self.split_ix['train'].append(ix) - - print('assigned %d images to split train in da' %len(self.split_ix['train'])) - print('assigned %d images to split val in da' %len(self.split_ix['val'])) - print('assigned %d images to split test in da' %len(self.split_ix['test'])) - - def __getitem__(self, index): - """This function returns a tuple that is further passed to collate_fn - """ - ix, it_pos_now, wrapped = index #self.split_ix[index] - if self.use_att: - # att_feat = self.att_loader.get(str(self.info['images'][ix]['id'])) - att_feat = self.att_loader.get(f"{self.info['images'][ix]['imgid']}_{ix%5}") - # Reshape to K x C - att_feat = att_feat.reshape(-1, att_feat.shape[-1]) - if self.norm_att_feat: - att_feat = att_feat / np.linalg.norm(att_feat, 2, 1, keepdims=True) - if self.use_box: - box_feat = self.box_loader.get(str(self.info['images'][ix]['id'])) - # devided by image width and height - x1,y1,x2,y2 = np.hsplit(box_feat, 4) - h,w = self.info['images'][ix]['height'], self.info['images'][ix]['width'] - box_feat = np.hstack((x1/w, y1/h, x2/w, y2/h, (x2-x1)*(y2-y1)/(w*h))) # question? x2-x1+1?? - if self.norm_box_feat: - box_feat = box_feat / np.linalg.norm(box_feat, 2, 1, keepdims=True) - att_feat = np.hstack([att_feat, box_feat]) - # sort the features by the size of boxes - att_feat = np.stack(sorted(att_feat, key=lambda x:x[-1], reverse=True)) - else: - att_feat = np.zeros((0,0), dtype='float32') - if self.use_fc: - try: - # fc_feat = self.fc_loader.get(str(self.info['images'][ix]['id'])) - fc_feat = self.fc_loader.get(f"{self.info['images'][ix]['imgid']}_{ix%5}") - except: - # Use average of attention when there is no fc provided (For bottomup feature) - fc_feat = att_feat.mean(0) - else: - fc_feat = np.zeros((0), dtype='float32') - if hasattr(self, 'h5_label_file'): - seq = self.get_captions(ix, self.seq_per_img) + if self.opt.data_augmentation: + return len(self.images) + len(self.info_da['images']) else: - seq = None - return (fc_feat, - att_feat, seq, - ix, it_pos_now, wrapped) - + return len(self.images) class DataLoader: def __init__(self, opt): self.opt = opt self.batch_size = self.opt.batch_size - if self.opt.data_augmentation: - self.dataset = data.ConcatDataset([Dataset(opt), Dataset_DA(opt)]) - else: - self.dataset = Dataset(opt) + self.dataset = Dataset(opt) # Initialize loaders and iters self.loaders, self.iters = {}, {} diff --git a/configs/fc.yml b/configs/fc.yml index 744459f4..e381ff74 100644 --- a/configs/fc.yml +++ b/configs/fc.yml @@ -18,6 +18,6 @@ max_epochs: 30 # configs for da data_augmentation: True input_json_da: /data/share/image-caption/f30ktalk_da.json -input_fc_dir_da: /data/share/image-caption/f30ktalk_fc_da -input_att_dir_da: /data/share/image-caption/f30ktalk_att_da +input_fc_dir_da: /data/share/image-caption/f30ktalk_da_fc +input_att_dir_da: /data/share/image-caption/f30ktalk_da_att input_label_h5_da: /data/share/image-caption/f30ktalk_da_label.h5 \ No newline at end of file diff --git a/prepro.sh b/prepro.sh index 1fbaabad..b81a4ef5 100644 --- a/prepro.sh +++ b/prepro.sh @@ -16,10 +16,10 @@ LBL_JSON_DA=$DATA_ROOT/f30ktalk_da.json LBL_H5_DA=$DATA_ROOT/f30ktalk_da # pre labels for da -# python scripts/prepro_labels.py \ -# --input_json $JSON_DA \ -# --output_json $LBL_JSON_DA \ -# --output_h5 $LBL_H5_DA +python scripts/prepro_labels_da.py \ + --input_json $JSON_DA \ + --output_json $LBL_JSON_DA \ + --output_h5 $LBL_H5_DA # pre ngram for da # python scripts/prepro_ngrams.py \ @@ -37,8 +37,8 @@ LBL_H5_DA=$DATA_ROOT/f30ktalk_da # --model_root $MODEL # pre feats -python scripts/prepro_feats.py \ - --input_json $JSON \ - --output_dir $OUT_DIR \ - --images_root $IMG_ROOT \ - --model_root $MODEL \ No newline at end of file +# python scripts/prepro_feats.py \ +# --input_json $JSON \ +# --output_dir $OUT_DIR \ +# --images_root $IMG_ROOT \ +# --model_root $MODEL \ No newline at end of file diff --git a/scripts/prepro_labels_da.py b/scripts/prepro_labels_da.py new file mode 100644 index 00000000..4d042d60 --- /dev/null +++ b/scripts/prepro_labels_da.py @@ -0,0 +1,161 @@ +""" +Preprocess a raw json dataset into hdf5/json files for use in data_loader.py + +Input: json file that has the form +[{ file_path: 'path/img.jpg', captions: ['a caption', ...] }, ...] +example element in this list would look like +{'captions': [u'A man with a red helmet on a small moped on a dirt road. ', u'Man riding a motor bike on a dirt road on the countryside.', u'A man riding on the back of a motorcycle.', u'A dirt path with a young person on a motor bike rests to the foreground of a verdant area with a bridge and a background of cloud-wreathed mountains. ', u'A man in a red shirt and a red hat is on a motorcycle on a hill side.'], 'file_path': u'val2014/COCO_val2014_000000391895.jpg', 'id': 391895} + +This script reads this json, does some basic preprocessing on the captions +(e.g. lowercase, etc.), creates a special UNK token, and encodes everything to arrays + +Output: a json file and an hdf5 file +The hdf5 file contains several fields: +/labels is (M,max_length) uint32 array of encoded labels, zero padded +/label_start_ix and /label_end_ix are (N,) uint32 arrays of pointers to the + first and last indices (in range 1..M) of labels for each image +/label_length stores the length of the sequence for each of the M sequences + +The json file has a dict that contains: +- an 'ix_to_word' field storing the vocab in form {ix:'word'}, where ix is 1-indexed +- an 'images' field that is a list holding auxiliary information for each image, + such as in particular the 'split' it was assigned to. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os +import json +import argparse +from random import shuffle, seed +import string +# non-standard dependencies: +import h5py +import numpy as np +import torch +import torchvision.models as models +import skimage.io +from PIL import Image + + +def encode_captions(imgs, params, wtoi): + """ + encode all captions into one large array, which will be 1-indexed. + also produces label_start_ix and label_end_ix which store 1-indexed + and inclusive (Lua-style) pointers to the first and last caption for + each image in the dataset. + """ + + max_length = params['max_length'] + N = len(imgs) + M = sum(len(img['final_captions']) for img in imgs) # total number of captions + + label_arrays = [] + label_start_ix = np.zeros(N, dtype='uint32') # note: these will be one-indexed + label_end_ix = np.zeros(N, dtype='uint32') + label_length = np.zeros(M, dtype='uint32') + caption_counter = 0 + counter = 1 + for i,img in enumerate(imgs): + n = len(img['final_captions']) + assert n > 0, 'error: some image has no captions' + + Li = np.zeros((n, max_length), dtype='uint32') + for j,s in enumerate(img['final_captions']): + label_length[caption_counter] = min(max_length, len(s)) # record the length of this sequence + caption_counter += 1 + for k,w in enumerate(s): + if k < max_length: + Li[j,k] = wtoi[w] + + # note: word indices are 1-indexed, and captions are padded with zeros + label_arrays.append(Li) + label_start_ix[i] = counter + label_end_ix[i] = counter + n - 1 + + counter += n + + L = np.concatenate(label_arrays, axis=0) # put all the labels together + assert L.shape[0] == M, 'lengths don\'t match? that\'s weird' + assert np.all(label_length > 0), 'error: some caption had no words?' + + print('encoded captions to array of size ', L.shape) + return L, label_start_ix, label_end_ix, label_length + + +def main(params): + + imgs = json.load(open(params['input_json'], 'r')) + imgs = imgs['images'] + + seed(123) # make reproducible + + # create the vocab + # vocab = build_vocab(imgs, params) + with open('/data/share/image-caption/cocotalk.json') as f: + talk = json.load(f) + itow = talk['ix_to_word'] + vocab = list(itow.values()) + itow = {i+1:w for i,w in enumerate(vocab)} # a 1-indexed vocab translation table + wtoi = {w:i+1 for i,w in enumerate(vocab)} # inverse table + + for img in imgs: + img['final_captions'] = [] + for sent in img['sentences']: + txt = sent['tokens'] + caption = [w if w in vocab else 'UNK' for w in txt] + img['final_captions'].append(caption) + + # encode captions in large arrays, ready to ship to hdf5 file + L, label_start_ix, label_end_ix, label_length = encode_captions(imgs, params, wtoi) + + # create output h5 file + N = len(imgs) + f_lb = h5py.File(params['output_h5']+'_label.h5', "w") + f_lb.create_dataset("labels", dtype='uint32', data=L) + f_lb.create_dataset("label_start_ix", dtype='uint32', data=label_start_ix) + f_lb.create_dataset("label_end_ix", dtype='uint32', data=label_end_ix) + f_lb.create_dataset("label_length", dtype='uint32', data=label_length) + f_lb.close() + + # create output json file + out = {} + out['ix_to_word'] = itow # encode the (1-indexed) vocab + out['images'] = [] + for i,img in enumerate(imgs): + + jimg = {} + jimg['split'] = img['split'] + if 'filename' in img: jimg['file_path'] = os.path.join(img.get('filepath', ''), img['filename']) # copy it over, might need + jimg['id'] = img['id'] + + if params['images_root'] != '': + with Image.open(os.path.join(params['images_root'], img['filepath'], img['filename'])) as _img: + jimg['width'], jimg['height'] = _img.size + + out['images'].append(jimg) + + json.dump(out, open(params['output_json'], 'w')) + print('wrote ', params['output_json']) + +if __name__ == "__main__": + + parser = argparse.ArgumentParser() + + # input json + parser.add_argument('--input_json', required=True, help='input json file to process into hdf5') + parser.add_argument('--output_json', default='data.json', help='output json file') + parser.add_argument('--output_h5', default='data', help='output h5 file') + parser.add_argument('--images_root', default='', help='root location in which images are stored, to be prepended to file_path in input json') + + # options + parser.add_argument('--max_length', default=16, type=int, help='max length of a caption, in number of words. captions longer than this get clipped.') + parser.add_argument('--word_count_threshold', default=5, type=int, help='only words that occur more than this number of times will be put in vocab') + + args = parser.parse_args() + params = vars(args) # convert to ordinary dict + print('parsed input parameters:') + print(json.dumps(params, indent = 2)) + main(params) diff --git a/scripts/write_f30k_da.py b/scripts/write_f30k_da.py index acd0b9ea..59af5f1e 100644 --- a/scripts/write_f30k_da.py +++ b/scripts/write_f30k_da.py @@ -8,12 +8,23 @@ def write_json(root, img_num): ''' with open(f'{root}/dataset_flickr30k.json', 'r') as fin: dataset = json.load(fin) - images = dataset['images'] + images = dataset['images'][:img_num] dataset_da = { 'dataset': 'flickr30k_da', - 'images': images[:img_num] + 'images': [] } + for img in images: + for i,s in enumerate(img['sentences']): + new_img = { + 'sentids': [img['sentids'][i]], + 'imgid': img['imgid'], + 'sentences': [s], + 'split': img['split'], + 'id': f"{img['imgid']}_{i}" + } + dataset_da['images'].append(new_img) + with open(f'{root}/dataset_flickr30k_da.json', 'w') as fout: json.dump(dataset_da, fout) @@ -22,5 +33,5 @@ def write_json(root, img_num): # root = '/data/private/mxy/code/T2I_CL/DM-GAN+CL/output/coco_DMGAN_2021_08_15_14_19_42/Model/netG_epoch_120/f30k' # out_path = '/data/private/mxy/data' root = '/data/share/image-caption' - img_num = 22685 + img_num = 22684 write_json(root, img_num) \ No newline at end of file From a8b93466d3a575ddd290acd444e16f8ee00a0459 Mon Sep 17 00:00:00 2001 From: Maxy Date: Tue, 28 Sep 2021 10:34:20 +0000 Subject: [PATCH 3/3] scripts --- .../__pycache__/dataloader.cpython-38.pyc | Bin 14756 -> 14761 bytes captioning/data/dataloader.py | 10 +-- .../utils/__pycache__/opts.cpython-38.pyc | Bin 13288 -> 13330 bytes captioning/utils/opts.py | 2 + configs/fc.yml | 11 +-- prepro.sh | 15 ++-- scripts/prepro_feats_da.py | 70 ++++++++++----- scripts/prepro_labels_da.py | 5 +- scripts/write_coco_da.py | 85 ++++++++++++++++++ scripts/write_f30k_da.py | 1 - 10 files changed, 159 insertions(+), 40 deletions(-) create mode 100644 scripts/write_coco_da.py diff --git a/captioning/data/__pycache__/dataloader.cpython-38.pyc b/captioning/data/__pycache__/dataloader.cpython-38.pyc index 5a0b1098cf39366b79a84f5ab6721d2e383e2622..dc50ffe8d6c3e230181a16ee205a78f182efd1a5 100644 GIT binary patch delta 1095 zcmZuw-Afcv6u)Oa*4@>aUDuCwbzMJKn`jk^LP8-bsE4u-D@ZqT^=dlirn5r?-N9n3 zhe#4{^cJiTK`&uVe}FF$ggw-wR4)lZZ-o%`5S=@_N{9pVo8SGNd+#~-ch5Y{J*qR++D5-O65a%3Ts-x1b-8~5TQto zfYl`wTR_@P)tyvegx`#Bz^FUZC`rDw2u$%CP4S-p)(i2PfEfX^0>p0CSpfqAvI0&B z80M=@t-g$)qkN-jPsl{-)jW}z8M7vNcXI;LZh!MVnDa}7Fi6Wz(Mn>H1+a$E7BD62 zFq4qUQz|lIDg_nMSJdhVS`IO^(^0easFxK~+mGJ>lZuf%h!|4~K`JrTET^b1NvIIB z!g|0|TQha@Qi!QkrV3T5Z(eC2UO0>;WN}*Mj$8U{tq@^8TDBym6{pPcI{_!SmDnF6 zTRgyDw;rqNVYmWQg>X$N6W7}s=Yq8S4#B(?^Rjmc9cM+UMs*sRFP2${b-csf&t*Vt;ZiEiZeWB~=0%bXMhbB+Q6qY#XB8T8bfz(t z>8Q#~8TSnkql$lT>*sy#9}bP+&Gs7!>$te=ZS9g1+Cc$cYm6gXlXJFJY?5apFv|Or zEpUa;B=?5Lgz!YbEdgo%DtQ+Yyr-iDTKQl{-#HoOBlzZaBcLs4j1{-N4#|V}Rq_xe z17uShAQVSdIZij!|Elvk)9cjiVk*i5owZQyHg^t7_z&GSx?qwo8_D9&a}n2CZ+XU7 p#hYss_m}ZI02%kkzMlXYzP^94*yW!VQ+4j610ca=_i>L7{{RZT3se9A delta 1071 zcmZuwO-NKx6u#%aA9eIS$MI)oOlL-O#sho!-1FVD^3lhe}W;6uD;!Ct$PfKD{7_si!X$-e3 z?D6c>$o#lHCHi6!NO_a7WjO1ngm4&A+>%YiBCB9CGyp7W_i~DnGNWNcEH$qo>I}l9 z#D@r4N<~+Dc!^3|3gBv7%lnwZHLIE_VN#&T5Ua*s<(gwyrd7)|rZSD`tYSsoM~bU{ ztZ3t|y1eF8;#kG;G`4f1?il$1*I8wO4ykU1Tj^H0zTI6_s0B0hVv0{}v|m^{$uXr^ zO8={rJgA;)T%V*;D9#$4CX`dnYVHygWUMHX&zP7UWG1U+{*{va5DMJFGH3}#Z+A>7 z#>5JjWtWXv!i`GFnX)HFrWv{B|K83hm#Zt6Gn{>#us}{0`prqcrC6*LeKa#Ft9MOq z;1x_{6|PR`10>n7c=77gbT&3|D>LRy&t-(KKG6mVF_AcxQjtD~ zZ&o%l-WUzDV@|Pya!?#q**iH1k^!>A4-kg4B<(L&O=^2Yrtz4PmRYgZ9Ta9~6NJ2Y tXPV-VwVdq1Km1KH4ujsuWU~*_-mk+y0Mg>y(MLGp$08JNcu$U-@CS~M22}t6 diff --git a/captioning/data/dataloader.py b/captioning/data/dataloader.py index beeeed59..e1558b82 100644 --- a/captioning/data/dataloader.py +++ b/captioning/data/dataloader.py @@ -157,7 +157,7 @@ def __init__(self, opt): self.split_ix['train'].append(ix) if opt.data_augmentation: - self.seq_per_img_da = 1 + self.seq_per_img_da = opt.seq_per_img_da # load the json file which contains additional information about the dataset print('DataLoader da loading json file: ', opt.input_json_da) @@ -192,7 +192,7 @@ def __init__(self, opt): for ix, img in enumerate(self.images_da): # data augmentation only for training - if img['split'] == 'train': + if img['split'] == 'train' or img['split'] == 'restval': self.split_ix['train'].append(ix + self.num_images) print('assigned %d images to split train' %len(self.split_ix['train'])) @@ -322,7 +322,7 @@ def __getitem__(self, index): ix, it_pos_now, wrapped = index #self.split_ix[index] if self.use_att: if self.opt.data_augmentation and ix >= self.num_images: - att_feat = self.att_loader_da.get(self.info_da['images'][ix - self.num_images]['id']) + att_feat = self.att_loader_da.get(self.images_da[ix - self.num_images]['id']) else: att_feat = self.att_loader.get(str(self.images[ix]['id'])) # Reshape to K x C @@ -345,7 +345,7 @@ def __getitem__(self, index): if self.use_fc: # try: if self.opt.data_augmentation and ix >= self.num_images: - fc_feat = self.fc_loader_da.get(str(self.info_da['images'][ix - self.num_images]['id'])) + fc_feat = self.fc_loader_da.get(str(self.images_da[ix - self.num_images]['id'])) else: fc_feat = self.fc_loader.get(str(self.images[ix]['id'])) # except: @@ -363,7 +363,7 @@ def __getitem__(self, index): def __len__(self): if self.opt.data_augmentation: - return len(self.images) + len(self.info_da['images']) + return len(self.images) + len(self.images_da) else: return len(self.images) diff --git a/captioning/utils/__pycache__/opts.cpython-38.pyc b/captioning/utils/__pycache__/opts.cpython-38.pyc index 145bcc93b0a47024573a409cdff30377f7dcd9e2..ddb2ab628543ec55401ca2de73fecdaf198f8e26 100644 GIT binary patch delta 1861 zcmZYAS#VQT7zgls(wn4>X$F(p&C(?a#L|?;)}=PCGo1mMQIUcx8mnI|G&Hu&Rk7ZB zl_-jm5_??01=&;-s6rH6Dk^RR59%9lj{4+`?@q^Y9RE2U#xJ!mz4v$i-}l{f&m{S6 zcxFRjU3In3z~9;G<%8V=M*~M(;x9Sp`ZMIFFh$7Bo5HY-5f??R3VFIB;<%}U;>bSvp$atY*8x{S%?N_v@G0V$u=XSsVRNmsJaRR|^NY9`kxxt7Uw zC6W8hHf(AItz;|LV^e3%NV|tt)6LA+prW4!n5>21g~;>WA!i-7d<)&mR&TRBbUWR_ zs1Cp>|gjxxzEsIib(O%5-5Q0gG|gm($`qJ951ApU z#VLIpUI2B_i!AgK8fEBZCOcK5T})n4ulQ9auR)UZI+NWh!{6u|=+jSo&Se~o--}`U z%Fb&4g4cdPHGcD45o+F2@-~x$XuN{>7 z>3U)sNAb}n|7i7;3t<&f9+%%$M#L)ldu7a-u!8ilZPu!>R>&5%X}fdyG!#7JYw!(M zWefTwFVv74N~TTB*k@fppXS7t(%@Qc$6<5Y>a?AKeWtBm+fT5Q+8QY7q}_PXKhUUw zI?@=L{TqSLb)Zodd9y*-7uqh=wiNcIwkBtKiLMFs~W&Tm=U= z2z;#rEjlobK#K}^v(s$7Rjb#U#ERl- zNlvWCIi;4`LhkTYh2~-H6s{{(wA*69X#ci&$ajMooLtn|nH?HWjSWqtHjWObXi&D) z>@3a)V(#LJn!^Eap9{ZHqEZgV>*bFTuiO^tl0Qd-Gf@w2n{nxvo@h`emiYgr`eGvb zt4l1Fq1b?@3AK|eXBv9tf!H$9So|T@Z~B^qVPrF0*~EbjSPD zQR)TM)uG1=Xw@QkZ&g@S)Z*O-65oyS$xDbaJm{k)M*lrM#c%5-vz_^!|M|X|Gkemp zt-t1LHda^141E6iY1x*GbBAiCO!1qXHvg)NP#Y!5Dp|ttjX{%Ao-L0@l0k&dqz>kt zo=wZBi%GYV9wy5nXHhSc6-v%#atGEb#hN<`%-F zCspVv778JIXpD)3Waw!o&!Er{J<3iggKW;LN?v2~I_j>a+1m+bxA+?<{ANYrx73sJ+j$$A$%@R> z|1$5OSzkrwUDaOK@m#i_-a|cO107;_c9_a^ByZ6B^g+p<`Nd9D?Keh*F9u^|6skhI z_&d%|Q|HdGk5}>Gwgp4gBPOzQ8E3dje(xqlR{rg#gUwzo9rdjSwT-po3dJc?K-1z$1U}V)CN(KqG+>`;TcWKV_PMrZZ5h}X+D_ND685FG1k4Ph#ZBm! zKtD6g;{t)Nbf85Cd<0rlAX@a<_)e4LH9hSBYl^fgA?8nEI6tvE}9m5@`(ZrPJM zLrx?Y;IC6oq!JO)ELWxb#S*zEofq-)pXs%hXp*Zteim`r)_Gj4E1&Ef5E!|vd!G<3 z