From 9bdbb46ca3123ee119d8890a0a6c02c8f66475fc Mon Sep 17 00:00:00 2001 From: ChiYeung Law <31592607+ChiYeungLaw@users.noreply.github.com> Date: Wed, 14 Jun 2023 21:58:28 +0800 Subject: [PATCH] update wizardcoder --- WizardCoder/README.md | 198 ++++++++++++++++++ WizardCoder/imgs/pass1.png | Bin 0 -> 65829 bytes WizardCoder/src/humaneval_gen.py | 177 ++++++++++++++++ WizardCoder/src/inference_wizardcoder.py | 121 +++++++++++ WizardCoder/src/process_humaneval.py | 69 +++++++ WizardCoder/src/train_wizardcoder.py | 248 +++++++++++++++++++++++ WizardCoder/test | 1 - 7 files changed, 813 insertions(+), 1 deletion(-) create mode 100644 WizardCoder/README.md create mode 100644 WizardCoder/imgs/pass1.png create mode 100644 WizardCoder/src/humaneval_gen.py create mode 100644 WizardCoder/src/inference_wizardcoder.py create mode 100644 WizardCoder/src/process_humaneval.py create mode 100644 WizardCoder/src/train_wizardcoder.py delete mode 100644 WizardCoder/test diff --git a/WizardCoder/README.md b/WizardCoder/README.md new file mode 100644 index 0000000..b8a0a19 --- /dev/null +++ b/WizardCoder/README.md @@ -0,0 +1,198 @@ +# WizardCoder: Empowering Code Large Language Models with Evol-Instruct + +[![Code License](https://img.shields.io/badge/Code%20License-Apache_2.0-green.svg)](https://github.com/tatsu-lab/stanford_alpaca/blob/main/LICENSE) +[![Data License](https://img.shields.io/badge/Data%20License-CC%20By%20NC%204.0-red.svg)](https://github.com/tatsu-lab/stanford_alpaca/blob/main/DATA_LICENSE) +[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/release/python-390/) + +To develop our WizardCoder model, we begin by adapting the Evol-Instruct method specifically for coding tasks. This involves tailoring the prompt to the domain of code-related instructions. Subsequently, we fine-tune the Code LLM, StarCoder, utilizing the newly created instruction-following training set. + +## News + +- 🔥 Our **WizardCoder-15B-v1.0** model achieves the **57.3 pass@1** on the [HumanEval Benchmarks](https://github.com/openai/human-eval), which is **22.3** points higher than the SOTA open-source Code LLMs. +- 🔥 We released **WizardCoder-15B-v1.0** trained with **78k** evolved code instructions. Please checkout the [Model Weights](https://huggingface.co/WizardLM/WizardCoder-15B-V1.0) and [Paper](). +- 📣 Please refer to our Twitter account https://twitter.com/WizardLM_AI and HuggingFace Repo https://huggingface.co/WizardLM . We will use them to announce any new release at the 1st time. + + +## Comparing WizardCoder with the Closed-Source Models. + +The SOTA LLMs for code generation, such as GPT4, Claude, and Bard, are predominantly closed-source. Acquiring access to the APIs of these models proves challenging. In this study, we adopt an alternative approach by retrieving the scores for HumanEval and HumanEval+ from the [LLM-Humaneval-Benchmarks](https://github.com/my-other-github-account/llm-humaneval-benchmarks). Notably, all the mentioned models generate code solutions for each problem utilizing a single attempt, and the resulting pass rate percentage is reported. Our **WizardCoder** generates answers using greedy decoding. + +🔥 The following figure shows that our **WizardCoder attains the third position in this benchmark**, surpassing Claude-Plus (59.8 vs. 53.0) and Bard (59.8 vs. 44.5). Notably, our model exhibits a substantially smaller size compared to these models. + +

+WizardCoder +

+ +## Comparing WizardCoder with the Open-Source Models. + +The following table conducts a comprehensive comparison of our **WizardCoder** with other models on the HumanEval and MBPP benchmarks. We adhere to the approach outlined in previous studies by generating n samples for each problem to estimate the pass@1 score. The findings clearly demonstrate that our **WizardCoder** exhibits a substantial performance advantage over all the open-source models. + + +| Model | HumanEval Pass@1 | MBPP Pass@1 | +|------------------|------------------|-------------| +| CodeGen-16B-Multi| 18.3 |20.9 | +| CodeGeeX | 22.9 |24.4 | +| LLaMA-33B | 21.7 |30.2 | +| LLaMA-65B | 23.7 |37.7 | +| PaLM-540B | 26.2 |36.8 | +| PaLM-Coder-540B | 36.0 |47.0 | +| PaLM 2-S | 37.6 |50.0 | +| CodeGen-16B-Mono | 29.3 |35.3 | +| Code-Cushman-001 | 33.5 |45.9 | +| StarCoder-15B | 33.6 |43.6* | +| InstructCodeT5+ | 35.0 |-- | +| WizardLM-30B 1.0| 37.8 |-- | +| WizardCoder-15B 1.0 | **57.3** |**51.8** | + +*: The reproduced result of StarCoder on MBPP. + +## Call for Feedbacks +We welcome everyone to use your professional and difficult instructions to evaluate WizardCoder, and show us examples of poor performance and your suggestions in the [issue discussion](https://github.com/nlpxucan/WizardLM/issues) area. We are focusing on improving the Evol-Instruct now and hope to relieve existing weaknesses and issues in the the next version of WizardCoder. After that, we will open the code and pipeline of up-to-date Evol-Instruct algorithm and work with you together to improve it. + + +## Contents + +1. [Online Demo](#online-demo) + +2. [Fine-tuning](#fine-tuning) + +3. [Inference](#inference) + +4. [Evaluation](#evaluation) + +5. [Citation](#citation) + +6. [Disclaimer](#disclaimer) + +## Online Demo + +We will provide our latest models for you to try for as long as possible. If you find a link is not working, please try another one. At the same time, please try as many **real-world** and **challenging** code-related problems that you encounter in your work and life as possible. We will continue to evolve our models with your feedbacks. + +[Demo Link](https://1c48cbf5c83110ed.gradio.app/) (We adopt the greedy decoding now.) + +## Fine-tuning + +We fine-tune WizardCoder using the modified code `train.py` from [Llama-X](https://github.com/AetherCortex/Llama-X). +We fine-tune StarCoder-15B with the following hyperparameters: + +| Hyperparameter | StarCoder-15B | +|----------------|---------------| +| Batch size | 512 | +| Learning rate | 2e-5 | +| Epochs | 3 | +| Max length | 2048 | +| Warmup step | 30 | +| LR scheduler | cosine | + +To reproduce our fine-tuning of WizardCoder, please follow the following steps: +1. According to the instructions of [Llama-X](https://github.com/AetherCortex/Llama-X), install the environment, download the training code, and deploy. (Note: `deepspeed==0.9.2` and `transformers==4.29.2`) +2. Replace the `train.py` with the `train_wizardcoder.py` in our repo (`src/train_wizardcoder.py`) +3. Login Huggingface: +```bash +huggingface-cli login +``` +4. Execute the following training command: +```bash +deepspeed train_wizardcoder.py \ + --model_name_or_path "bigcode/starcoder" \ + --data_path "/your/path/to/code_instruction_data.json" \ + --output_dir "/your/path/to/ckpt" \ + --num_train_epochs 3 \ + --model_max_length 2048 \ + --per_device_train_batch_size 16 \ + --per_device_eval_batch_size 1 \ + --gradient_accumulation_steps 4 \ + --evaluation_strategy "no" \ + --save_strategy "steps" \ + --save_steps 50 \ + --save_total_limit 2 \ + --learning_rate 2e-5 \ + --warmup_steps 30 \ + --logging_steps 2 \ + --lr_scheduler_type "cosine" \ + --report_to "tensorboard" \ + --gradient_checkpointing True \ + --deepspeed configs/deepspeed_config.json \ + --fp16 True +``` + +## Inference + +We provide the decoding script for WizardCoder, which reads a input file and generates corresponding responses for each sample, and finally consolidates them into an output file. + +You can specify `base_model`, `input_data_path` and `output_data_path` in `src\inference_wizardcoder.py` to set the decoding model, path of input file and path of output file. + +```bash +pip install jsonlines +``` + +The decoding command is: +``` +python src\inference_wizardcoder.py \ + --base_model "/your/path/to/ckpt" \ + --input_data_path "/your/path/to/input/data.jsonl" \ + --output_data_path "/your/path/to/output/result.jsonl" +``` + + +### Evaluation + +We provide the evaluation script on HumanEval for WizardCoder. + +1. According to the instructions of [HumanEval](https://github.com/openai/human-eval), install the environment. +2. Run the following script to generate the answer. +```bash +model="/path/to/your/model" +temp=0.2 +max_len=2048 +pred_num=200 +num_seqs_per_iter=2 + +output_path=preds/T${temp}_N${pred_num} + +mkdir -p ${output_path} +echo 'Output path: '$output_path +echo 'Model to eval: '$model + +# 164 problems, 21 per GPU if GPU=8 +index=0 +gpu_num=8 +for ((i = 0; i < $gpu_num; i++)); do + start_index=$((i * 21)) + end_index=$(((i + 1) * 21)) + + gpu=$((i)) + echo 'Running process #' ${i} 'from' $start_index 'to' $end_index 'on GPU' ${gpu} + ((index++)) + ( + CUDA_VISIBLE_DEVICES=$gpu python humaneval_gen.py --model ${model} \ + --start_index ${start_index} --end_index ${end_index} --temperature ${temp} \ + --num_seqs_per_iter ${num_seqs_per_iter} --N ${pred_num} --max_len ${max_len} --output_path ${output_path} + ) & + if (($index % $gpu_num == 0)); then wait; fi +done +``` +3. Run the post processing code `src/process_humaneval.py` to collect the code completions from all answer files. +```bash +output_path=preds/T${temp}_N${pred_num} + +echo 'Output path: '$output_path +python process_humaneval.py --path ${output_path} --out_path ${output_path}.jsonl --add_prompt + +evaluate_functional_correctness ${output_path}.jsonl +``` + +### Citation + +Please cite the repo if you use the data or code in this repo. + +``` +@misc{luo2023wizardcoder, + title={WizardCoder: Empowering Code Large Language Models with Evol-Instruct}, + author={Ziyang Luo and Can Xu and Pu Zhao and Qingfeng Sun and Xiubo Geng and Wenxiang Hu and Chongyang Tao and Jing Ma and Qingwei Lin and Daxin Jiang}, + year={2023}, +} +``` +## Disclaimer + +The resources, including code, data, and model weights, associated with this project are restricted for academic research purposes only and cannot be used for commercial purposes. The content produced by any version of WizardCoder is influenced by uncontrollable variables such as randomness, and therefore, the accuracy of the output cannot be guaranteed by this project. This project does not accept any legal liability for the content of the model output, nor does it assume responsibility for any losses incurred due to the use of associated resources and output results. \ No newline at end of file diff --git a/WizardCoder/imgs/pass1.png b/WizardCoder/imgs/pass1.png new file mode 100644 index 0000000000000000000000000000000000000000..a2fe82f458274ddd4538e6554e3ec099d91e3a0c GIT binary patch literal 65829 zcmeFZby!qw*Efu!A`H?nBIN*z5`!QhsW5bbLDTXbM5h$lM%az^8g0}1LK~AxQGG<1~v!- z1M~M?OyDnK1MM}y+YMUp;w6(S0zkmPX z!v{S*JtHF{V`F0z6BBcDa|;U#OG`^DD=S-DTYGzZ2L}g7M@J_oCpR}YcXxLW4-YRd zuTP&od3$?*{`}e3*VoU_&)?raARr(xF!0NlFF`>;!NI{HAt7O5Vd3H75fKrQk&!Vm zF|o0+adC0+@$m@>35kh`2m~T2DJeNQIVB|piA1KRrlzH(rKhK7WMpJ!W@cq&WoKvS z=H}+*<>lw+7ZnwK{ra`IxVWUGq_niOtgH-$LY0@7SL7sDR8&;vrc_o|R^=h9s;a8< zQ>&}1YYNh8YHDf=(`&1WYHMrjiZbe|zt+{&)ql;budi1?a(?Ck8SEA8s)>hA9DX+ZUKHum)N^fs3F_V)Jm_4W5O_xJY? zG*=A_3=FnZ4-O6v4Gj$sv<(jrkF?c|jEs!7*N={l0(uFMd2zSfzUnc4pDv$L~vb8|mthJO6`F+bEXKR>^)u(0@Jba8QU>675n($ccG z(DGQ%^78VE4{T*+Wfd;Gx-_}Ey1MpRWNmG2-B)ydYG8eReZ%kd#>U2`zu0Ds_U78` z=H}*BfcRFd&erVk*4EaxJKJ`k#CF`f?d|QI`1d=T^E*2`yBW5-^JBZayFY*a+*|#z zx3jdjx3|B)f3UZHaBy%qn{aq|cr=%IbaZt5195zO{A)hx*RNl{7m|Ok&? zWHIGrwcupE{N&{1bP0L7R(N`PdbXT;wqA60c6Pp!c7A?-adB~Zd3l9_apmcGwGwxA zbtT(c6AU=OT{{ICMe%n|(Q~W6F7w^A0)f9_TZyaLVqma0qW|4^qNhHEfpMN9AtI#Y zq`h_F>7=QlzUx23-$&mU%psYd^S1cI(&pQ}w^h})A7uH1+x#u1dt0o(+T1QHIz3$X zw!~w>rM^{<#c{{<+336UZ`pER;EHd*y=~`tdzZZJA#T&VTYlfc8xKPpGiiOEX%3N) zxO2}Oj?6jFp++z3Br_&aZ&0cuzYgqZb|)LkW+#D4LC zMZ{IH`YrVDMlUZJu0&YSKVopuZNW6If9kobc^Cad5AOfI|Nk!s4!zQe+o-!3MrJ=? z7#IY)?=AVPVRbh#I5?~z81aD_W?ULdU0g!~Gc#=}BcMh-HTWccwkn3#=R43_g2ctv zw5=h(hjZ?sU|$=V6otT@0b0Ua2*oK=i8-O8vB@d?UxzYzs2Xg|yIzlP0(TrdtCQuQ z_xDaJeAY@=Yamx=vqsAIc^$Z+J^WOW3(H~N@{SkiM!!fS9K+~AKrqfp(*EwxO~noP zo2TFn>xRw@RpW`0*_nwD^u{wufP;Qh)+WE%2 zais-YB(?SQR~Hl}ho{RNw2R(1w=xho6{7Ts?P&mY*%De#|fk-?nUW=!V2D>?5M~x2E{0A*zB#iIx1Nifh?U z`sAxJ>48=PK7d8(TfnjgLY1M=4a(Vd2CAy>` z_Ktxhz1vtd*aITq$)|pE@N&IYImx=o`}~h(#XTohRjVU5uH^hvElKsY)9TvD&`c7e znS{Uhs18gLGJA36G=IN{@x?Q#T^QI50y&n>j?&No3oIW03ODlen1m`Aed4=~UD^Ni zL}7S=r0+yQklH~&#!=ydhEe{ofJa|ksaxnDmW}+gW8VvN!>$72X|>163P1=#CMqTO zkgbR%!BkpAlW2`%s29lpmUt)prstUX+z>+P%2$%a-LyANe4F-)9@zH^24G*MU%zoO zp4(Mq7zL|tjo8KSrP)H>%WLZy7_lYz2F}8p&^(29%hEqMQgV&MQije*9C>Do3@C@# zyh3m9CYmL0BU9ceD*a^f0t-^nh|mbzV)_SQ1@inF2CnGG>9F?o?q2`p9c-HvA@z^n5Z+(KsJNc+K;$d|JmoAc1vco)e$CFayhA3jA8WuTA(@6rRn+Z z6HeZ=a&zzR@O1t54}mskJlbS|Pe<4c3u8=eqPK2|t~xD|Sq^TIH15JSqL>@Qj_X#3 zHBE9S#a2p35lUyilB8Pj2@hQhQ`7MsgInO6DNoU!!;i}s#bFr0UQ}kV6GWguR0}O* zWY$aORbeD!C?k`gRK6XI<$TkyJH~^7^(FX$($TAuh=aov8w!}id6-s%J;O)=lbYS5 z0CuWh;U=u!07VkCFVd2^RDP!$5s(UTjs8TUTAVL)c#EOSAPY7Gt9>EXRdZCBsSX!a^;EQPRl%m72h8aFJ5Wa(^WG)gS8@0+0XQ*ntMD$96giVQ6ZXFRS}G4wxRr-<4otb3_>jp5HTt-x485Z`V9QY9xGp~m7;`bF+P z)5R@54p*{#@T%#>78wg~)#6K`YcfkP-?}IjVna8~%?yySY_6JW$=b&|oPV45V?AX% zig=X~;U(@qDZo#Jt7IZhqJqtMGc5bzw6nB-6NttHqGe79(ChKFy6140EmZSM|H34h>+ujqhGzS^U9+au zhGTRuH{GuhHVAlddr7n^)MZ&Tz4P(0JQmz~=r{%Ekhit4Bj-BaNPs5^HqF7|t5MK&z8nj!j?}KDfZWKT z+z3KIM49O5WD7Cikj@!>HsGi(rTcB$;1Q!s6EvN7EPbmpX5pv5_w+?}rQ_V~r)m@> zA;C&`6CzO?8Z-Cn-`X&+W=ys7&$tIL7L-{*{)$=KiWSNCS)hlx)RX94W1`JORNm<0 zjAPc5eO&YG!lwAA4flgUIsQ^p_|%)*+X4)yKdfk|spA@cDOqy4%E%nntLC@pk<<9c6}O}NoiN%F{XUtto?udYuA81W#Hr?}2xL4mU2&pN?j8g!<{xy~g`kN6J~ zU2gwKR0oT!!YFnQ$#B9PCM`6=xfu?Xs`~SY_?(|?!(K*AFsXOwdH%!CiV{;?imF}r z=YIvJXpxH$mtzo_-W9nz#ZFUxe^tNAUg8`_uOA95!1OEN<;17d56_svj~w<>zBlHn zz6>ad`odE}?8Q^zx$JV1@WD!Dg10v3_*6lO5Z)0+1}0?rX8wrJ;DV`7;D}cnWU0?4wgh!{K^r{ zY`YpA4kzB1mQ#8t445Yo7J9L}HT@O>8WNoRhq)-e6ay&9lh;2Aavtfur)JC!{sg)a z$_<2r*w+`DewX|87Far)!^a#cpaGqiDYJe%J4raa!a&Rrrv&JQw=@i;l}m1Hzy_4) z!a)~U0w#Isg-|7-)>tec^!Z}~&KoYbLo8w*)d<%FsCsRD1nAR8p52nycw_~MxnqSS z-rkPF?ZCun!3RPzJ#w1CZ;}y6NNRm|4|rUiPRj1@)eVnLxylFCC1;9WEyj!<9tXRO zc6h6>9v;g$RX6!4eHeL>n5aF$hb2>a1}&RsiMyyL(Z5k;b^Ne$es=b~PDcQ-=3#z~ z3{_eG)Km;hr?ON#Jh~*wyPT}VA=CQ@wl`Enf3{8pFkUM3o_uVQ9|GG*2T2bdOt68~ z$wJ-^hGJ^8K0hp}b7!S!Y$7an8=tT}>eepKth zeDzrt;6>z-?H9T?uKu?NLlrgNlo4Bd%^}t?bx(YV=gXceT_z&RBq+;Vr1GV-l$bc7 z!;2?@8m&sMXU|xDWP!~4&>8@F;zXV9pRxG7lz z*^z|q38uSzJY`oYbmfe()&F%Nk@&`d4m!@ceESd8hR)rFKGBPR2yc6Tj*M9E1 z%xE1-G010PJ`U$T30GukV_4|tWbpY*^YqxuFNKCxBwC?mzXV;+is!#6mYK}G%N4LU zShzWM4?%Edu9dRrS_*ojR`^YBb698wEY??idmIIUfG1NADCGm0FFG003-j~U=Dw7$=kkqmd>rOnO!Hq`7N-dXA@&rAw43SH(|axZDFo} z#bdPQh=Re%Uw_o)^*OHN{)|3S1T7 zZiKv6i_w}3-sw2lGsS(KZBvxQroRb}?)2+2K9fkah5efYDeb@vT;X0wcxj+(cML#y; z01*}?>AD^i@n(Pb28II_pf?Fvl`H{DnWzH9L+%(Gpyk)5V(l2;Y(XLW+vj^_XoR(2 z&N#dmRk6#|IdJ5#n*0;HZuLWUcj!Zm4bVSP(0>B{c&(#uH!+qyRnHS?uIY!Bg@`B+ z%Vh{!@yV(AtXPQ~st{b1|HK(xEks1O9kH|NqmHl^EvZI2ll@?&r5Jd2jW|hCx5$zb_-~H*4mS1w8gUZ9CFZN>4mce(HAxki~;VSs?aiq)I=D$&HxlB9V zBB`RQc+0p7OJ_fJ5Yu9`fy}-X!WusaU;7W92cdTT2&qp=wS2aaoyF7#qm{{)x4;Xf zgIC*2pQ?#Zdn*+GWFt<#S``BE5@3h06^v~*NXFu454y|XE zft;;MqykrK`K~^%_}E)P3X@D;_4nv6-t@NzV$3b4^ok1KFDru1;Qe=jDU~;6SZ%g} zOvf=WmFE(ViLrrhOZ}B?_xOL37yXc$I^a!QtaDg_@$1(;8uypWHbcc2TJ2ZO=nc{o80+cSycf((gQh?d_S_lNT_CLjP@9fcOs+1sg&WpmbdZ_g9E^kHF$ z-vUC+gP;JWcYcrh7Wv3@U%jX2O1&Gq$Z1V@hJ6EL?j_x}%aQ2YSy zG$b1#LRSESH*+#Th~2W-i}g?H#ZeP z%hE^n_&st+5Z3kA%GNUvl`iZlsRl46@ERvIS%x_irFfvE?hJ1*N7*AIqV4(bUF)*3 zP>u=w00QpJS%w%oV?jASsG0NpMb=lnO77K(tTsz2xMP{BvfqRH9nf?qK(1`E@p100re?kbnq=IKfHXURqw&< z`QQ>a9t);PMpvjl%72vO@7+D9mb*+ZNs@{4jPKA}dvy?JM^RvbI>4VGQHXwI2n8C8 z0G)S^*$7ZjnYv2Xx6ew`*yexaGugc4Q*hZyt4`Qh_!uo1 zjBG+q8%R>{i2030%HeN9AB@#km#Bz-jMWM#+hf7Wj^WP?usEV#|E0W}0Oe7dnr6BA%DU-+G?%-}Lo9lH zxt8`Zj~E~aKvx2wer+1@tN!i0gTir_L(}-;ctu%o0j+hrQvyq~Vpo}iP*)kT?HhmV zTLnr#D8pyh$@s94xkQ3z#{ZAWW>aw5A`8ydT;ILTGUu%-ODq^EnLHh4{M6beBbv@# zqt#||y_a!7JA_r;jVE7T`O85fBtmY5%dI}YLAtQ6F>);qL6)4HYeizvQ~nWhJo*3Q zJ?>Ubzu;s@{NM*uQ#SXqQhm?Wb)ve3 zwuXwTm5gfUoP?7*e`NXf`Cy~$C#`lF4RboJ?9hefUYW3Oyrkub(pqh6@q+8)#wS9b zw4S6*e!s;MJ@RPOra&&9^hw*$91VDr|4VO8=*Qu;xMAa8=gmls z=F{JzP$R}_<|YI08}w1#ffl1vye8lc$fq27|?*QZ7UvC62jp zDde(%%vvKp;iGkm6-j6@2`=BY&rFMj3`RpnfV$Xkyq`<-nbjJf#Ws`;uj=dsZ9PM5mKvvjc;3Ta$x z`ls$EP}sR$w&*uC9zHSIN>(|V0*A3vSN!YyeI^9JrzUE{!9v`!p_Fn(?S97G`O$U{ zt3tO}3dpekm_(<%DOe$1k$mIr?W~w%r5npN;?}$_}^E7aB z;5EV*(MH9aY+0I*Hz82n5T)0?{j}$2JDca6@s$HwAQ-QJ7`go1r$15F@$z z?!MILDITVqmHMa&gBG6bkX(HU?gj!IpkV?;C_cbxB@N#CM6S4ch0?CA^|1D?>@hAHiUD0enJR>tm4K(lv6qKGIhf zohhY}VRBl_XpgR|FS zlA?&C+vteX$h_bP(STRVtZh^5KP*X6H8ss^ku-KX07g0T#+(NsC6GtLXAz{g&uvv8Vb8(KLUi@5z$0>_-u$RX3yls;r0rvTloV>^G zolG!N#+Y}Bx7SDSY&~i)wV+<5YGlKtL>+Q%(*z$UEBmvOOK{AWbP6tK8v+|?3hK1y z+=ST=f9PCxS@S0IR`i?;o;s|e=`hbO)ri|70nJ3>!OVeqAJK@TO0A1WE0IMahq zk1`U&A#Aa@&@$oc*e=*Q+z1yz^(ROpVX~fz>(Zj<3$$JRcPhfa!5M)w2!Mi;l(gnr z#KKh1(O6zO%r;DOZ8hmD?(&N{t5yO{qc)%s1V}ui@opib;pzdkUiob#Fzi zcI6d~Ap#)w4qpYX4Jbud*B(;XRGbGNwRzLGLNj~Cnel>jPw}~}td3s*q$UiPtBm+B z&VU2bl8XM#k>yr~kR93AHpt{ym*2l9_8EWOR|wLxLL&YoImX8h;BaZZ%_5sOAlv5? z-9?ObIes(=ZqK?%g*Ow{1hlaKTG`|WESUt9uxZMg{L0zxs@DjRN0`4W{wg>|^&r7= z9!B~X!OX|dLx<-L)#qj&kNG?m+F!F*_R9dBLSWaViXiZUydKjV zP$~Xnu_Ay&no8%h%e1UM+oI8VSk&L-xS(a1jK9v;({lX7m>SX#GimF8o&K)#4M}d$ z^ad#ds(%bVkWP>vj1 zaZM=Y>+)lpy}j-k4OJ}Ks(wcgZkN?>lp(ouZW@t_;W-T$<;`Z`(|8eJJ_Ep|{=c=> zB4nNJPwbbTtal6`0~=D%oNRZtZ=V4TvnIG6UEC??h&E*ea& z*m@vWgo#UOj><3Z1!~aQd)*F@(l_{k3%><~+S+80q-nJC4Kcv9D{6zZq^f0OJ6bc= z)yxjHC?;_o&YhM=HI?wPn;?AMVkV7H(N-Di@5-fPwOjzUzAMmy)wQ%^>pWfce$I$P z4_RW0KwxzUSOJkWqLU3e5n$#72H*}Y12vV$WlfL!b5%|&SQu2RyDRnxsSxKnrXmIT z0tQn)6B$@Q_+>^}L1b9V4kNYJY&-!GuDofEa}7zVeF!eW0wf}bkCV^QCc5~xut+w4 zl2;Q7_TT7%T|S5u0sRDsxXm6Qh!)EGumg`;n^?GW{7FF9sV7FfRy zInnkq2O3R)5rTj|VE55S%YY?Va)Z+mSxtWzX9H-UL7$+eD%(aGvqmZf+su?{PFIae z!3&2f+Fps7$4T1QG*H_|6UXRp8V<;-qQQiKQ0Sr4fCFTKNoW3Q$IFW-CtTFG1~W$# z-_Y1ROFV6ynEFY%IK|n>{^%%m*M@xHJHBhr-+&bptTS2N*X$Z*v|LQ{to+;z`}#*|FQy)B;E7Ca$s1^BQ0YmV5SpQn5-QE`LOR|=c$9v;t<(GZ& z!dE&veXBY#bjdn8It)xiy!2*I@W3W0EV%V66VAEsw6fw3Ys|7@qrCL0&h6OVL}lUZ z$lXJkzZB$ehwLi`N%yhTwindVHbAglj!s7=CmD+s9?aI3e-{hI^S5|y-5y$0BMYty zdK)|OLCsY>Y@c7qS77dQOsVduO5TB6lFSc>^g9{Xp)ecdZ;$N20)jYm#7wO&NL{U? zPE%W(sJ0*Sf`zrAds*C1+&BH6GOMtFHh5v8)rsD{zNh54tGt-df!bVmUGp_=b(R)q z;Vsy8IvIwiSNE?ic~4{ozNw0eifTCFTY1x>6XorAO0<2x%zCwc`uTpa4hlC6lrfxY zM*m@#$3o5A;RVtRAl+D=<$*SQTr~XTOn&9dy zqs#Tu)>e3yXAux~JzSq=>4~Glb7)m?WwWi-S#0NwEuJMH?RVVrR$1~g^m_1jH+rS- zl>co-WAIGsUICAipNAcqKwOIwjYi&^kW>i*nQwz5tnR)wYvIphnLSYn;k|$x=vf-$ z2U10kAfvw>|0B?zsb2=vJ+D0YU!DE5MCY`=SMSL?E$in4khz+EZNVM4gqXQ(Ebhrb zBMCS4=;HIadChNU86%8K^U1a!J|_UW?-&?>K@Y)a&wj3mIGoM{v#(;$@qyIS*Zb1B zzV_5*Cg;9_Xnx{NhOiyp1airPHD{hHw@mo$rx7!qUw+o=J)IEw&KIz4SdeD;JPtWT zyQs@eOxJ(i(7JXw)z|EfG*Yk5*V5K=wse?)M4p@!=XwfL!kq}x8*s(V>EQBH$$@ET zV4A>FqI{zX`MvYTAuMCh%`F;e|AH9ykTB@s#IM5X`we_Y?_6A5Ol)X@^S`Cx=C;0Q z)vC5o6Z$~y=VeBHebqV=_-wn~6O#_JJM?yu+WvQnoq6uX>7&xUsr&>b{s~4l8<*Z^ zijEB5FBQdYn1nrYvtmbY zccyEWoTlnQ2i(KS%*Msxy|E})IHx}dX@IUTvtxa#`EarL)7#O~PE;_FQ!tU>H z2^G7qTu%9anNa<8&b;ol!WX4swW_@Y@y(gCl(`UiiYk<2>^qHS0NafOBE$-{7v z`y8scX6T?VdTc)do|>4fOj`2sFT@j}3GSS(vSoPgaIt^pnTZAvzJ`v7Mfsxy%Rn1R zcIEavz@#TYQAvtcYfY!Y)!B*J_Uw3jZ@@HWHfRYpXRp6>d@%ukG!H8Yllm`orNA2v z{QQzo4(rVo6h2hM2Dy9x500>{B6yvFuFy$=#UC^XgppH_SO1O!{BipqsGtl%Zv#UU zC<=+m=V5c`Jz5!a+wKlTfR|v$g6zuT=+7}?TB`e{9gqVQG}`CiuaeQyYWgRs=Z0Qi z1eI_^GKFcaO*%zuKKaGS294b2o0*t!wn<6RCXVAMSm#exWVJs$XfZHBS5z3q5l zD+&M~U^T&$7{0^V&qk(?RD9J-IX}o6S|W~rhb15A)?Bw2y~ZWVKtZbOF#L!`m>7IW zd^k6Z>?oc)ee`xmhx;%SnqG@clh)d4)UQ*(v#u5wMl`T(kc;#jRO{WND({yn_mf&x zx=aKbDmmxoUQ-O&=!a5wEp*Zy$W4gf2pJd&>u5slqGgu8iWN+EQYn}+cP)zH1!|el zMd=A%w?`fTvzbsiqkDM-MWsyMpW8Xo$!@QG&Hv~znXRoNrhY$2w?Un!FG_(0fk@pP zbVdzkgr<(T@($^`zELSa+vy6_H#GF9X{f9Z72ld(L)eGz}`&PM}77UUzP|n z(6vW~c3TBSnw-W%CU~G=E%DGT`{@=k5?38*E)K(YYN1%ozCW0seJIBD1aJxbHB#JHS`3 zXc?EK8r!|HH$#VID%91^Dngk>tNqH6k9jj&y-Y$)C4uOnR#w*A6ES}mw1BsWtdQ(? z6A172oY)!qxsOTR2)^S0vb5 z@qol~mL8R`eg7-nd4bGBLdl!bgpwnLx7%LR2S-id)||4_8Ec!($3wU%8$h?&Ws}SE z-Y?i8N9|M%fnwGEK_53;*b?s=thU!=b1HsL7R%R#qxHHR7t{g>23nD8`1q`mIDQ%* z8h|7XH)Cir|#hC-pTz#a=el*UPBT+xeOKX=aa{oH0fbOob%q z_=!bTp+rQo^bwI8;XKk)Ysm;)!@2qFLw?7b3Qx&eOVZ6lGLIQ5lFKPKnAo(`^0&UC zs|&%}M!00t;}*sidzj^1C|20*I&}?=M=Fxd31g$9+I77VcwzU#a4*J|2;HF8Dx=gk z5X%*6wO@;GPMD{geJ$!s9`Y5aJ^lJ)oGM#PZ=@f22~%|%KF1|Z4?+ZG(GgQ2fJMCS zU_zB}%E37>_{{_Y=l%`Jo#p-g@E>rzDr4speW(~r1OT5#p&Se3$w9JR-s@_vZbQY# zw=}X$HesnqD1Y*6MvmAZtYgRa`jLZa&j+?v097OYAFxO79Ovs{=>Y3yg$RH@G$o5!UPu6PNSn`7UJQXgwtiBt1x*(Bd!eA=tt1bG9 zzF;lCRWjQHoQX1@OldyziL{CGaN%>u_j13pY5tKr!YT$ndQ?7n(R;YL%_Lk(gEG@nb>plNovcQ`ym~3 z5=D;yl8sM(z+A>PHA&SqoKN_2%}DoZn3#nvuPpTlOLYCtma^l?_P>zvgqzJCrxg?+ zBS)5(&cc9}T_qCAaivySL*Qs)Gem975Q=tMDJ*EZSS$!jk$|!6oF9NGrwYOPMtiRw z2mNN<^*CDgoit}wKz8++%R#}u`KnG2V?j}G=%kP7lXPqP?2N-}MJJ?-v9tSP&QSdbbg-)W_o!>YcApbg^+c6swcET`HMVqr)>5Q$ezEV_ z`Ovd%!bjeXZhXhy))Wu93+RPQyQq$y)a5q?T9VI8Ws_Ko>a{%;pLne3cxbH-*nJ-( ze=;^EkuD(MXtQqjuKu*3)y^eL0K>!KguVfP%YeyUV3lUUP*0k(7V?4CfRDM>6VGTqIyM#t1hk(=UZQ1F#S${B zSk_DbIsF3a8uYSX=>O?)DNisi_vsJI_ozi>>YnyK4L)KY-<*n83{+x&QA?FNOC{88am`{R~0YR*%ngw&XkN zG>+C|agP1Cas2>={J?~!3*Q1(-(Fr=jH{_A#6v^JbF0H%ZysvphLiH&#u*vev5&1G zy)E0iMTb(hB{~HM;j1~bB;v{U0B^FT}OmFXo(%1y7G`5MG#~Ye9>7fq}D1X3s zHUH=4S#B)}siG~|#vnp}D~UJt@$2HONWGE?TI^)wBDqE7~`R%X)}Sli*Wk*T^4t#Ca|^7nmNvYQI) z$nJh?bvb+1_cGyp11sA=GdN?cIuxXedt1veY;>aZr{#8Wewqm(K-LRnymyN+CGU&{{>()(f7*3 zGMMw;uykt6lB3W_tr2eLqq#O{_y?@hMgvdPEckGwN^VI(sXN%5mQf!Vp}{l$Txv-wF3MiB3==ap%{JvI zuzk+F;$8&Y!NCW(PrM)i;#cz+MO(MVAEzpOhgbTR$gr<{h2}gft|Ay3h{H>=^mvDF ze)r^U%DZYU^ldD_ZPvF-x}ar?^6Tr^JkH58mMtr7W!CmE4y+$a#gmLCjdxiR_ zF>33hh)DXoFFkNGn=I#Y<*d(HS*40SJms&}Ox}YBBZ+}@5dp*kLe+!{qzc7G0fD&*+b#psb-3D#lcDSO)vj6IF;MT7JGdH*)`o9rZ35MG=MII#C|Ks_y_ ze%mnCv2&=%RkYC7N@^!5>&Em{ii6$TNb)S%@Y%O28jY?cWipByBA0gJ%)t4DPvY$w ztfPVIMyaDM7^Vp@@PA(n$T5v~py_mlg6SXu?J_OD-<_(z8~7fkh~(;xr1h!-$BMqp zWA);##T|f=%AmF&V+N>pvoApZc!z{4cVWImFcJI3E&kx{kuu7+^G zV=)nxc>MpcqeCan*W70LWuH`WASSx-btjWLUVWIK3ji6AAVItT;OiWLxQ z{Fp-fbSNJXL(AfI=G@+}D>-RAoW+@43!`wZ`1Fi6u(uNrZfLpaV~w%qD~7BVag>05LR2`PwS*?K@2Y(xQ<8nn5n< z57dC&7~H-VbRr^f1m+2?RLlwi$Vx_h?--#mB^^`Elk(2F`OZl@^ zAbT}|lG|pV`(ne@hd!XFV`idGxo@OxRJG7`>X8ymO;T$oED~D9BY{V_G2+SovW9rK zXTJ_x*(Z9K%2T4Ern{DxAAJK+*$nOPcWDOu_}Ph1%GtBijxOTZHw(&)ZA0qp)r-GGBl|NZ_OFo${rN4=KGpuMfg@!efVC9EghZz&~evn@yfwp z`SbJSjuZ#ryA*V9Tm7OQg; zJ#oq>o7YFcq9!>t`yaW95o~?1RNfzn8Lj#I9j^t|aC% zeUG^4M;sU=nR()#`986lm>RBGmfr3ePy6ZCZP!uWHjGC?0)!AAAdoHK0M7~bdQUL) zX)_aFrF5zgdX^l!xF6i>3eAs}f0Tdg+tkF$$Es$}dEK2qQ0lUHUG;La1N4UM`#*tOB?S zU*+xR*?>5$31h|>h;^hK682HrkKc$>{mslPW&C&FAhMu1aE|!aBa{vD4jsd0Rpfy< z+djdDP88T4mXmbYae;17`(a)y9hl6x8aN*^3=9OxUTLE-;J-S=JNI9HKEnyasdF!_ zx;({yb?Lqk*kXxphSP4J@3na{t8Zj#l5bcU+yK3-)qh~LlSjFR#03uy>8nj+^J`ww zaiMZV!4_M5ET%O!YwQ40Wcy?6P5;qThTweC=y+?YMCgc1LznyCY9?{iu)sxkana@$ z|HV-fmSepvXYkYXouxa2-*^-c)(=m%sb{1&VB}qr)}L+$f%aOAky7GYN#Y^Ede(o^ z!V>I-^R4CvW&J$AWX~q{NCv`_r6E~0^s+cK3%UK1on3SCz=v}M)ft8S z08y%F&X(ud#+qwD)8|%=40tjTdlvAQ767{e3CnG#4&PEnFN+5F1_=+21E7I6qRRKU~y!D5UY?&7jS2VQJJqH)l;4zK+|4NuH$?A>TC^r zc-7gv(|Iu(X<7X%b+kHM40hH*@@#r9!j)GbZeH1BE~QVE*48a^0kFubGNn*~89xZO zDNyOs#Rf0TA0g`pd>_yY0L?4Y92_%erZc_ttrz{x`hlu|Yu^NWUEd0UX& zuzb6RXkbS4Qt*4prAw5O5fs>kW)zD(pOEV5=haGa+#F!)`yR*?zkpH>{ap?VfBe-4 zBrP-JY2~@G(=x7uW+lY7@H@B)b=0$O&t4r*N{FKhGOyKkm0&#l`2r2g4Rg(MFkK9o z*7$sj6kNdUl#`xJ-33ADGv7p>CeROl#m~hfQ1i8nmZF0eJ^gd&V(+}yk#-gV_!k4C zQ4S`bL1W4BK?5+%Z9iXpkeI$sZ+Jm=xb}DrkbxZJZ-!ZQ8tl(3xcGS3xZIZe(6tPf zJhje@x*D~^kCDp6xTr>y5jplS~mO<(IoImWYq`(vagDmb6JZZ(nHy| zdp=7CP~u-MKmLDfn&&maUp^({8XMDo>AAmli5LKy(R9}|Z3cS>3#^gO>KPq$^alJq z94+CI`c$x7z|pg*ukD~%%gAP7G>C}?U8mG=IXJppPVB3Dl*&aOl;*-d1CJM3zBtJ1 z>RT}^Sse3JLklO~YWfec!Mb?qB>|698r2&8k(r8T|QI*6VM(YQ4^Qg$gPCEo3xu%Ay#qB7tU zSP^NeKs}4E)USZ$ARaRWd9nrLf1d{pA?RyFlIe?W&>tnQ_=rPth6?Gq4Pv$*=VAgQ zjA%fv@o-n0n!0Pc-rd#ha5BRYpyK#X2^9&Tu!4q8fh_UIuM5w^88t-REN`>ezcUQ& zu*+`!3MxMExez)=s03jHo#n*y!qJM90{@qit8;ps*O zNWeLkil&|tC|GLH(t7T~Y$}`O<9cal=a4xAq8wWVsn!_H^EA$ zCj31ZW=??dkQ{BDO5u>NMT>ZGVl9d<0aD zxK{PIfFFh-leBrBr^6*7s+mtOPGt#9MxnnQ;33Z+(ob&FYOW;! zwE!4;&u~Y;W&e}r6Pt(0KcAH>8y*ZrqzQvPg#caG7vuttw8+unl`L!$*j}1#$+eK1 zGGwCWCvS5Kl8-y98tMJ-q^ZOW4!`??8`u;c>fFCi-SCUBMvnB~>r7K+7W)+3ZvswrG^D8ro$??rgM)0Pr_EHe85iqTUJ}MLk9+jc8#%h8d zUcL~qMdJH_lqvp8U0IbIU9B#h@zi^gE$5b|-&FlM<_6D98Kq}3!;SC>mlZ|Ghg>=a zmOQawvx8fyS~=*W>u_|yKmVw-3fsH&uVPY3`Hhx&3cFG;FW389nrqy3hYSo$oO-0V zrt+q%>vjx%l4MC=^JPNot!Kp>j*V0&(VgE8Abh5=VM4&w)D!SS7W?bEsW?rB`ksC8 zt$9a?#TgE6_G*$)C%mO3-Cm?1_Z4EW>wGdz%ea|scXw-xKXRuHw-m#R7qI<-*pFZT z3~KD7hUM8r0+0c6j{-35Ut=*!lMe@TZSecQPuglLxclI1eze!}m#yMs?mYIyV83IY z_}Dw_VK2hl>Q2hUY6d@Qi4V9?=aAjNum=!znmlGsFI?p5=(1o*ry_Ny**H9H;=+Qg9u}Ixd0h=p#39Y#(jATwVDSt zq0hdU6JF17eB>p(GS1*h_E5k>(NKi(P6|>;J~OTgFGOpl`KzmRKpqAgu79`~qeerAqx5;O1tM%rA%Fov!0E8xfz4SDI z*D&;CTkORNmeY;Yt`6Mw0qSi@0e(Vr-PbKQu1j#dTSQ)3YlT+G^Y0YVka^PFUVYbb zKL;hcgAk%4$q?>DX}>F8HG+=bKjnvM*f1mnmo#{%SiOe7IhIUz7r9yzCAnGi2%mCB zR%5C<{P~_JQF|rv=So1iYM&^#tu$FMF$AfA5=!vljhCL*MpBjiCLZnwEFE$}N~m^8 zu>2L#Yl~^;pl+a3LdpCG2mi+?Bz-SEBYmx!#VLE~cUbiY*>XuDrak0pQ8D7G$b|pXSy*X9BhUf9eqP z?U7CX_0a6$68*JDj!Ju#1X*lLi7zx?3=^4vy8K5FZHl#H5-1`w^Qr)&Aq8Fir{Y5I zV*pbHOyZ;9O}GEU*H?!{y}WNDAT81>NVlYd(&f?}f=DYZ2+|#*EZr#ri%UoeNJ%Lu zqI7picS^Usv!LgA&iVe{zxTR!W}bOwK2O~DJwLr&5Qng=cmn_7F;_^*>9oO54H2_vXRL|+1@Lh`Z-V$ z5aL)t>+pH2ZwV(`>sEp#Os?kWbDFwhkmdQ`oup`~QR5RONO5>kYFqJ$8XtHC$VzYC zj{1?aQ;A9kp-A<*dF*6=qoSl_i{H1=EBGMa;fIP*dTQOtocL~E)6G18uBo150!+aR zpe8c>a@7Vo!tF9L4qb)s!-d5yptSeBoG7bug z&i7gI%6k<{_^2q)3_yAJOP56b9d`T6^?zVmknx%LW?z$9F{ zuIWLi3FStd18E2VHP0(!gtZ+yRTnuKH(e5A-NT<01`HI_F!rD)BWjHpC^-OSOcj8k zVe${S4043EzNVb}JRp{@4{UlcU{n-?(+;49X`Uc^nDBCpr2bsw&DA#@pDI$YpAEI$ zokv^U8ldZs|4N;7?UR+@+LA-;%%;6k_IQ9wzLY{i_Dm|t9o2;^r_viONqnC8Kgp9) zo@Mfc*8+MN@aKslbdaI!!Wsp6;k9%Jt$C=|GaNH-&WfEU zKc`NO?jiH1R{$RKJf!=nj^|;!lR=Q4g9w)t-_!=7Q`~!D8wHUlCC&i|PuW21X_DWJ zCT;e32xen>3UxD{-7P(|a0@9&!8&D&Y0q0F+ZtL>dSt(_za+$C<4ZE}@KPX<5Tt1r z2?Bw-^to=xj)olb`6j2H+Gab1=8C&c@t2<_x!;(H7^8U|tV83t$C{HJ*_!zTV9Cfu01rrt4opeG2)c^m^U9w`T>c0V_}r;?b(89NZF( z)YLq`T;+m7ZUz;j>xJYoN%T6;Rw>?6w_06!|C}o8X5S+?dJHJ;fo`hPH43>k{l(G^ zgQ{CTuFIc+L|@;m6o6aF>o9x~4bWuJnL~o9-&NJ_Z5@K%&qf#$^P7ZBjW);6`mA*D zD%BohkyJ(1)paGjYL0(A_X<+ZcLPto%Je$Tp{fE0fj;K#*9oMDWU3}c+2$pnGH%HS z>K%AjGQV-%7IZ%Qe41NBY~qVNRN`FLrUL`~8GsJ*-Hh?DAB2g#!<#F0Z;`*B=&IEYqJwL>Xq-T^C*@?L0OzC#|Y$#>hAZA7NZD#wJH*rmo zHzl$x4GtODWY-_x3YtCd?Pr%%JpF!vi*#mnb%xRpuO_^%e}}n|47c>uufAyR&Ij2T zrY$m|GSV41GiF+Zia4~zE(935duU(^Y9@;{Qks$X!5>u)D7X+=?JlcoTQ1*bt68xh zlY6h60&EL3p^&y!2C$X_;bG7H64!6Cg-F|x-QiJWFNx+T`FDD-vAgMc^^8AZSNzzB zO5lgH(Y|vk)hz0NoEGC=8}9ppiQ2*17_7}tMu#44G2WQN!ca|`iUf~r!RHwJQp8o@>_NXw3VoZd@1pgilH~)t}WZ;3RgU$}h8Evcc*nT#ZrXvyVDh>643< z+89_qF?QHmPi<@JR|M8O219LbCt61v$l){Sc~%{`&94{aqk&(9-MOOA?yN*!bqVBZ zwXzS;r@DH>Ke4hJcQbi%j(twubEIk86jRQQGJ8rQg(N^^38t&rg`=uWK(%BkYdM`<)riv~o;{D_<1+b)+i+ilx71125$6x400HXIuYt#7)IJvZ-cs|># zN|c{h{gVIf{A;mACkuf}bd`Uz( zWO7d&KgK#Z>rt?=23G5#Q(#bPzn_1TvlOKR<@1%Y2)6v>DLfPh_0I*yaBAo%Ms;$S zonzL+I;AVGo9xZy?uHh|x+b-Q022WHFI+#a%5g!(eD!E<|4a#)1OK;N?Nol3PTE_@d} z;#Tk`tuQ3~4Vw1(NALiQ}0rT=VU?tpzrYz=&*W3~RWVT_a#4D+tzB{W1 z)Z6C0EGnn7+sr|?HDdMzE>7^3zYk214t#aVT2<8}r%2XKw*N&FE49o^0&6uS!sZ@omDVb2 zSW_{b4+EqsMWtO1&hSuIHQ-w`3{3G9hM%Q*Y#LGYd?vNPiAiB!Y68+7x_B5^vP(wZhi&_tH4;d6ef2krdB4oNr6$tj zpNUnBVxz#PknZx3Ge&4O9#IfYRzjVf^km*kD0RJcwkmnwr(C68kV%wHmMbpE-n2t{ zXnC6?O5v$uZUj#<_^<|E9LNF#OL~T1893@49xZ*abH7`NO+LcjT(eFMHlj<)ZDaIS z91S_nZ|2HMZ}E`eV>YlcN(GTvdW)v{8YFo6czdV5m0OH@M`zC_#!@kV`OPRTK8)Fd z$zp5h>c_nCRbv=2Sq|!!6WyrOFJs_b7-Ik?5Y@IAewEwDBvYc6ASXI^I^A7J+K>-o_=%>-Y0X~Oj zLD%b`jy;YT&Cga2Jh1FJ)h!7;y_+?nfXZb{S{TLfzVyAIhmJDZByZa#Co{1bVEP7f?==c+toQSv|Gw^6- zz@Fttsa29A8*H?HSsq^VlbxCpte-#k zMi9$6fUQ)J=-LF1}*bYiyxkg6kJ%YTs(f}eJmxs#cE zxtQJP=<7;^6SL!<0QY4f%=lqS7(Ppq2BON6YsXPlE`aQ3m74_a-jEhqii_-2?${((uT`JcU9o|rnawcy zg_j?K4oxa#*B&#BzQK16u}bRQddnf4sxa?n82yRS&WukM(ORM#k=DLLwNTMj6uiwz zR@7IeHg(&E7Qnx+;hvvef7z^L5;@|ksJCPzZXfqzG#96y0*uJF&bD*Z`;q*HZ*?N| zWUA;{%N@VNUJ>`xpM_-=Xo~BW7Kyz_aiSglGE|K-Np;WZ!S`~is>kK|kPekxlOyR- zVM-$y*W&&aO&=$M@L2#I^#?{<-DRiBPym zAUl|KM_PZtoihG{J87A3{?tsle;RBcbMhE~I~Q`ucUx&)c2}ZA5V3BrgvF{d=HvxW zzMQo_Da40kH#mx|3y79wxzgpwE8+qT4pZP;1ac^qYWiI?nLsIXz-TUkY~H*i$q00Y z!v13L(KiyLX@fyvXIykjL3iZZGPtw*Af+qTQF>n^rm**u?%z>t`p_vl^1S4pxuf%s zW~~aQ`L|6vxj+e_`z_E&_{Tsb(IkJ=?ST_95bslWaB~&S^G^(XhYf)*&exk$wbu-| zW6df+jJF^!h05@?m47g9xclz6sNhNJDJuygVJ*J5NB7Z99OAQ0LwenRyWSHhuEajzh`=+sJ;p8YOG~#!nw|!~~b~Nadf) zjZ0Zy0kACEa&b}GShbPfcRxZ0I^Y@5;Cgw!aRSVn^xvW~M9B73vK&9F@b#j%s%<;~j+wZv?teE5d$D}>+P?S*tKo1og1rmZw z-ozpj_T+H*WM_7<>#Gqc_JlpH;c+&=k z+|P3E@riJBrN1=`?|M=gvyBWfmpv`!k7pwIyL3&CR7UL_o<;51qRfPc^)RjesKCDxfQqaIWx)wi^Q$N#nX z>JPW0=rI5LsU7}IvXj+aQC5}K+jBlCQxW4*3KsX^pVzIp-VaR*?tCdXr>gcd#Xka9 z!NukF48sZGJY1%EPQE03{o=cXc{a1Hel0;a3?ra%!@GftWbOqaCOt_ZS%_KvsdKim zTxx9xiLU9M^?Ke~w(>f`Cl|V>L}{nCEN`|&_!Otf-xf{IqHq;QOimm2@B~*te!j>m zby|tM^cn#QNT6a((E&3o{6#cvmyXXjUQ|oNHHch?4t^`0g?5Ww>t`6hOslbUPt5ExeZMyG-Wsm23 zCpj+dsflopQlUwhfTR{2#@h+F?W~aLDw>AdOFMGhfkUC2i&3Dd*!5^!BF_oYL=(0= z#V^4ZednA1@!gZsNyXdwlusZOC4tkSA{}{jA&~ghw*2?}+yF!^>7?>TRwZL&CP(;o zN23e4c90k}aP3@~gSx_S=lC0>9*8Ilomya4D$IgG^qIQUKTD)nMU8If!nqn4rYTJ7 zy_r4xa*J=Q4(=E;E59?yv8&y#d|J8&lEz}?aXzI{*J=bU|Bh%U6xC-W!AglpuieW>X}u$i(JN+?uCvG(r; zIcQ!<3#NYzwvH)1u+%KKCHlusW?9dDbU5DqwMi<1r|vRIn|2(-GT)RZd6DzXVCzn>$Jdu6a+N#O(>Sx+8>P7I+#CfTS{8I1 z7tdB!4o_AbX1BrX8m7|FK>sW1(extaGdMs808VI6`Z|QkWXM1V&}6U zD%AN*iv2^-1+i2G17Hd5S$@X5MTT{MvnLL)dg!{_0k!ihY0wJPZxw|!Jw!{;>B$#x zY+{5bzB*RAx}*;Fr#Lygj8KK(un70N6&I9Im@(T^v<$Im-qXx361DMV?R=zc5MdvuK_dDd zy2p})W?(CxKpaZYfV*O0( zUv!C(kZN{o|B%${))EaXbr)=H{0gdZaAYySk;S`$&ZSA_fKaFY^06{bPeO>1EXVKkZ^m8k%@M+N0K{wI|GI588FG)%FXA3LGHH^<{(QB2pyq;l=3V3NJZhj;r zEL>Gkuv=+RQPw!fCIcsYi5q6nG(Wp%;7;E)#Z1gbNl!Y`OmM3c*d540gOVdEgblZ* zD-{!f{69Wm{)y~&|Ma)e(l&6)c|a;wIfEkwwD$376*4d?<5lAr_lN-Po3xNsGUWv*5>(={oW2sp%)=UG3N zehx>PXPTZVwW!bD$=B_LP5?`wc?+~MNO@~CirKsz8cDHgqEe|O`tAke$LTsMpycpj%@_^xASjn+`soeVSKh1v5kjHa28e0Gtr9Jz95u_gUPgoRNrZZi zN@9*;qSMM5w~Nekwc`5Te^q*eQQ(czhyQ!ar7OEmWw%w+IJvk)PCb#ewl$<2N58_G zkV@WZ9I5ai&ks$#uR^+Q=r6~#!5DkM!PA>*j+;d{{^@^QPRsjAa!n5qb>OdYVSy{` zZZ3HIr0S*^Zxg896WsS``Xw6gW$m(t(wUl2TKJE3;!c^!^!Xois^Flg5Ko#&*kIA- zeN!t&S3HiRrBSgUuRH|y51Q+HS&-5m+GIKbtXTc8?+wQa7fP(IqI3!fqeSld zN}g#4M@eZga#~>yq+9Va9iraytUsl=)n}~Z+3@KEUwA2fa?!=xeuz)xSVl@nG4I~{ znk-Y7XCEFc^{@>%q%(G#LPVGsnv>%+;zigl#fHjmiZp%S8AXv}zK8|D#{9SOkp^@| zq?rO?i81#9n9I<=(q>lI_>dHX>enDPRm<7~ZgR$? z(>&;$0CoVcCOqGwtp7$#AK%AdHZ$h>1Fwabh_Y0-qG#&$wn&E2*lgDyslsL0XE(!AoAWkegBrI-;o2qG?ht&l0(fetHV4 zY8*<*+nXu$pcA#jQqV_EK0o}p6Sg}+mnRpTAz2<^pGKK2=|mH(?&fhd8Ffw9+ZXW? zA%IFAjRjvbOy;NwUC;x1Z(lB2@D!~u=InLxN^xgb<5g~CKdezudj=PNRDZte78n>7 z!cEVFnZtEY;|ZXWXl6G{Sz&h8vmN2uYemJnRrB6Pe1!0Y5xpo~Zd5m@NxdJ174lFO zP_vbOtqvp?|DzMq{aYGF#p;s>I=zAAGD9@gq|R+f9vgBXR!H8cDg9>5wvy>L9go>Q zqm*@>cCXJSZ#A7;%}X+=!F1_rhH5-=hLY`X^-^1WqTD;P3%XkHEmn z3e;F?tWCCbJ95Dyqe@LHDaIJr-yjDb8{7~oSI|4){vrPbMAJO2pdj|ds#BfX(5yTs z+CarTt+cFb03blj_lDnGDo~jPvS6y5qjtw>DlV22WMtsInVWd$8Rct){=a%77fc~R z(dv~|XD<`xKgr7DDx`2fDF?cyR8a+*nl8 z=6NZszYT0sr8eJ@Dx6fjk~ne3i@fHksq5}EW-RkhkS&qxr%9N0W;c88(}+~2rH1`p zrN##E+Pi!+<^oPSZ9`g6imfE2rRY6pSpKu;#03so(W<*7R4&#O`dHU6F5)){c=D^3aq7rF2l=%3W47RPfH2%lGLf*PliUteNCE(15E)i$(q_y;feL`( z@gx*ncoCnk6W@!3Gk-?nAWMg9T)#b)aYW?tFnuMH`^Z9kX|(=clshv)^p8q`b~}Ds z<$LAQe_7#^Ejkh0Y(g@{FDzmL-V_#&XKbIEMV94Ga1%vb)_3|-JPXJP=4>V1J)`)4+9X*YeIuX1ye{hADju2 zw?PUBBSJh02t@}IR$Yf@%fEMIlP+S9mPqeuTjhaA$dXmiWDAs?V&@m&V@IF?uB=_cr`bfD)A&v*8;M8`ah z+qB&$cR2-^g5g)*>qPgZ2uqsB zZui?_fnLU7AjZ+}%My8m2>kFPi6fz#612*TBC=nrrFU>H74zaHmAO+A{Le zoxdVCWgJF*kE;QO(KnG;pxaVhw_O|stp0EX^)B_Wn{)%Tw>}4^tVnho6Ux|<3Nza> zM=jH3yz}6}?}&Z5&2hAA4Bx|hL!0M^Ywy>W4rGw})v30W@>RJ=UK!VMrIc%w7(9(; z_f!dWF4MRZK3AE`q4_yW6r&I6l%Oy__;xY`-SRes;yM+1?Uqm3EK0wbLI`J+hbnVA zfow6hJfZO=4Px&m9dV}|st2Jl0>pA!vr9`WS^c&N(@6)X26idf`=Wrsr4n!!QhS|$ zZ80u^l40FJ%49q#C}Q+lybza=cd9vSl-lLQU$#F_U=lDK5V(L%AUg~j59OmRJpWE0 zZ&1c*S{)W-zb*%1j!Fm{0kY1|_0qhC3Ld6dbo)pt%2-;z+s(t^a?;E|d;16xaeJ(E zu~J`TMczH`T6I%DU?HyTg}aZKNTmFgKZGwp`3tYqLJEl8oRQx`GJQq-2Rc%f1mz}v1X+W*N{?N?{)Of zqlquI-||ST-1CGVv&HK=Ar{}~A9PcrO<%yfCZT4Jr6ARm-(h)8aU@2xPV&owNmyTG z)dT<-5XmobW)kG}lY)}a^DHxBKezTo)p0uliFQ6SZ{#(2xdm&ZXjn`)bpFMMH`1I! zcCoWAXyZA!;D6I?AZtBTb2s;1E>hp7Ru$-*%2D+IzIVF@K`CH z0{N5|GdboG`ZT02HR{{TFWZ_=Ni-Ll)p*iNsiBVRJ;9KAra1|Yj&a@28IvDlB4AaY&_mTzqnTvF+^&|Qm^GM7}tId@!; zerH`WSX`d_=}X!U%L};}ItUB)Lj4z1ubXYg)g&1^Q@6p0MTDY3FzUVRT9ZsBevt_} z9hp3trmtl{O{bht8DCs+uON-GVXgVV?Fq`KRe1RZjV{|yf=qZQqr$Vva-!x>Bt0P0 zkDVS9FK};t^g}YQ0UYHMtOXUa6~zI`8#JY-Cm~n5o71(m4JtA#x7yJ)?$A@8w^QCm z0ok=9XSvkkbrQE7UE1MErFo9)s*0BQ_(>r-CbaDsU1V_G_qBFxPbw12L<2|Ry`xzg zag9+Z@3VLhpR2LBE=P0J;gaMHz&dpALpG8ij$>BiY)AqvK)Bf*`++#ZF0J$p;`9rr5@(htN+lxmvKm1J*{%ytz z!`o$17LU8CG(@KiyKD^-L}z{?wnw#an@x`NI`i&x+1@O}+$&ZtztTmm3!xZ#T0^Uy zwPfCR`LE5m%r#4QgG+OwSxSnPE#X3E5>#gwc6j}MXpqlF+@kAf0wNtbbTKj|z5dxN z#sb?WcA2O0+Rp--;`bQyl9dt^X36K%mR48eK$%+M>d&i`l6H5JbXPO^X*=I!Y;e!2 zy{?N6BN@ZKXoCa4^TTzQ3&ZVq<{>2g2S*0%yu}AzzbHk!TWnIKITQ`|nn`pyY_R1kdt-Qjzw$#ktb?WJA9U zKD+23{@QOwNZyJmy$kD#byb$d_SKEES4ans6D1C6J7@}W)DB|1FV%2EDobsKG2E1t zO^_T?0ne4bd{x!8-H(ysu0g=|4F@s@oO{dBn^?ol}(j=^sIK|JFA^I#-wls{2oeJ9Tu*&pl~F zFT?@UwZa4r*qsb0SRQ9Cvgqr=D(6_<#n%AwUV5f0w2h~^-tTE0wy>&iOI9wHaoH=+ zd`_*_+ld!XpNRZ^xheGLHMW{Mvv(hGyANae@94$+X^)tBsB^NCl$6GhzLDqQ3E&sG z%4Ae^MJkPX3hXPH*Uif?3(|(x+)TVXM1UmoR^ku_;uKp)Wkx%^)Rb_+Iknv;;1Al2 zx{Wg?&g~3hrF>VpL^HA!BIL+7tG)X8(!BSrE$neGoff%XgB3LjKfb0KNz`8E8a2 z6rc$c1U5?aCXsDp_wdhU472jF>|bX%ynu zwI}g9Sq7}GaHv4TJR}=uPX7Bzfzx5HgNVMunaiBVNW(mQh=Otk^ojs9;ny*W!|iwE zA&l^sjV|EeY!iD)`H`X>=JU|hcsYY5lv`VeQ4Yu~nnkvKXXrVw1Qo58RfdR}{2p3) z2JAx=XM%ACha0Yx3Tf-J1ui9ULPIjeu{?9)-3-|k*oRn?m0eZV@r zn@boYgVTrB=VwidqoPWa{$>5BjaM`06^^hUUGGSvsF{buThc;KJBHZru^s^llH(bA z+Ef$;&8}4iMj{G^Hq?tkc_Rl3i^N|i2y?t{Y(Vc@tLpPHIy@j+X8N+>qo^(9og%Ee zhJ8xM1~+qtuTJGXOo&ihepf&-32`z&CiIvYMx7yFT#_mGcngD z<`wsWwOo!u2l8P=vP|LfwdH9(Fxfko;o0?t4|GHd-&ed@D)!2Gkya}PnLhlO@{L01 zW}lGs=!`SnUMW26MUD_1gW_&5TQ0e6?!7Nf4*IsmnZhy!_(xKYA#7dI-j6o}hB*VZ z=<}Wpx%XMqPsnT_-s+pIVZRsLq7@XJpm3dMQ}6Ql^V_0XKoCwGWQQnE42`r{AFmc7 zXovkwX6JwE$S~{>vAq*D^TDGWSUE+_ScyVsYQb}`sfR)ACg)06MHYMK-s^F8V4hqn^heORsrkRmhAItD#3Gv8nZ*{#`2%rSw_<4 z&hkjn`PD|!rMoM)3*@U=g|xIAt|TxWM-Gyn2LQK5VYSR4W4z`eT@UJt`c0B}|3*TV zwHyq5zMi!D<?Y)d9GuUe?;bN>&{~G-;CvcPsMwm<5pcARrL*`qwTp+S$ zJ*wx2&Yg6FW(rxJZ+HU1NgRKH&}LN92VsIp1EJg_URAE!`;#4u@%uovqCy{36BH>n zy>2A7o*vgR)Ha-G=``4u9VgbEWcQoy5z01=LyH;1 zp=E4_5TVhf48?&ik|WQ`;@phhDZR#^6>a+GA zE|OvsYtw>rvxy#)4GL*o`?8LRr__}m)?1%tBOzpI++v~gP+v7=bDHrqhG3@0vnSJA zR^v%a_v2oialT-cg>F_6T=jG?pM40yi5N)N3OOAS6k0~lSCh4jBP#}T(F)KC1+V6C zz=>!J(Q%xxrV&q65L$b!v!EgYuJlLAH*X+;Z-4ujmtUKT@>r!8T{^C}o%2q|X`rJj zoJ+)eYd_C2esdIT+|;QVV?RrH3fv$HyKK0np7%|>H0cfb1Hg?7U6taO6zLLPgpLae zXnNj9WzB^OJ0K+O_xd(#2>4@k0xM0^rxgkvC-HS`wFo{2-H3Mm1!w>74jF+R8qBEf zTHwYHR~Z9zY~eFEAHG+(rwVtHyxn|57{Rsxp0@8!+_HWD(C8tg}B6Ut|6&jky*V=m}*puS2d3M2b@BniabOhsl z$8t5Vy;igysT8-d%6F2h=e;1J_t>#W(B2=O4grt?pK60v>_=n&rb8#g*a+o}7ZN#m}&wlgyS$N}j7qCy;U83DP+ zDHhcasmk5N)FF{x)3q>uW@qZ8bPs5>4R`VN|iS^b08 z$8=?N->(p_4?Y8OckTIJ;ekh0n2JkcJ!6~)Il6`1A|I_3!f!BW_I?k0$y*Q`r@|3L zb~`#fDA5hE<|Y&y-5cF7#7F8x=V=&Wx6JT7)@lPns#_I%Quqbtf}`J*<^FqC>!AJI zWidhZ!3PM6o?>2Iw_rKbMMhQCuiC`u38tW0%;6^;S7kTG7Vsr^Tx#Qa8%yScuuUtZ ze?>P(#7=bdY0d9$+NFQ8sYYK5wqW(xbvwR!tK@x%=dv4tjrtQ;2mM5DX>~77QwoT# zsg?;oj6Dq}{hCdx=@V2P=QzdekJE80#5^nl`8L@4tHMAWb0pb^R^_}n0$QAkJcYqB`{>|DE`cqCXRn&K?V#J=(7!>h{)XVHgrua%60#Jnusy)NsZbnJ*QA zpWD+<6^?u3=zSuU$2S%RI+cfFMDZno7PCKSUo{Mgn#vuPM%;D&DikBS2pvlp8W`;C z=_Tr5Pn}tpA}Wj^7VYnJ7|k4Fvv;MtZBT}zl3A?6yYd^wqaR~04I+RTLf~ve4W@ja z`yB`EAn}?3lT!E7OuN0M$8FR(cD#tKt!5J>XyWQ!f2ylC0*k02PFk8AU@CuL6Az=~f7D~_`%3uv%CWvI^CWcjM#_s~+BeVp!;&_DUCf5DK|2m`-qIBf)gzMGMj}$^$WDfKS$1(sdzfkF;>0#2zdFU zr{*+-vT?|P-EBmU?be64T2{0Vo#76?6)a;#m%)Q*@MS+ihLf(+kM}rHF#%OOIVFQ7 zrSM@g2YOjmd}Eo8fgcXP1Hy(RkQdfkIAMI?bA?Cy%LS&3F40sU2r32oqxbkOjkN0# z6DNTx8rNABG9h}MI&AShg29TmpDFxjJ}>s4 zh)C`%E1$uH{OHN|Ye%%8rdijvy6wP;4+(+a-g=R5;`+t{LYC;rp6~q9u8uFx4&g`E zDi0nn%ME<8p0jPbFpy>gUsVoOH5J%+W*yCBJ$cPGno?lAWltv0J4VIfC_gG)H3{#fVl5tXZG|2%-&#f^OQt zqx#dzXmU3pO$=(mFya1a>Si*Yl*C4>Qq0GW+GLwf9Z@u?3x!P!Rf!7L;TZPU!&OZy zFSXi~FFWMh)u&|c*F+I-fhpO4Np>Xna{T!1qIrMTm70H zL$L2~aB+ydyb$#5VbHwcDnT0sROCNrG6}m z<^zuf6#bUUG-P|1V(GKgTHG7Ia5)7PM*8?)JG+$4#;VF)UkcX(wSsfd-`{&v5Bnw~ z8&LG&Oc~5;l-mfxRR-E&bwG?}JdmHIgqNn@y8hH=DmMP^8nzW0dl;t9-7mX*_*PL6 z_E0A}RfVODLj3>(xsnCTp}@+GvQg|{!v^sz@_&t^33|sH-X>)dwV!nGX++)cf;J;^ z2xPzcNV&4ePgPxT`5^(j*qXh0xdBORA&;nv9@&kkCj`mGEc7c5vg3jJO|jY|fO++D zw!n&3Cqi#z5-ObKPGYR5gaZI4guvKRmw7y9MkYNc2sop-@$e$IS0QlI(7ZTd+V>aD z<=qrJaTF?p%gc<-lJQT4``~YAN9etYVqHhR;C$e`xyL>Ul}runS=vFUOlcxa!^sUt zdlQ>8BSWeF$5>D}&Ltih3dDrYF&OFoxXqV#rXH)7h}dIx_wY)V_}iQQ%L@LNHL`H~ML~tUa-+WS z7WI{*Ob#nqK6+=jGI46&JVyNbK~QOYoT1Wp6TFYPPT7}7_OUHRIh&W);Tgpi?poZu zdXUYn8=%f34el}mEMg-5BN22VLcK|w-c6FeT&c7 z*G%Ex!_|qRd$j39*uEWpeU#?9;?URON0h7S%$k-u>kF^-!5A&y{8RyN0F(5^?f^O0O}QCoNW%3pah?TRF*MF7a`@Zu=kOQGJ>UQlR6b+j`yqTgQvjVe}6juT(R+leJfxcEO3G8GA~MJ`kZ5 zr-Tk5Nk^u&>Bv{k5~XDyU18gs#EQhsUO=coUL#u{$70W+ZII3E?k{haC{I<@J ziS!l7Ud8wQ%~-kf-ZY>WFYnY~o~aYc_mk@97d;5T;km?cMzcg=aslX;pyg`DLdTi9 z{s&qWc+q0cW|(n6U&H%I|Ay^KQ>@WPRgdm zF~rn(wuL)4UF+MvAx&pT8+XWfFNY-kcl=0s?q7+Y777g$w|qR(fTXE^~l{`BDSkBbMiKrFkCtdY)>CEPch`zn|Bys3hL zc;m-ACn^VPk;m=_w1jfyV4&~eN-iEQ;Bgj>mb$x0;6$Drd?sN0tEiF}}5BhU^2<49E(tI$?wQ9zG zZM)NoL7=W|K0i+*?tNs+g)Fy-Y)Aiy4wHQ$MLP06hf37>iTarr=S6f3KFd6)pJNBX z=Z=CSO0_bv{qIM+KsjVW$jt;4*Be1`z2Tt~5fyb*t>cWMlG%LLVv(3nipN)9-u6ZOa3Ev@tWv(6yCnYsMC9C$D2yd3CY zzPB{3u|C)8TX+1}U6U5MQOSOY=(gF9$5-F1e{Vj?@`z-gVfhG7t4v9YFwp|Gwilcdhj3M@o-L5T7fl|`UlpSyuXA)= zHgw@}mCy*0yF;p<=B+yR-qyl4+_2_k0a-bEaZ#7#YO40W%Z39pnfg~8 z;vJhIaJZKvYW-(4`fC`$sHXlK3`sRtsl_~QA9pJOyB(nBzInbZ}!ZJ0-g+?Q174 ztP@x!@(NiStXr>h+Wj_ka4-}OUflt)!o>E}Vk)MiNS@EOjk?RAde#^rW#!)rmf%9> zZef-kwrFyuGh}-))_s?h9a+k|APazy_B>I*MSWz9JBzfI&x*_5c%SCpo*OV?16fnu zI!&dA$<9DQ_lhJMl2%ny7&cZbUh|GoKWBQBEHK^p*Yg3OD;SYXx%7RO$E#lFdb5L@ zc| z!COMQmNUB}iW-Q(x^uthJ)S-6X~`GS##6r`&`Wj!`S8 zXtJ-{*&wol18#J-%Nw!B?$fpO7pXV<7VNK=3PYTvLHP48%I#QkQnu^17Q6<$^GSwlJbqu$FaKq$ z){8OS<0XUh&e31#2T1BJ(xkJV^5hC;=H>VVmUZ^u+4owA*Q1ASC*!iK(C$Jk>3S~k zHf2TGP5#*SlRRYc4O}gea1kRio`zT3=8n9#dTZ+cFc`KwFc>D%V95HJ2+2po^F_=; z6EkYM$~QTx)cV-z>8Hr|jeaR(f=`IJX$h|*E+|AB8d2aF<9V||f!vu=^9|AR{lCb^&CO6|4dX8#zrV8vuu=Mn$%|1?4e zm|z{fR<#GT#I5_P2*7eWtI|L_~*@uVY5pQ*#pthp>-WMnSht-7HLUQYcx^XspWMW4P5f-A>VAD6&?utj;inR}ec zzJ-02LC=RzTvSlV>DEYt)r804(ZTm6Bv(aD&+S)>YHz2|g*}PQr2o$YbZd;7FbEec zNt1S03Av*GB*B6*t&>O3~tj09)O! zkC#(b@}J5o^UyQ%$>pIn&)!ILiv{ZTlLDlxil}Z>CqVlDAA4^Z7v;D04=W)xgdoCz z#1K*nGJv2oLrJGJ2nZLG8h|1SIfZ`^fe#Xy!BsB8oZ#D3Q#Wy~>Rr~(%rAZpW6B}DjE>Yv@4i@$z! zov)#A#>dYp<9thg+gf3u1wSaCvI2mo!i5X|tNJ{*jlvBCu3@-=mIGovL7!z@ki#g^ z-wxF7#BVivZd3@UU}Ys?uYr0lDawyA#r|S7KySi7D=b0|&>>>DSgZbGAG$yQH1^nQ zhK3Q(0RAppW%Ek6gZE3nEwRYOt3(dO6u;@dFyX({m2|iP3yX!}0JuNdcSsk4M6ULi zPs<0=dwT`{=_WjPI9U0U0e#`4BThf}m*O&S9GPa?(YysKmHkkM(dNAfC>0V}60B8H zH~LZikb{>0mM&N+@7)Gy-IQYk@G)lo>UaW@?EjUFtsC|fqX%K&`HSiQiM?C%+bJr* z|HHJeYS(_tr;pzIV-KRs6#wcK1fdv=SMjmV+<~jGIK4RF9ZO(rw>Z1NAHmL+{@rQ= z(ZK9N4m+!VPr~jW(~{r@i248CPppqX4+8w{50fhJYV*PG9Nxw2e(Q@t3Du$DLOC7g zzl{5TPf;$58~{oRmkIaJ0sdd2=D%t1`q=-7D|6+a@9OeGQH?c>CwR$gN|JTI-=a90)pI!gc=l{nOm3MsE#+S{%ot4z!BMp`Dc%RBSfslzUT12Y65HmeKh@_ z^ZS0W?&W^~Sq4^_&|8x;GrTl;E48TbQ zh@D1@=@#I?Q}wJ+_*0g%@Zd|^3my99Ll)KNbXkmy^D zcV(G5134sKy1iJ`{A}REpE99I@o(aipH46+?c3df$-xuTt8vkK$+|<05W@iGC+G&z zzbi4XDJ>&YgesDZ_%~6Sm;!(YLrx;S(ZYwVDa$xhop$emPsUJtZfo>+BSAU%5 z>#e^Z1x#oEhJ(Lr%bgD(UmfYRyW}~g)#&&OsC<>*hUd%Quj&N6YUk^vMz1$;8|U4P zAHfGwaSc2FF}6Jt17+Gza2-k5OubEWwGmDiIiFJJBqLNNLbmO93edI!glQfPl6GI` z`j(ymeI!wG{Qr{X>1nyT?rd90t+ieF*eEAiP{jFf2U``Oj0fD+ze?g5P^L@QZ6iG; zb8Zg-QUOEiIuhWSK^Ung$^_6t|)%Zh#o+GCes<74*#n3J!}#D z*&=%3doKD@>sM0gFRh!T&yn!`-%$gFr4~VMkG(O^CHtShblcd5G7db?j(+?FA;dzb z&YV6-NGMq!?a>LqYJS)t>M(lGD+7Jv{Ku!fDe(M_uNfxxvnV~20U=9*i=le1-sHMygC(3;~Iiw0}tR60C{CaKTv~LDNhX0IcYCT5CQ;+#;)7I z)B$L&d_UZ|j_C<@n%RtJJMZ>tEJF|AR)WC4mB|CM4k=UoL-F|z`X3*F60g5r=}w&r z4D&bJ`>j4;V*yAL;5RWa@EF2B{CBLFe}3E=1pPnoQ-wI}V4#2*0Q?3(jsYlXR$9v? z0x-rcCX70j2ughwfP;rHdv7lq_#0IA1A(GVKKA(bo8Mn~MXe-`xOEE;qROcuH04vr z4bM-qJqA#sZ)IUZ<9Y9MnkP*#$YM|jbT@diOc1E3ldek$JKNddf=$SG*a|5K+rAVO z>tfJzQpqF2C^++(VcUXY)7du~*%O2Ox4hY!86uotB<4;^9LdHKq?1jfZo~5rHCP*K zg@cS(=xwQ9Af6J{)}9rbkUg;izaPfx`5k(F?q~hB<6!y)EOSv7k%fh?R-e z53zk;+}72v@~uF{fiB2d0re!MErW;9#Y`d3559#JR53j7IXx!M-qLZ+QVOp=>nPOa zjft>rxPR5~qI_;n89-iC%TcBf{`w*)mRc@W-=Yk4L-Y9)pFV<0Iq1AE2QgXE`{;-A zlKM{2=%bVL@B4287cU4IVMW)YQ0<_tp++}*7t;^E=rnbX!Q3Pt9l9rH(gUaO3&N|S z*X2#l-3&AwXfm>ibsRJIUpOK^q0y$-3A}m;DwQCCT=8lPD1qnpzT?As!YM)XZTiRe%lMEg=q0IF*-R3`}C=Q;V{Mfx_Q!!EiROLQ)7 zO+pW6R$Xe`3|K`A(+>ew@1pB6Y9Jnvu!9hKv$~I*Cum35y9+39lQ<{bc9PBa<8JCc zuK{sAoY#4H_O(5Q3{nDJyP7s5Jc^};h`oj@#}lB&7c&z0Jxt9kLb>Ynj?0`6wcUoL zvW5Mde|D=Y>ztOvqVS2%* zEJ-geD*1Zr6IW<&CrhQR=yHw+Lpj-G-k!`rgk_|NnxsG7Zm1)q;gj>YXe5Xb{lP8Z zo~GYXS2LVgjk$p;o6XyfTsE zhcw(Zk&)Esg|p1Y9el}Q1K5o9IWe7&s{nMZK$LH)cDweA@u=VPa%em&#m6&_PB(XE zm~+50`svg_Mwmq4nXA*Iy|8ZB+{p9mbvW`j3$66oHe8{f(h%RIX@FJ7Dw!aF~W)^j=Hz+ zltODXlN-fo;x*YYj!6k;pdG^!$@^mOQ!Mz)O!5WZ7VGupDZ4Yo7h z@a~Yl(9KzMn`4AI;iXO$EOM@?LukK@aB?-)l~Z0dYU%%A+~>NxpPGN4(8Q(8ZKV0u zL(zgT;T?GvT@H1tM4g6FN=L1`G{f^;z1*tqJE{+7fRn#MXSD&;IL34RTgT#A@TlL^v}R8W|06(td(V|9dFb{%Mz4rIL*pAgwq z$Lx6@hcv!?^{j@GR;^jG{9cp;C+~`(zs-`=)v=T)#oIu~Bgl;^IqUm)$N}ufJ9n_~ z#Y&!T50)BWJ9;$ZG{0*bQQ#MnERoblkt}H_6^z0i103{E(9d(Y@-6ujEb}6D(92M5 z#AW{4%8qte_lMbgnV>_-`jgoU^AWeMvc&hsswbLibjk~CLEH_~#ciY}M+)M#O_YdU z5?^j~{a%q8LP%G!#%g{yZssG1XvSQu%|j~E zImMg_3T646Eiqp?ly8On)L!^wKD_-#cLZi*Jyaa{^rZjkq_-08sK`a<6FCIu(UqO= zd&-OOg`2SPfUiv;*sQ$RbWb3ih=jM78F3GZx0!-`s$^8e2#@~LXH0Jz4m=u2MD+t& zETUr0*vUzuaEr?KbhLQzm$^iV?hdK~^@-EP=1NsMpSv8@iQmfzv} z(@vS+{*dV@63qCaP<%N764}u+V-8S+iYv!^uWUN3jub>IBi=@59?!LrglU*pe{kN} zlu!%uk7CLQ?yTz8c-C{EhH$Z6c->FE^kAV#e0pG{z77TDb&I4n684siXyt$vX3N#e zTG?tRH8^E>JZi45J!lluT3r?SHfy@opEYZ+wetO`=?DSin#R)LKmz1xSTsknpB5sl zJ#Pz^2+ayoD3zcElU@)_$eAwFDk0`Azi&tsOQ2Q_wwmr>PIZV52)Lu_!OQPYi#M3vY zV1Cc!DB2ViSc`c&^~K(<)L5~U;-Y_5nM<{awV7eBd%$X&k@fyhrMDVeA798x+BNP0 zCxy&bxq!Mf8p3eMuFj#hJO2Kze&RHzbd7cz0ZRl#)QDk3#0QtR0^h% z%J<&9n(PjGxqYL0=B&t*{LH>o$lR{LL)4aT?se<2me-6v_#@ZDg-uko1jTz<+p^`7 z#Z^cjsW%Q28Bi4eq9DJY32ffs^?6gol`z&u2KS8aXz7uq3 zfxqJwm^3c;h1}_&XI?4`rILsbc-7hGbrZWJhXUo{ZqXuXt#c;Gfv?IP$}3|;BFOmL zI^U+zaceC=qXVR(@Et@*4#ovmR4h9`Fdg!}a0xGjO!7=$4Jvb{X^u|P*;@YSC0QbP z9}D647kA*sLYI|CFd_<6o|+}OefaX#ddcOPj{I2H*xW>KP!oAlS071~+?#`K;kIEG zGBStrAN?fC5ZK1De!?Pp&F$C24sTQyD0EcoA&BizeEJKG@E8dc4ma?{&0vWWQ60ju zi4^g$8pYb;cxBfz$lxxd(`L0YS+n@_+^N@7oQGOTM#G#8KmJU zw0<{J5linaq{2dkU{NbkP=jM4J+q-Z}5 z@@x~qR8Jgh;D#im<*zsk{6!HT1C^xsnivG&{%*@8ItZ(KK|5NM;+F}>rpG@x=gswy z+>kyk=$l7~=(8zj>eIU20vA%t&n$+c^W@hCMHoI0?%W;D8v@nQ&9-45Xk7&^H4wg{ z20u6N(MXhE@WT}%^qjXFFXLEUj)V9F;>JG3NlsdDg=FxEEf2fA{s`K$Ub5~MYstqr zmAD4EK27ykMo7k1c)z~mS>IhQzO4js!xkMZa#v1sYGJu+#;+pzhW5sBUj}>Pq#Ly$ zdPlnMYB@@-PM{~i)e`eHv&tZJMi#cRX))No<(|jahK#{X-bybe7TA%X$m^biT09^6 z24dpK)?>lo;OIiBZ$ElRb_7?893S;A#Td9kW{`@@cD;Pch^@2wqXh@RX9_E>vJrQ*|80?UQ?fpTk}gQC!ZODT7~fbu8T( z9;T~=_o#nkeCTgVROoqGbu&I>kO0Sk{gmQe^uW2yDquFn?T@BhZ##8<=nPuhToktx zSH8&bZ@<$RZG_lrj2(P<N$7{hKZb9GT$y?1d6%X^48pNW0B2X~fjqlDKpzwgh-jtiaUsa8vv(7J7eln_Yw zMsJQY?Z{ylqK&OJg}m{W9z4Y}V_0u#3+o@*Gej`sYdlT~FhWS878dty9W}@r6rBV= zEqqg0xw48Q0P|xe%Znjmhdwuu_}Z*T!G;1TC@WvZf=<(stWVQsNtIGwRSrj5ef5b4 zw_&tg;WS`YdUF~Gje6URylWI<`x3gppAkMrc@1UaiFhKd7-%T8a;>}EtTz-dxA{K# zyGk2o@oW?>onuOjLM-HLyei&G`E)cr%$HFBbvx;DOK;p->{Il4@ zkZq`|#fkh_+0wqVCDGV{KjBl{7EtoxvWuwV35r7b;iz>_Q%<^xKp?Psh&y_f7cA*_ z@mMa`?36vnNLoMz=JJjHBBT0`#`#ikSW{oka5{S)Jzb(^hBab$_Rk#ecsM9}LE&A2`}=v@@#8D+?QVDVM)w5g#350rYek(b;$*xO zRGSveQNfU?1@wlH?z@Pwbft(v%$|s(&|M_tG4oTeY^Wx}bvWo;D|FM~W0L3O$9BnD zyl3v`B3n0sV^JOPHjFUwgtS0NYT_AQU29S=2s|rHvFa0?G0fqHs~K*7R|9f{JFFNZ(u_zJ zYJX-uzp|0rOHwp)_b2?2OHYtOIiNS%X8F~7S-MRL3YqGlaC;#rob+hixz#C&Um20O zB^P47+#>#_E(JAwEB%$5yg}AMD}u7HNDibmMLX!Cu8Ty;slNY??!Gh%*)E(ftUBTUB(M*z>%*WO$ka zDnUDPg`ze=SIDoSHO&Cu?x*a2d&H7YT^0c3B20i{7}DLegw&Y1iN zMr>NE9M64k`}q4_FK2Ty;w@HDw3F1^fmmVO>QT6bHAgDn8$*hH49>Zet-cXABYm;} zvz~7fUA-hX3fWu%N{E_=kuGj6rV^|YK@Zkf*=uO?Qq9HiJfg5+QT6+bblvfQP)k!}|p9$Mo5PiN z8CUB{CY`MF2-bMp_%lGsZ5_$pI3BwiN>Uf^*S8zauQ)yYsOJ?nZg}y)(9Jm??Zx^k zC>&pee06?o2yP|aM`D0Kii)Z9Nuzuq+?gc((|MVe+11sFhXi%Ivv!dMHr5*Pir=7z z5jKx+x_qWOk#z3!;7%4-j_L~=oAZF(0?M5?q5>AO<`SP-&xG{6$8jmyV4Vbu)D7-X zILqGdp$>+38oU8?Nbt2szd;uaWhGm@drHW!ji`IP^j**swM2+%77{k#Bgg+ZncFln zwQcAu2kPD$&GX<7Ri?~aCUc?f@|tm*40-@a!1GsMdBUREt0F*$0W(eJAcQAw zeJ#1|S4zI@%gpfvfQ`hKBjX_ByeXM)-|BndFNAFhD0HsA^$c6oW?C@ALe#Z&nl{xd ztQ}Qh%fSTmet5d5`#j%P@lJoVY}LAta8HoX5;+7WEv)(WEB6bMg)b9>uA$AsYyl$b zv_zgevYjmN2Se0{?9^SNAGx*%-S_a^b9>RJlE~SN*UpbwhM&H=&-RIvE6U~x^J}?x z%nc=VPPlZuTQAw3dk8`$r+col)`wGd6czN#&B{Oy%_*qv4*gGyKi@eiNq=VCqeyz6 zGXCjncXn;37ULtBxyffx?`H7LsrE;+YSt&EFEjQH3WD&4g~MBZO0PAZ@bJqGgOq6s zJ~-6e;IfE3c}>(a+om{Zt?`T3Q^i><9VDqP4iA^F_cdj?cKh*$Hb>B~J}X;+!X^_+xGX5KJo>or}QN8d|UU9zZzc$63RbxOx0i&foB zGC`yK?V>+76l#s95gZ62+G8K}Er^NI;^WkMjjLC+2Q#%|OTMAcjLvi$NA65pjlCIl zFkpji6sq8Xb$7@qmWp+eE2af#@7L0w>cZVwh{s)4=ezE>;GOo82-m&eEnR13g?$Mc z*lOxzpJ1eu5&E9PVL#6(;gv%~JrxG2;7y~d?w@}P<&BX8^C3DJVa|+CO)~uVq8B)Y z(6CzLcB)gnW+|Eb}nlATT0U%FihiiqZ-}_A9fSw5$H<=a4 zrs_FwZ02%4v!VyD`VL-J2o4i-HXwyb_1AWvXKZ$1lNrd)zY9puN%Cm)6kyBc26BCp z{%7ib&+ov@w&L}0@?($@jC)rd#UfqNW}7iNYp{L;)7k3^IfORDfR*3qwkNX(34z;|JsTKb=Wep=Y_j$R@R+9lu;`tr1xnbo$Obn{DXwwop#t z3%8-i7{ZssYDeOlsGi1=29y99zQt`Y7;Sudm%y20#*;f$m=Q}By`U$|Vwf{< zRn6QS?&Zc`;ucL?3L-yCj*WNrm)ncn`kqZuB{6|BDkH!CRK!H>+xMv{djUopSTBij z3jWM6F?%z_$;UPd$XsL~u06lh0X^Batq7y2l!g(!hMwnAPc#rhTksy%uoJ>~@T;)2 zt^WN;wFkR_pO0IRhya8i!P71mdzVHD>0#B*5L&vel2?X%mA;BV zWKxmq#;{hjZeF#g^&XB-hnquYn<^;t`N-Z{1_^Y+C(@AUfG>}+Q41lNH^O+}e2l5( zU3Y)rd!N+ppwpBuysVujmT9B(T@bUL0R>y*mz~3RGA11_Uxm&Ftd)C+3AO61!IwOb z&GHjDCG8@o9k$;%`S7MN!`i!vYU#YXf29HO@-RVY~H*%kiGhtq4;U@Du( zPs|X4Km8BVP01_f!XuK9R1G973TFbJ^YKAF(RcBMN}$_?v|Uf7`&DXCH(9#yy5xa; zZu{p7iAd(j^e#pbD{Z<3YfuB~E40IiC_yw*&vX(LztN`2vZz}xl1<%JJCq-Y^S+mT zLIBG9$Tx;4EHb4-#FV}9yr_ISTVkVMh0tJ&0i|UXL&AP98p7BmdfY^X?`wzi4Q|n% zzLD0qPhI~a%H&wu@ea2Z;;qMx_vW3>Vb~^FLc$W|@VEY3IUtA6>?}k>tCq{W0~kMr zH%ll#a^xf`ti`j$Ma-~IutFuzjNO$D_tuT(3&`$x&q#69+1>M7focF0W_B5KKfbU# zx1JTVqj&j`&!bDG(uj-bA=Ilm=zL)>fg#JJ#R+G>q1EyAcBvd2)&T6J0=hhjZ+uJS=w^trJ?FDt#01UK!lu z+tY+(jD6K6Pf1)NKQ;@=q}%YB=|r*H-+ur5sVcReGWz7~@Ucp{Hg>qLuDgYD7IjfH zgbXo#Uut?D^8i#B#rEsvSj81J_E$SOUO`QM+7q!7|JD@o`v9) zj52DZF&}DvM)eS8NKTp#kP(<{eY5Fs7V(-ESbfae7_;dzwOlT;3Um2dX(RR+D`Cw0 zNxhv&!N}sB049(mL9w+(OyoyXtuPvPYth?zM zZ__?pzA0M%iE%Z58Ao=<_wfbey3fs4!fn%$3z03M6wa?Stedhy8&sc?2b!5qX681p zwwP&jg5^9)GU`mmOt|3)e>tCo>o@F@rEx|a zl5fo{Aj~Z!?A|)2y(9~d22*eiu64=2o}C*v(_no0 zP@dd$RO+Qv}LmPikoJ<)0L#s-c)CXhZv2Lb(Tf?`2ZC=`q*oGcJ0Mu&jUQRJAr zlH4>HH#B6A>V&7ZwNil{NeI7cXp};@^N|-Nmt8+V4=fLs zs(VXiyy)19*96QA_i(~5!yX_sVBi`T#Q~GLBuRZX6(&-2StBfXd}WyJ?4vZrt6pBe z?tK|wPR{BB)^)MH=$;+6NuK*?oUJkUBrp3QtS<)WrZFLLURo`ciE&C!eFP)uEaniC z?;g;Ed>$u_Fx9d3;XBr2qxwX8ZhHl-uJl3nEHE73Tzv zGExLoZq?UlPZe$v$o26%%IjE)mWqxp?}@f*D_!dXMGX6ygd5-lI9~c+JG; z3K3LKR_uF6ET(#T=C)6(hEQls9eBq0uD(lrykIXR^tfw;xYL`RbIj%&!>iuK%{Or; zLV0y|&Gfuz(p_3Lwc(jzqh?y*I<3qp%b90*x2r$eI9uOE<(^gOig=T2p;!A>xvw^t zefPE4FeJO69)JsSpo2(7-S#eeD(N0Y=s~OPnhA;1{`x%^{wd_kAg2_pVXpYn=bm}G zT57(;b$jc$cjTc^o`zD3x(d=mecqbicw}L*ZdPW}+5lyukOWch-MJ6?m2aT|*fFz8GkF z(!GeHx5nk>KImQJ!*h!d)Kj)~B&q}EgIgJ20)DEmY*-${W#@?#K%rG(ZIQa&16xU7 zOx=Zkit)P~ax7>fKRzpXzV&#z$~XBs%}Ql!oY}WFq9(7S1p#TkZk{9e?NNAfMwbJD zhs<|t5h{z;558qUl&MSSDn*zF6W7$zX?%}(ouuDGxyAWm?4n@?%G+(As@_QvU8hQ8y{V^^ZUd(G>gboKyX6(P-w|9azXTn8vud}T;?YA$61&>DBt1)J zyaCc>OQ>)|@{ipKY@!N1?q1$^9?-*1rI?uDyAA4>UV8>vrzfRIJpXnm{3XthRY|+j zIkWLsV0NN2eCuVF{@d<6=D7>-**nm57ZII}DZ3!xQ34zMfuAttiBS-} zS-@e|!-R+KiPp9e&EIKycuu`z$0Pql1RTpaoE{EMuOP!_$y9I7P!^$wxdcvlEd4l< z;PPc7>LHFZP=!xSr#BOlq1uPOP3eOjcedx<8-Yv8A(@6Ey8oh`s3(s=dBJSpjXZtY z=NsAlD_n!5S27x=05DCx>`vBwJfs8=5wq?hF9nUpeK0yP$MNhe>B_p`00QoqYih3J z>!jXN5tf}<)TX`g%15xS9!pbhg|oL1lA<(4*oHjYQC>0X=EGdR@hysuV|Ev5@>zYb zNr1qp=RSR~si=;EOw|($E;XPG$*)oVg$vVmeo=z6n12s7h%q5KhFpZO6pTSZ0@~dyx&Cu`&?>*1S zy?M{qnft<%))U{tQc=t}n)Qka$VWl+?DU+DG-cj*13}Lm`uLt@qE z+%H-0Z+ePD;o8!!4XPJDjpZx36wx$Jh`$|mv$9U5 z*c@>V$imy_Izh3>61*7_UzNyUUPMr+;UNl;OKFIJ`A7oWes+{beiOCDgE^OT`8U6w zh?+({;L{6|-G-at=lKG34t>%3O3?PjvxS85(D>4yIm5*d<@dcC_Sh)xqD43jjg9qBdlDh|EZ>2N z1Zd_REm=M!19OJN*LFR9Y$E#)6xq+xQta%2E-u@L_m{@ZVc6>(~t5vF!g%~7AVr^WS5Lj zZ|+Sm)pe6EJeVU6!B?=$gkY}hj}oEHSc%XNzRoRJYd4Ou`^Ng$N?Z;k^EFJp^^uoitm6(3MelWbV41^*iLMU>JNPKfZLl8KRTkhBI8&b!8 zD%rOI*~>H>EMsyw(CF`KZ6QMOt#|dQ@MA;!&YXDCCva|BNnczG73U80{8|t=*9=L8 zZ=FTS1chCJ(kO1;rymWKN=TDzO-Yt&e-~oswb+OXvc&H{v)+5}XLJ&T+7sRsifXx& z2m#i1$ZGcmQQ@)=ZI^HWs<%lY5#k{`lkf|;aj$+ph=V9sKKe?C!Cs}Ab8rm?Qk&%A z?q`>bR_s`qnyjp_&Dj|#VRp$gBOcF*QI1E5D$(?mD_mh&6-X@J@ zCiN$hm6gSP6hm^1C3O24TYupEX__s>~ork#YW$(!;YnqQNw*hmR5pB zD~-u(JU72H-HOk3;_iHUKXhK$KRT*`Vo~5PlZ9_!rk;^=F4X(i@KegsHb{e4ukv4W z-az5s-4fS-a0=nDvU?QJx%*#72AudfiRtQ#z_Divzc7M5qI)QW$QvY63NNsM>K(D& z0u`j15}5Pj3JuS|&RVByJC7kNOnf=93Dy@@;e}nN(vyCan{#(kw&2d<#TkoEGgkz( zBV6);?w0|)N1>+kncne2{@&Y9cinwb@??d~l9LI!hb+~m$bWdmCVBJ*#QTa#b2a5| z_a#9tM0UhK{FMIAiABewA>1=`C>pQ(h5{;klMO;CS8r1Rm9{=CHzr860qO23^VUGy zcF*Id4;ci<_zyC+jFybLj<-JQA)1u3-uI1Z>Xx%RGr}A}PgX}osIfTQv3D6SnwpTp zCmLJD`O2Bg{Uj}Gs3ZDhAIcpM8Kp@?(;H*UmN>5*3xe|{xWz|Xr)2`54)+J`v&VD& z2bVoTshX!nJ^WBd`+?N6of3u2ml62Z8$*%Z@Hp~ZrRE*rQYf9gNlhnP2>EQ_p3L1fw3b6J%PVM5 zibK`Y*I|f#9S%Y<+an+D9$Y^daMP*TI8&eYN;sW=iMb@5?o*9l(4UpVeP zBB6R=d}`JPW8%ikga6Qw3>tDBd(%UL;v<^N)g&mEK;H`8>we$Y`yDp-!YJm={YN=3 z5jJ+!i;onyIEtvai^M$g6CQ4+eUu!Ja8?0?p8xzYEv4o&5Q_+Q^k_^H(>ir88~V~J z7gs-XBpf4s%=^(v_a6Dob47lDDdZ|kWD4C+2}{DEM4 zZLAulS)V_b$U&VwPVuSGMN8PNj{HO?aa`dZVz)MZzn4URi>PsDFeguZ8uLlEjH_AH zG>;*&&J^kq?~=|zUa2O2QC7;?J1#Rzehxmi`iNqIoixLBj_0BPdW!Kw_z0P5E*b}UqajH2nR0t(8*^pV zC871$=1Y4b1&?HH<;8~*=LlVyBkY;3=jAdQY_v%(4O1?R64P*8&J#TB$7DQjf+;B+ zk$#O2ZDjMMrd`7(+}X`!{c+5N7E4V_K1sKJXWNn4q=AP_;i3$DR^L5OqSJkJ5tzpM zU2FTD9!mqc54(n`WzOJ&FyGkOx=X){oh% zEA{reT_V;3qG9BTwhY_McMM~HTFInZll-N6x~SNF66&1arSl@!R|f16U|yMtLE%A{ z%i9nIv^4W1esm0%B5nrND|nda6iwh-aO{{_*uD>E&pk7AiNU*kxO4cT5;Ur1f_bm* zNxzW5Lv=ZI#EI>U_wBlot5+Po?yuorURr)pqlXFVZDkb`fm+=J>`$A;shaN0n&sOu zxK)qW?omf$naSVlAF0}%<6DW57g|NZEq+bi#G4M|)ZP?OW)a#O z5hCewUUL8!gUN&?>+m}^VjIg89h>(9pPOyX*IqdhTm>~DK`1_Zq?-g>cFFxKOCS`h zH7a^^-XzXT&bfaiD~vRSzj2T~;Mf3VC$8Ufa>}_K`DyOZHs`YMXdwm=1U;k?;`)>x zj(eTLC^Miq=G;{(Ezn+=h$L>HPDT>NG?o~o6gDLhcoP`LGuDqhoO(bhz!qn~VxBKB z`}Vuq>T^bzJ@QPV9z_0)-kJ=G<%8Ekg0@jeSw+a)Pchq4IFPpU988K?*ss7gp`ar7 zKbHKi-WzWT|^zin;tgrhY5@tlCSbf)YBu8s*}Zh2x_yH?-CAEM2t_W1Z}l> z{4xM6z7O<@m73>q=CoTZ#1XP6=KH-d*MX39A9F_iv_^~ObRk{8wCO30eT>&M%jbpo;36MM74@J7 z0F?Qe43*@nq+D~?3E?o|;LDi16l@gzPCl98(^RABrLPhnq`BqCQ|UN$9lr z@5cq-5fvPh=Wi85%&RtuNFwG}?JYde81H^tK-422KU0_D_uHe+sr?tR!n^&e_)?E`>0wpRJOWBWiUEM z-Y78G)a330T`MB71C@l?RPNcP;Ooos-BjGiDRB?pT>hYN_o&UCz95%9qxRA4+{L>j z6`L`u;Ep~LQMK9Fo?IWr-H*>B@iaqx;nmCctDfsJkKml5H+g-yF|>Z>>J{kArh;g; zzfBQ?>PI9QYPt-n0fDu8*ySoMG9r-7Z-SVx-_WwBKQIQbTbW**QxuvSv|0C1T%T*K z5HrB8pVVhom*J*u>Ss{!YshX?_1Y@c7Q=!*qsnBv;z1JMTx+msuQ+oQ5ds%29`N}# zjJsM_#|r?pQpz>#zV>Fu_VE@&=WcbcipUd0Bid$O2eDr9(TF9*HBDTvtfaolC-Wtu zsvtV>*3=Y?lgVwZAUd^bzNQddtrg@-w`Ohos&6~{(saVl=jF2-UGP!0MiQuZcBVHv z{^y2aMc^1?{}O;46juA7en=6%B>Q4RAK3znb#Zq*rBVoRjD8bjgLP zUqUrTc;$~1^CV>4wp8nNyjg>Pzg5STmBqz~dzh;#|9Y&bM`a|#zlEE8GB-#U zan7zg#!zRFy$7^d&{?eKg3&MXXb!(+gco~~Gf?dy-(zGZ;ep5@<@1Fwo2 z?1X7pZmo5Z=*u!65xx9|^Ej~N*T~r6hiad2H0jk{B0 z$Gk|0JNQW_TabrhIfU~? zu$?31THpXv)sY(IeF(nS0rE3y+vNKBu)Sss`v%HH{b&-=C)vC3<51o-sAk=hU3nm^ zxu!uGht@ZE|M+_xQfOVghQBg%rZ3vJOlAoO9rOyM z2zQL1$U!P?OWN3162OAo`7TlK@Fn_%#-#IUfAooa8PplF?WCDMN9fBqxb;|slFOR5 zCnEnjLmaVk9gch|NYlWc@d`TxP>S=-u?}HHYqOH#=3>NM1{vcITjuqHnU< zyxt&l8@ssG8o^MJORq{7m!ByU_ZT5~RsD|J%wwR@{VU6Q%_pF#;(k%OS*nISKGNKK zG9LMyoO4z8bNYNXc1y4!7T1)xMm_}A@DxGBKo}T@bP(3M$F?{s#GEqrxBGkIo@CuW<44%Ee3lFLB6<6mM661b}G%DCl-a;?Wir=Jomi} zBM-K#BZ6+J&Z4)%kJUMZk)W9Vna-X*$s?9qODrGUb%NB~hpfP3*xfjaH9Cpz@bHN}4mZ+*-am=i#C=FT>mhs^JAU zg?3$@6LnJk5N;ZcAy|lc3?M#MQ<(GJ=n z%hA_~Ow5}$dsR2ukpwY39}52-YWO08QN#kL=_S{xhZ@n9m08Ht-lRj&OYrK3DZA8( z2(SMPmtT?KAj%ZV{666A^JjD6Qme%v7oDJGiIkqCpH6b6pA@M%x;U|m)_YPZymt#z zf55jGMlpqIoKMQAgfKGP`}O<0bX^PXf(s)}C+(@Pb5s#q>CNgd9EzO#(i8a^bG=`( zz^dM%ot({NX?wI*(De;P681=RWudDtqAjk0r>2NlbeKJs{u!7L@A<-8D3i-t$P2e6 zsUMyI6R2%FP2DZ6g=S@f2o>s2&w%PI2OGpQKr{lerFliS9$QdfM=#u;S>=~8 zut^V`J!cMlJ3Gb#YigV<7JS?&wJ*oqtwt~4t2O@eDT>aRFVEi;v0JWVTCxBjK|Bs- zq@Q2rBsvf-;mvpel-7@ObQ9AlBR+R4tlYh2LGCy;JC9hFDg7*pyu&&8%`<(V@e&>) z>RLIwYS{Y{{n{&@V4?s_rKqQ`uc50%#$LCqQlF6G!W%FEbixXzDIIjRAV8x_#S{4? zrGwdm)=oxR#2Q&OYP=}#%IVZ>f6A`EV?v_;j=0Ti?6>_vLy!dQTRP1Z|T#POz()h z;S))zC-+t55sMTi?a-Nf>_=%nUPv-M!otG_!9%8{#Ft-v?`80niR^kr%_NzTXS2l+ z!O=*QFhcE=CogyQj-Z-WNs`N`P7EQCAr>L?1M%xd86S*69C&uExW-Imx7~aEUV<#` zjf%_$(Zn9!sKq49=8BxMxvnqvJn8qR`9+gGuBJfC9t>8Wo*XGq<)y~KOI&NrBG^!y zWo4)32aF%-nol(C4K_iknBILLanW0L8yJ7BwKV9${)&h~@`z(vhm`~Nk*Knf75Ra!!!EIBiWU z#eEcWL}yZ;_vR(Mdq4c2W~M#U3rrS3*BX*z|K^VJx%#M&rqcV1Nn4=-s>H3X5BCp! zG{2}%vv$V<54S|rd#|6acqe(M{`Z9-=VY=nLa4Gse*7`7cbJH>ny)Os*-p!ZOA1s& z-O}PzG^fKjYw8pruOwa49-gEB1aXu&kS}jJYMdpnyiEh^{K1!cJjGmEy@R0HAT(#0 zkQ}4)aw*vHT}2#|OlU~n5#-8JOKpd0NQjGFIv2JWX2!ri>KntkolyLFv6R+TG(5l* z{cc>#B#uqSHqwU374upEbsVA;eRs$Iv~{INq>Wd=a9@L?Ejr*UbFa@K2EPbqnQ_>*#7pIH;gGbTD8}gpmx{2{)y8{|3-~SJ34okCSz;fR+Gc#FroHUD-qyUz$_~7#-vTNAe*JI($9{t z9s{ddQ8g&)hq}&>SDvNBA^Y)hC2e%O{3@lW&=&5-rPg&tiUh3xo?%p;1tGkft40RK zq{+B%hNifDk090Q_RI?1(S-<4Y5G0jJkoL~+%Wosu9<3zyvpKeE^goJqG@HKc({er zTI27oBe{YwkD$H=24W4-BhgCUGIvjbjm`s`3#7d(l(Hosns2uRJTRhCL?k?=qtEm; z-^>Xn+5HMx>t^@FZ(gB$tI-0VWY!10|i6pUVnJ!AWdam?5>=C+Xv3AhFp?_@ZZCq6~ zVx1+}@iSZ%9mzjK13WrvMp~GhX&I6cPF{7pSR~4OITUq^SAQ8{Da>Nec*h6(yOUF^ zws>qrN88r?m23Tv_wC64tR{y)J3sB6zMrpyZcM1u2qQn!L-*z_KKFLgXTGOm0t+Ku zz!#n}>Xr0#=AjgcAWyh`-1^!nYxQJllsc0ba-glBl&HXGo;Txu=qa{nBqs3w^&edB zQLocH@cPW0hP2+@Bo3WNO#`LXaOCK-I2qMXgivbuUq{)7OJ)FBs7ZCdtPBzESQ%-L zUcj-|Le9I6HjAzO~%dJV}GMP$(kUma$p7@*C?q$1`2M4Sl z=g9`umZMDn;3jVMkZ4XUb0l-nf2Z!VzZe>Sr61%xQxit~m(pK7Z-%;5Z+jYcLmpTv zbUxo5&t+vjpB9{nadKr=Zq($5K*DU*&5hzvT9FH^4eG?>kqr-}j(SfG7EBjQw+E$c z`S^@y&6t#k11``dVw%2QYo_Blq5u~Bc{Jyhv|hV*y;fZY3S`Eld?r$=z4v3Ghq z!y?KTIG$64moF-zCG{tL@aZoY0TcglroFG0nVrra!wO*=k!Uy?qDW%0jeL%i%}^*V z@jUop5a{*&aH7;d|51EJ3BOQBE~UkP+;4~SJg?wK1FwnFJ5h7}T||$9E%L}4Cd@n_ za`^sIE2c6(EF<_aidZPX$MBs$+Rk}GZTlXvIP z&5|OS50TKjiy0fEaksIRV6>4P(2SY*G~Y-FDT{$c3A>ZL;k(c4IitB`|24x_r@(P# zW!>m=rj66~J@*UV9n0}@x;GEiJ<7uj=x;6{>n!sqi^ed(nuLBnxvo!@fm)tCWR1n} z_8SHD;1Z2D1y5}w23pBqJoZZ`7@gi64@pjDCE)M23SY2*l|)CZ%!X&g#$|SR9BN4@ zqS`4eit1K9nULWo)4xeW)b!bm?{Sj}zkZ8;`Om~zex}}UqgaYW!ApsZT^moi{*kdB zONd2LpPJ^8szXM&G<(kT$vZ zKb~dOT!``;Asi2@W4hyZIw9T^usG#$Hd{A-8vb*^ah{>sph3}%%wV$=z|I#X4?8^< zCa+YUHc0lpBg+j^AR%t9ler`64o#qQLFrIz`3Q@Oy5}YXmy4MWYIArHDCe2yPGd?dH$E zwYeM2E7|uVrX9Kl+_QwVz()ebUMx_1C@WyD?7Wx!`_rSEwVN;?iA(J}AQID9>=cjT zRQyW|)st?D;XZ0+;y8%@QfO%n_?>n$A?$_?tkGdv*ek&_BBdw(OALFfFPP7d)O3!c z4ZI6BXf=oQ{#0zezZRFKh&HNq?Kr!M^qeMa^b?5GpxII>TAq#j`)hzozAR+VSUAM& zF$XqyrJs^nRFj5c171}UQJPa}-8!oOcXK-~{vQ)SGm|rY2JGlxvbz)!UD)Kcq@ZGm zy`c2m8tTqRGPB1o1z<8-jFXAjilp{BfumQ;g{1JK2LKUGh|iB9sy_Xh&vI%lb7S;% zW#4qdqcEY*YH4+Xn5F9aFxdPROPQf|*WbB92A+@BSr`MCNMRQYAh*oWFJxpK-f+K5 z4qa`e@qDElQ;TBZt90&xKBJyq(yP~T-|{0;IR5GjC#)QlC(;IU3|Qb|GOK^q?)@Wz zT?m6B=6@m8g#!`;y;(LRCELlgjeN^;*H#n0{BYugPmNYEUexG(z7gCn8ha9DtV0Yx z(!N&NbKt97wJdBUU1U`;S{xf?I%B`zgFjBbrTK`a{=My0G?b)y@K-A(t$&VnM=d9E zGXSLMDetXcw<&{aAd6Hr*4f3Zxzn~ z0hMA*5BO>Q#=p;BU&dyUqwGwNH70E_?DS`>vohHq-z5o_26M&&LfC~g=%N5@)k5jt zQtvVh@00xm8$-dyc|$iN*0=OWDbQ~cX8}6(4*mTVx7d{xKzb^Bkt$J7-+@SjBIch> zQH;9!Zi(*MHwUcEZ8&{N{$@&aP=&d~+<;x&y%xQkEFj{!==#xsT1B(*Xn2Z9ujAQq z7S{qJO@kFdAz$jO86qCbb_3KIb310|37#~-L-D?u6^Wt0+R(P;6D!yd?qd7gf*_+A zKNPfveHs}P0lB5jpwHd*zezL{?^{WE=Hp)l432%$ipLKgjDq&p1y%#My6aMX*gzb;Ie(Nh>C7W)22%J-tU%(pOBvNZekc zbiB_WY0Q#2h+rXQ$;ZSQyv{&&j0U2Eb-=B*(?Ncj33%JZFT(mufKl58g!WR%)J7{d zTS%qq-z~6nhO71dER!{dJH1WRu<()x#Ae0cz2;4D%v;5A*!G9{Y-*$4(fGk5u_{wi zj>0x;)9)eQL8dNyT0&p%cX`DUoXpmN?*8Hv-YjCo+j#eVEE=T8gDa5cN!l+IPv9d* z`}_2^7|z}P{D0Rz%~;9`!iiTc5LyI|y-eVKY*~evp3fNB`+_XNN9E4RZ3G6F_dw;|EZBZc8@VoI_>`E0Y&Gj zY2x2oVzpZV-zr9-(jgaek_{TsOFuz%D{YlO`!!8=6!w(2(48ee+_lE|*0pXq%&Y&w zTbloP^>&;4hPi8QANlkA@WY{HPM5uh#-Z?<{zbHS&qdiap7HPx%@$7IM%N;PhRQoQY1uC zkKzUOM9joaCyq9Ww29IZeE8p{yboX|tw%mV1<#BlqTAE&1~ljEqZ?Hc%`2^ldIeWW zLn>>S&g=390$fYYSutB?Zd*~QU|48{o}iVe^oBPypEQit0Tu5E_wOLptFyoOB|Ue# z*p)qE#}6Kwc=t?5S-PC%8;>8rY!2xxl+c;q@yxOaVXK+pAq86h$u z)a~=Gn-fe%T%WA-Gw7HngfDdskq(mt z7Zwg)8R8#aqoH^QHi57eh|Io;w1mCV2}n&B!l)#?D)sH)zIM~jn2GW1(PG>}2sBh} zd5EfEsfcq__yHMFYC}Yev_#y!mhjo^MfO`j9xicp1Zn>kkej5uZygT7kxLK8Id>m%KdF@fdJlp z`#w;X`V4;5HJb9nC6z|xCNM(%=*2tL8h%mW=QoLJ!)O@80KJyjl5ng+;j#kwl#uHp zvP`42zLwaJggA+;--6Te*r_A?8&0(o1WRhbdSNMt!-kg+`S<>{ojIx~zr-Zvi$3-( zCde+xOHt=jF5Y5HBJqz8jaV8QlNhaj=+qYF1BT;PXhprf|RJkeJBkrp{Ug-X&r(E_PJ)d1P>NNw%+f*^7D zfbGwm>eW23GS>b>^GdN%)e@CnbM~x1v+O3thhy=VGN423fg1M?Y>Oy|g5B&4K$e zDsG0Nozig;{B89&`dcnHVg9_J9#&i@_dFDexT3j}p0lO(6G*KzW%KSOgpy)SSKh>E8mK4Z_k`@{o3?HIKZWdoR9A^hh)F|CA zrUFwo3P`1U3!Y1{{Xu6pR#OoA-3MDF(-Tp~{sBN!d8UGjH5A3pI!CA%1hazw6VMY< z_IOHdfB?{Kbr41R!hz$@e2pl^!cf)6_F8_o>)gI>DJkL&a4d*QFQ)=VN?%|Bf(71| z932(4rhCAB?etKa*eK25sX@K}_`CC~_xt1P8UEzZmu-#za>B6n3D;#?@U6eDy3OP! z2Lrt=;{L!SD$jtp9Pahzi-FN^DY$s=U7|BQwIGpgsnG1a>{p3rA}c+1|NXu#=d?6_ ztqER}a_y{$NiaW3Pe56~w`@Ef@D}l9B>97f(;1zWs;0pgum6SN=H7QDf#K(G2qF1? z>wfV1x+WHCyLZ#}TLK3pDDsXnaEb0|`0!lhq3)Je?oCje^Gwdk8j`LFhY1Y!zqr#> z&&y5BWTJ@jP(WxUBrr`AeS@C!BnjCX=Am$n)$`$!kJdQS-$?uXEej2!tw%V-*Fh?Q zMd^v=7Q(r(By0g`Xa^n`AY@dL=#w|-3;U&^c>eo;1FFk+w2?&(ZQ1Bkv0S_}8BS%f zqOt2aHG5>dQ77r;ca=W}z{Pv9=<6Jw6=AeG#zvaT%Lk5cETMa^~>>{nFGc=CAk__(N2hJ3#P^QQ-a zA~uWLvk-5L+YyOGiAsLs75gQxKhr7zuX?~a??Kr5WZINGL=6w-54;fx6kdwYDTeMI z_sieweNv_#e{@@5Tu57Z=CEY+kp9`{E<8W>`2Xt=elb_K^yXrt4h_dmJ508$z$BsX z*iqE>qymJM0J+I?t!Nj6A@o0Q4{W$@ntZb4BH#Sb$+~GfcK)&N;7vLR%%2c!)RPM4 z!NVZrdC~K2aq=(4#n z8D^1kXhRyQg1a3ipO}&EA|4HpwE#@8u8HM zN7dW9U=(KUYga3=iTG6~TFjDk-m6!Zh;$r%R|_x|6vk!T-y+cRJrI<1Rl@swy-<1P zt{A_iy_@J$=Oe0I(x5x&w{@S6ZO`O=qZ1^=1N5C*LZaB)PnK-E@LC#bnoBsYFJUF; z4nz?jR7?rU4JGrnTprRZas8pNpJuNi)rBe$fHey?#Uoj~%5FmVz~JFW)h5foIyCmk zN^x`UT}s>1J$E3>o!*s(zzcF930fg;`RQ=y$xl*h7?V{+VY}#D^Sc6G4{ejvY-srk z@_8i$vJU}gj9dCP>w9n&B%h`_ua_#b+xm=yCH({ju(vt50h2>-EpWltkwc_+WjW`W z#IpQee2?dVEYgF@iVD7k89hh#t@+KeVY0p*l9Y~h-Pza29e{ZC9|hPSzHQDg zZPL$2_s@zR;?gEM`o~7HZ`^<^l7Lm8A!+4Rwnu?HIoc}|o_v92D_Ul0qNUH!K8*&CDZ#x4^vpul!rN^(5BxO$AL;IO zb=WtSOb7&zL_= "2" and sys.platform != "win32": + model = torch.compile(model) + + return tokenizer, model + + +def main(): + parser = argparse.ArgumentParser() + + parser.add_argument('--model', type=str, default='bigcode/starcoder', help="") + parser.add_argument('--output_path', type=str, help="") + parser.add_argument('--start_index', type=int, default=0, help="") + parser.add_argument('--end_index', type=int, default=164, help="") + parser.add_argument('--temperature', type=float, default=0.8, help="") + parser.add_argument('--N', type=int, default=200, help="") + parser.add_argument('--max_len', type=int, default=512, help="") + parser.add_argument('--decoding_style', type=str, default='sampling', help="") + parser.add_argument('--num_seqs_per_iter', type=int, default=50, help='') + parser.add_argument('--overwrite', action='store_true', help='') + + args = parser.parse_args() + + argsdict = vars(args) + print(pprint.pformat(argsdict)) + + STOP_SEQS = ['\nclass', '\ndef', '\n#', '\nif', '\nprint'] + + problems = read_problems() + + task_ids = sorted(problems.keys())[args.start_index: args.end_index] + prompts = [problems[task_id]['prompt'] for task_id in task_ids] + num_samples = len(prompts) + print("Number of samples: {}".format(num_samples)) + + tokenizer, model = get_model(base_model=args.model) + generation_config = GenerationConfig( + pad_token_id=tokenizer.pad_token_id, + do_sample=True, + temperature=args.temperature, + max_length=args.max_len, + num_return_sequences=args.num_seqs_per_iter, + eos_token_id=tokenizer.eos_token_id, + top_p=0.95 + ) + + print(f"Loaded {args.model}.") + for i in tqdm(range(num_samples), ncols=0, total=num_samples): + output_file = args.output_path + '/{}.jsonl'.format(args.start_index + i) + + if os.path.exists(output_file) and not args.overwrite: + print(f'Skip {output_file} as it already exists') + continue + + prompt = prompts[i].replace(' ', '\t') + prompt_batch = [generate_prompt(prompt)] + + ids_batch = [task_ids[i]] + + completion_seqs = [] + + encoding = tokenizer(prompt_batch, return_tensors="pt", truncation=True, max_length=args.max_len).to(device) + + if args.decoding_style == 'sampling': + loops = int(args.N / args.num_seqs_per_iter) + else: + loops = 1 + + for _ in tqdm(range(loops), total=loops, leave=False, ncols=0): + + with torch.no_grad(): + if args.decoding_style == 'sampling': + gen_tokens = model.generate( + **encoding, + generation_config=generation_config + ) + + if gen_tokens is not None: + gen_seqs = tokenizer.batch_decode(gen_tokens, skip_special_tokens=True) + else: + gen_seqs = None + + if gen_seqs is not None: + assert len(ids_batch) == 1 + task_id = ids_batch[0] + + for seq_idx, gen_seq in enumerate(gen_seqs): + completion_seq = gen_seq.split("### Response:")[1] + completion_seq = completion_seq.replace('\t', ' ') + all_code = gen_seq.replace('\t', ' ') + + completion_seqs.append( + {'task_id': task_id, + 'completion': completion_seq, + 'all_code': all_code, + } + ) + + print("Saving results to {}".format(output_file)) + write_jsonl(output_file, completion_seqs) + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/WizardCoder/src/inference_wizardcoder.py b/WizardCoder/src/inference_wizardcoder.py new file mode 100644 index 0000000..1ec8f1c --- /dev/null +++ b/WizardCoder/src/inference_wizardcoder.py @@ -0,0 +1,121 @@ +import sys +import os +import fire +import torch +import transformers +import json +import jsonlines + +from transformers import AutoTokenizer, AutoModelForCausalLM, GenerationConfig + +if torch.cuda.is_available(): + device = "cuda" +else: + device = "cpu" + +try: + if torch.backends.mps.is_available(): + device = "mps" +except: + pass + +def evaluate( + batch_data, + tokenizer, + model, + input=None, + temperature=1, + top_p=0.9, + top_k=40, + num_beams=1, + max_new_tokens=2048, + **kwargs, +): + prompts = generate_prompt(batch_data, input) + inputs = tokenizer(prompts, return_tensors="pt", max_length=256, truncation=True, padding=True) + input_ids = inputs["input_ids"].to(device) + generation_config = GenerationConfig( + temperature=temperature, + top_p=top_p, + top_k=top_k, + num_beams=num_beams, + eos_token_id=tokenizer.eos_token_id, + pad_token_id=tokenizer.pad_token_id, + **kwargs, + ) + with torch.no_grad(): + generation_output = model.generate( + input_ids=input_ids, + generation_config=generation_config, + return_dict_in_generate=True, + output_scores=True, + max_new_tokens=max_new_tokens, + ) + s = generation_output.sequences + output = tokenizer.batch_decode(s, skip_special_tokens=True) + return output + + +def generate_prompt(instruction, input=None): + return f"""Below is an instruction that describes a task. Write a response that appropriately completes the request. + +### Instruction: +{instruction} + +### Response:""" + + +def main( + load_8bit: bool = False, + base_model: str = "Model_Path", + input_data_path = "Input.jsonl", + output_data_path = "Output.jsonl", +): + assert base_model, ( + "Please specify a --base_model, e.g. --base_model='bigcode/starcoder'" + ) + + tokenizer = AutoTokenizer.from_pretrained(base_model) + if device == "cuda": + model = AutoModelForCausalLM.from_pretrained( + base_model, + load_in_8bit=load_8bit, + torch_dtype=torch.float16, + device_map="auto", + ) + elif device == "mps": + model = AutoModelForCausalLM.from_pretrained( + base_model, + device_map={"": device}, + torch_dtype=torch.float16, + ) + + model.config.pad_token_id = tokenizer.pad_token_id + + if not load_8bit: + model.half() + + model.eval() + if torch.__version__ >= "2" and sys.platform != "win32": + model = torch.compile(model) + + input_data = jsonlines.open(input_data_path, mode='r') + output_data = jsonlines.open(output_data_path, mode='w') + + for num, line in enumerate(input_data): + one_data = line + id = one_data["idx"] + instruction = one_data["Instruction"] + print(instruction) + _output = evaluate(instruction, tokenizer, model) + final_output = _output[0].split("### Response:")[1].strip() + new_data = { + "id": id, + "instruction": instruction, + "wizardcoder": final_output + } + output_data.write(new_data) + + +if __name__ == "__main__": + fire.Fire(main) \ No newline at end of file diff --git a/WizardCoder/src/process_humaneval.py b/WizardCoder/src/process_humaneval.py new file mode 100644 index 0000000..1023a09 --- /dev/null +++ b/WizardCoder/src/process_humaneval.py @@ -0,0 +1,69 @@ +from human_eval.data import read_problems, write_jsonl, stream_jsonl +import glob +from tqdm import tqdm +import argparse + +parser = argparse.ArgumentParser() + +# Inputs +parser.add_argument( + '--path', + type=str, + help="") +parser.add_argument( + '--out_path', + type=str, + help="") +parser.add_argument( + '--add_prompt', + action='store_true', + help='') + +args = parser.parse_args() + + +files = sorted(glob.glob(args.path + '/*.jsonl')) +print("{} files in {}".format(len(files), args.path)) + +problems = read_problems() + +output = [] +a = 0 +for code_file in tqdm(files, total=len(files)): + codes = [c for c in stream_jsonl(code_file)] + if args.add_prompt: + for code in codes: + task_id = code['task_id'] + prompt = problems[task_id]['prompt'] + completion = code['completion'] + completion = completion.replace("\r", "") + if '```python' in completion: + def_line = completion.index('```python') + completion = completion[def_line:].strip() + completion = completion.replace('```python', '') + # print(completion) + try: + next_line = completion.index('```') + completion = completion[:next_line].strip() + except: + a += 1 + print(completion) + print("================\n") + # print(completion) + if "__name__ == \"__main__\"" in completion: + next_line = completion.index('if __name__ == "__main__":') + completion = completion[:next_line].strip() + # print(completion) + + if "# Example usage" in completion: + # print(completion) + next_line = completion.index('# Example usage') + completion = completion[:next_line].strip() + + code['completion'] = completion + + output += codes + +print("save to {}".format(args.out_path)) +write_jsonl(args.out_path, output) +print(a) \ No newline at end of file diff --git a/WizardCoder/src/train_wizardcoder.py b/WizardCoder/src/train_wizardcoder.py new file mode 100644 index 0000000..245a9ef --- /dev/null +++ b/WizardCoder/src/train_wizardcoder.py @@ -0,0 +1,248 @@ +# Copyright 2023 Rohan Taori, Ishaan Gulrajani, Tianyi Zhang, Yann Dubois, Xuechen Li +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import logging +import random +from dataclasses import dataclass, field +from typing import Optional, Dict, Sequence + +import torch +import torch.distributed +import transformers +from torch.utils.data import Dataset +from transformers import Trainer +from datasets import load_dataset +import utils + +IGNORE_INDEX = -100 +DEFAULT_PAD_TOKEN = "[PAD]" +DEFAULT_EOS_TOKEN = "<|endoftext|>" +DEFAULT_BOS_TOKEN = "<|endoftext|>" +DEFAULT_UNK_TOKEN = "<|endoftext|>" +PROMPT_DICT = { + "prompt_input": ( + "Below is an instruction that describes a task, paired with an input that provides further context. " + "Write a response that appropriately completes the request.\n\n" + "### Instruction:\n{instruction}\n\n### Input:\n{input}\n\n### Response:" + ), + "prompt_no_input": ( + "Below is an instruction that describes a task. " + "Write a response that appropriately completes the request.\n\n" + "### Instruction:\n{instruction}\n\n### Response:" + ), +} + + +@dataclass +class ModelArguments: + model_name_or_path: Optional[str] = field(default="bigcode/starcoder") + + +@dataclass +class DataArguments: + data_path: str = field(default=None, metadata={"help": "Path to the training data."}) + + +@dataclass +class TrainingArguments(transformers.TrainingArguments): + cache_dir: Optional[str] = field(default=None) + optim: str = field(default="adamw_torch") + model_max_length: int = field( + default=512, + metadata={"help": "Maximum sequence length. Sequences will be right padded (and possibly truncated)."}, + ) + + +def safe_save_model_for_hf_trainer(trainer: transformers.Trainer, output_dir: str): + """Collects the state dict and dump to disk.""" + state_dict = trainer.model.state_dict() + if trainer.args.should_save: + cpu_state_dict = {key: value.cpu() for key, value in state_dict.items()} + del state_dict + trainer._save(output_dir, state_dict=cpu_state_dict) # noqa + + +def smart_tokenizer_and_embedding_resize( + special_tokens_dict: Dict, + tokenizer: transformers.PreTrainedTokenizer, + model: transformers.PreTrainedModel, +): + """Resize tokenizer and embedding. + + Note: This is the unoptimized version that may make your embedding size not be divisible by 64. + """ + num_new_tokens = tokenizer.add_special_tokens(special_tokens_dict) + model.resize_token_embeddings(len(tokenizer)) + + if num_new_tokens > 0: + input_embeddings = model.get_input_embeddings().weight.data + output_embeddings = model.get_output_embeddings().weight.data + + input_embeddings_avg = input_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True) + output_embeddings_avg = output_embeddings[:-num_new_tokens].mean(dim=0, keepdim=True) + + input_embeddings[-num_new_tokens:] = input_embeddings_avg + output_embeddings[-num_new_tokens:] = output_embeddings_avg + + +def _tokenize_fn(strings: Sequence[str], tokenizer: transformers.PreTrainedTokenizer) -> Dict: + """Tokenize a list of strings.""" + tokenized_list = [ + tokenizer( + text, + return_tensors="pt", + padding="longest", + max_length=tokenizer.model_max_length, + truncation=True, + ) + for text in strings + ] + input_ids = labels = [tokenized.input_ids[0] for tokenized in tokenized_list] + input_ids_lens = labels_lens = [ + tokenized.input_ids.ne(tokenizer.pad_token_id).sum().item() for tokenized in tokenized_list + ] + return dict( + input_ids=input_ids, + labels=labels, + input_ids_lens=input_ids_lens, + labels_lens=labels_lens, + ) + + +def preprocess( + sources: Sequence[str], + targets: Sequence[str], + tokenizer: transformers.PreTrainedTokenizer, +) -> Dict: + """Preprocess the data by tokenizing.""" + examples = [s + t for s, t in zip(sources, targets)] + examples_tokenized, sources_tokenized = [_tokenize_fn(strings, tokenizer) for strings in (examples, sources)] + input_ids = examples_tokenized["input_ids"] + labels = copy.deepcopy(input_ids) + for label, source_len in zip(labels, sources_tokenized["input_ids_lens"]): + label[:source_len] = IGNORE_INDEX + return dict(input_ids=input_ids, labels=labels) + + +@dataclass +class DataCollatorForSupervisedDataset(object): + """Collate examples for supervised fine-tuning.""" + + tokenizer: transformers.PreTrainedTokenizer + + def __call__(self, instances: Sequence[Dict]) -> Dict[str, torch.Tensor]: + input_ids, labels = tuple([instance[key] for instance in instances] for key in ("input_ids", "labels")) + input_ids = [torch.tensor(x) for x in input_ids] + input_ids = torch.nn.utils.rnn.pad_sequence( + input_ids, batch_first=True, padding_value=self.tokenizer.pad_token_id + ) + labels = [torch.tensor(x) for x in labels] + labels = torch.nn.utils.rnn.pad_sequence(labels, batch_first=True, padding_value=IGNORE_INDEX) + return dict( + input_ids=input_ids, + labels=labels, + attention_mask=input_ids.ne(self.tokenizer.pad_token_id), + ) + +def train_tokenize_function(examples, tokenizer): + prompt_input, prompt_no_input = PROMPT_DICT["prompt_input"], PROMPT_DICT["prompt_no_input"] + if 'input' in examples: + sources = [ + prompt_input.format_map(dict(instruction=instruction, input=input)) if input != "" \ + else prompt_no_input.format_map(dict(instruction=instruction)) \ + for instruction, input in zip(examples['instruction'], examples['input']) + ] + else: + sources = [ + prompt_no_input.format_map(dict(instruction=instruction)) \ + for instruction in examples['instruction'] + ] + targets = [f"{output}{tokenizer.eos_token}" for output in examples['output']] + data_dict = preprocess(sources, targets, tokenizer) + return data_dict + + +def train(): + parser = transformers.HfArgumentParser((ModelArguments, DataArguments, TrainingArguments)) + model_args, data_args, training_args = parser.parse_args_into_dataclasses() + + model = transformers.AutoModelForCausalLM.from_pretrained( + model_args.model_name_or_path, + cache_dir=training_args.cache_dir, + ) + + tokenizer = transformers.AutoTokenizer.from_pretrained( + model_args.model_name_or_path, + cache_dir=training_args.cache_dir, + model_max_length=training_args.model_max_length, + padding_side="right", + use_fast=True, + ) + if tokenizer.pad_token is None: + smart_tokenizer_and_embedding_resize( + special_tokens_dict=dict(pad_token=DEFAULT_PAD_TOKEN), + tokenizer=tokenizer, + model=model, + ) + if "starcoder" in model_args.model_name_or_path: + tokenizer.add_special_tokens( + { + "eos_token": DEFAULT_EOS_TOKEN, + "bos_token": DEFAULT_BOS_TOKEN, + "unk_token": DEFAULT_UNK_TOKEN, + "pad_token": DEFAULT_PAD_TOKEN, + } + ) + + raw_train_datasets = load_dataset('json', data_files=data_args.data_path, split="train", cache_dir=training_args.cache_dir) + if training_args.local_rank > 0: + torch.distributed.barrier() + + train_dataset = raw_train_datasets.map( + train_tokenize_function, + batched=True, + batch_size=3000, + num_proc=32, + remove_columns=raw_train_datasets.column_names, + load_from_cache_file=True, # not args.overwrite_cache + desc="Running tokenizer on train dataset", + fn_kwargs={"tokenizer": tokenizer} + ) + + if training_args.local_rank == 0: + torch.distributed.barrier() + + if training_args.local_rank == 0: + print(len(train_dataset)) + for index in random.sample(range(len(train_dataset)), 3): + print(f"Sample {index} of the training set: {train_dataset[index]}.") + + data_collator = DataCollatorForSupervisedDataset(tokenizer=tokenizer) + data_module = dict(train_dataset=train_dataset, eval_dataset=None, data_collator=data_collator) + + #Tell Trainer not to attempt DataParallel + model.is_parallelizable = True + model.model_parallel = True + + trainer = Trainer(model=model, tokenizer=tokenizer, args=training_args, **data_module) + model.config.use_cache = False + + trainer.train() + trainer.save_state() + safe_save_model_for_hf_trainer(trainer=trainer, output_dir=training_args.output_dir) + + +if __name__ == "__main__": + train() \ No newline at end of file diff --git a/WizardCoder/test b/WizardCoder/test deleted file mode 100644 index 8b13789..0000000 --- a/WizardCoder/test +++ /dev/null @@ -1 +0,0 @@ -