From c9908bedb45aa868c272f3a64018ceb260e53f79 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Wed, 13 Dec 2023 11:42:46 +0000 Subject: [PATCH 1/5] Copy parquet files to exports directory --- .gitignore | 3 + .pre-commit-config.yaml | 2 +- cli/tests/conftest.py | 18 ++++ .../resources/omop/public/CARE_SITE.parquet | Bin 0 -> 4084 bytes .../resources/omop/public/CDM_SOURCE.parquet | Bin 0 -> 5823 bytes .../omop/public/CONDITION_OCCURRENCE.parquet | Bin 0 -> 6770 bytes .../omop/public/DEVICE_EXPOSURE.parquet | Bin 0 -> 4524 bytes .../omop/public/DRUG_EXPOSURE.parquet | Bin 0 -> 5782 bytes .../omop/public/FACT_RELATIONSHIP.parquet | Bin 0 -> 2167 bytes .../resources/omop/public/LOCATION.parquet | Bin 0 -> 1865 bytes .../resources/omop/public/MEASUREMENT.parquet | Bin 0 -> 6742 bytes .../resources/omop/public/OBSERVATION.parquet | Bin 0 -> 5614 bytes .../omop/public/OBSERVATION_PERIOD.parquet | Bin 0 -> 2183 bytes .../resources/omop/public/PERSON.parquet | Bin 0 -> 5420 bytes .../omop/public/PROCEDURE_OCCURRENCE.parquet | Bin 0 -> 5230 bytes .../resources/omop/public/SPECIMEN.parquet | Bin 0 -> 4873 bytes .../omop/public/VISIT_DETAIL.parquet | Bin 0 -> 3228 bytes .../omop/public/VISIT_OCCURRENCE.parquet | Bin 0 -> 5259 bytes cli/tests/test_copy_omop.py | 82 ++++++++++++++++++ docker-compose.yml | 3 + pixl_core/pyproject.toml | 1 + pixl_core/src/core/omop.py | 72 +++++++++++++++ 22 files changed, 180 insertions(+), 1 deletion(-) create mode 100644 cli/tests/conftest.py create mode 100644 cli/tests/resources/omop/public/CARE_SITE.parquet create mode 100644 cli/tests/resources/omop/public/CDM_SOURCE.parquet create mode 100644 cli/tests/resources/omop/public/CONDITION_OCCURRENCE.parquet create mode 100644 cli/tests/resources/omop/public/DEVICE_EXPOSURE.parquet create mode 100644 cli/tests/resources/omop/public/DRUG_EXPOSURE.parquet create mode 100644 cli/tests/resources/omop/public/FACT_RELATIONSHIP.parquet create mode 100644 cli/tests/resources/omop/public/LOCATION.parquet create mode 100644 cli/tests/resources/omop/public/MEASUREMENT.parquet create mode 100644 cli/tests/resources/omop/public/OBSERVATION.parquet create mode 100644 cli/tests/resources/omop/public/OBSERVATION_PERIOD.parquet create mode 100644 cli/tests/resources/omop/public/PERSON.parquet create mode 100644 cli/tests/resources/omop/public/PROCEDURE_OCCURRENCE.parquet create mode 100644 cli/tests/resources/omop/public/SPECIMEN.parquet create mode 100644 cli/tests/resources/omop/public/VISIT_DETAIL.parquet create mode 100644 cli/tests/resources/omop/public/VISIT_OCCURRENCE.parquet create mode 100644 cli/tests/test_copy_omop.py create mode 100644 pixl_core/src/core/omop.py diff --git a/.gitignore b/.gitignore index 5bf1ebaa1..7cea36e68 100644 --- a/.gitignore +++ b/.gitignore @@ -143,3 +143,6 @@ dmypy.json # VS Code .vscode + +# project specific files +/exports/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 616bed3de..83bdba3b7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,4 +31,4 @@ repos: pass_filenames: false entry: mypy args: ['--config-file=setup.cfg'] - additional_dependencies: ['mypy', 'types-PyYAML', 'types-requests'] + additional_dependencies: ['mypy', 'types-PyYAML', 'types-requests', 'types-python-slugify'] diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py new file mode 100644 index 000000000..3da3052a7 --- /dev/null +++ b/cli/tests/conftest.py @@ -0,0 +1,18 @@ +"""CLI testing fixtures.""" +import pathlib + +import pytest +from core.omop import OmopExtract + + +@pytest.fixture() +def omop_files(tmp_path_factory: pytest.TempPathFactory) -> OmopExtract: + """Create an OmopFiles instance using a temporary directory""" + export_dir = tmp_path_factory.mktemp("repo_base") + return OmopExtract(export_dir) + + +@pytest.fixture() +def resources() -> pathlib.Path: + """Test resources directory path.""" + return pathlib.Path(__file__).parent / "resources" diff --git a/cli/tests/resources/omop/public/CARE_SITE.parquet b/cli/tests/resources/omop/public/CARE_SITE.parquet new file mode 100644 index 0000000000000000000000000000000000000000..18da482a3476066d2155eaa5bf34662eaf46975e GIT binary patch literal 4084 zcmd5A0CuH5x~W2vUVK77J!-J5AcKyK(bs8c$Q@=}qN0X;Rl|Lf)xZ zAjE%9MvXL;rc!>ssy48!mY|AY^G47^Q#jb94~SCDJd0Q1q@4%iR3SfH%W$1=hzUzle@ zQD$v36!aT{@GOGZgd3n)Lip&zn-C&5 zxPlZBdb=frf4t`*e7|SuM)Rb5dNOGC$7irKx)(;XH(Uh#h82Eg0kUUv$-|A1H~FV- zQrE*PfTE-vW|k8h1E|OkQo027r?oKecar(oOXh{%Y0`9l?hAb76MkP6Uay9i4Bdzb zl6nXLu*{g;$NUqo!ZA$rgCJ7)3qP)VS=z9=hCV#&UbtB)1~lQ#1L3y^VbFS8PhWCg zkO42qL3h-LdW>~zWCx2*e|aeU@bEsI{<+~f{oq;?-01uDhVaeCZN5XFMVa8*z-Tu+ zI`4-(KVui$-)7y_Zw&7TrKK`IV-CYuEVMUY23z=+bh%1s@9y9n$p9M)hpcyVc7=cE zyb1SuR^F=pg?s8~;eMMm@ZD;F71jekuRdkq&<1_|;1_Fzi*RyGjz!~~6yM$3+vB2K zshD~W_lU&Fl!}}r$}tYMFgn%C0sS&snf`U|)BxZEk%ELC9%TKy9)y&Fj@jRC=o=0Bpw{{e>lq;8y z(}%@YPA%rAikWrp1Z2oeZFP|+=gp)9`jXO#w(FJhrD7&!#RM6%B!Z30e4z{btfUg~ zQ7kOk0^h}24rI{(6#T%xkgLh5nO2Tp>eXYbDx=+!>r=`Xuj{CV-l$fQECb^JecN|( zk~eGMF9|piQx^EOS-x08j1#4k5*wL=UhSeRYE|%i^~?bO@j;^^jTY_ESY}R4j!nRF zrI(#49kp$A%BO~GiHN@*DbH$h`LxkVfxZ7WhQ%6Q&tcv?&l%}y8}Zg<4Pq$I8Xe30 zzs2G7-c=t!AI|kocByx)^SK_KsgvHMo=#y*W6g?RYSjZ-uc**t>bTxfEhBS$s#mao zuKGk=swwTWIh)gbp*N_@YP*rP#&s*+)3DFoUWT4q=;eqyZd4D@zg(+KC2hX%|4IJo z*}A{}hFOrtc~Hc$a)d)f*~5JsNs`VFoKYllw7b2H-ZYV{P!DS{fsL`Cp zJJkr}^g`=lPZ@GtG9bs*h}N!W$`;CNM=$`Vmq5^6GDV%(Lys;|h9qi8%v$vTbk$xX zeUmN4z)@O8dlX-mg33?>9zF%1rKe-4iKtiNqpED3ljOeRpCssz%$m;L#Y6QyJTRY^ z@zgu1p$1|wZ$S>_l|3-Bje>R($pF^)+`a0!xaGW=bvbHk)r3g#9lCfNd|(7#4gj@A z5)(p8;5bE-uD`CmK}s(wx6q3iJH1-?UI2RrQEUcKyXZ`CtEcxr9W{r~jb87$-tTuO gTjOSb0AFZZQsjmBLJV2()v)wqzV|cC5q$6e1sjU!pa1{> literal 0 HcmV?d00001 diff --git a/cli/tests/resources/omop/public/CDM_SOURCE.parquet b/cli/tests/resources/omop/public/CDM_SOURCE.parquet new file mode 100644 index 0000000000000000000000000000000000000000..c8f6c17025c81043cbce7be8b1543bc99f52eba7 GIT binary patch literal 5823 zcmeHLOK%%h6rQGSTBebJ7Mnzhs%~NlmDZkdoMw=^xOV)Go!I!jiRKkQGGn`b+e?;+ zAHae|m#E8%6{=VwgjgVc04p}^_yH_9=guS7apKYxRaM0(_KfG8^WE>f?zze#$qSxq z9?r9}4`1AEj^lj3P0!lK7Pq>w{=TZ2NBz#Qr|L&-+0;ENUe3Gez3bzAH$U05?7-qI zz_(XigpU~nSy6iWs3{Mcoi>B?<&`B+KdiV=JVG@(hNkxz>!&MAtiQjy=~=N{xnsN0 z(+yqj>qnYAVBPuV>JpfruDLKDAc+ewo?Tml@#}RL!;2Urz1N309I2hQs&@xRO^tEt z`Sm48f2_J8xjpKrvNAN}-qZnPP<~upg7VjmQxrQin%-A?&F-1>{B~mr%HKC#l)DZ} zM;)4adqC3+p*q%kI=IO2`h8jLJmp=)4>3;qz&ID=3w{Z$XSZCCTTHMS4qx4xGvD|n z#I3LP)rM}$9xvxv+ge#D{w;2Q6N?>X7nL=u!_=$udIe0+E5xm}eFLFUzEEMT@Pg8@ zNS#v+&cev~wv(XKG_&HD#JC33NZGH9-P(vw~W|3A@jI=eER*X@;3H&auuk2+!#l7agYG!fsse5l zu+R>4TGeT$Pz0P}S}|jTS|wv+C50>jti2@Y6%i9LrVNKkhIAWRB0rVODM5=P7nff! zVrEB5qBKz24j$zO_7+E%x8<9VDFBy3H4W=vY$gzcbT-!=R?C8+V!R4Do7Y(%l8g`0 zam0)Tv_z^~ZI=R;FOGj|x++S2tt<>3y!klW>o59zLELG4EM8}QfH*lmKtEE@*?z{pW@@zvJ(3?(g4vN) zNwsPviS{4U*V7EtY%MMvtD+>RX4a^Pv1YZMGBnd@XmMktoUMJt6B{EX4z)igJ4w43 z!IYuIOAS>l9oOQqKy8jw)#kKB2lh&fcg;}S*{&jnj-l6wRk1YIVp6w;eP#Dt2KHCY zlv-8Tvy^yjs+T0OR!Pm|vNUA;oE(tuObM2zfB~FJvuZHaRTFu@-e&fj%sA3^V|Aj4 zS?J}`Py-(juL6G6%+j=04z=ce!o3LNQVJ+B3FN@rECJslgAIGMnLSbF`Te(hG96WY zm`vdgj;#cauY)m+u}BAJn}V||f@5v3gYyeDNu18R06v0%kZeKn`*-0X8lyuo?959P z;zjXk&1et{Rm2+EBmQH*pBz5&`%yhtq?0s7_30yhisECO#=y|xlVgaH3Ar*9s5?}U z4ugLGtZ;vB{nMdKF-UkD?qMdEQVV8)p|tZ>y&7}z2UihK3iwCB4-CV33Uh+r)L;6O ze2zD_enqHlF*}tjVVZZ)?4bRVm-&bBPt-8zy z0!Vm1zJ>&I1!@lXR73;0)ALCLQ*512;1vneLrSCoha#H4B*l-=_|r#JvYfW?A^!bZ zIW~iR;RI$H%>l_Mwml)0LwKb5tW%JE3c%zOWeK-mpzbF|wSs#H=Aww&apVW*F}T}T zYUc2}puVs)Eq-!b>O-%>Su2tRBmdOpW7QWN_TW0@6GJ{$eYy1k`$7}^6y>tj!z8nZ z>*q)~2d#j-RO?&^#oz|%g8JP%2cMp?a%c*Wl+WYG-1@z)+Xu|Nq?`-`(+KIc+4udiOs5 z@Be=9_y0PYMn@Jtc#gl#`|iQq2EWR$2K^qK)mk6-u4x3* zH@4i+x?L_eC;ZkG@_peF zLc-dsbO-JMIF5HCo(*^aupbiaf8JnVZw#8l5g@g|`}7Ur@9zH(VC92fIE1Sv&sq zAn(yV*Uz|S_{XPR?GAr*`Tpb*KK_yL!W(4jSnq=49Pea_yn~ON_{Ukl2U{eGjqLb; z%=iy5{Nwqc`NYv-p4}8)z6C~L8g2S72MiFF4dB69KW$MV8$r*UD*k5C86t@RvJ)l&P{Q zc0_9RaIVn$*H?}nuCIX0={-=s=(AZH`fuK59`0L4Z}acJ`h+ih=-;ndLMU%bP(zcW1@osKW8D*p%wUj~4Drpu<;5h~P~L#Q)Y1ktH)^0!(2 z+d)-t`mu#S;k#gvU)OVmiOEraGK2~@t*H|&Q{;VB;rBt|H^Cq%a6Ht)4`~12!|CbI z89(N5(l9w^fS(C3fBJpHDe!HC^RJGSlV-QZ`+>B(?h7luw#rJYHg>FJOG_;A-d#Ub zkr@x1bYW(`yLtH8t>8VRP*Zhr4%F7o9NANjCgBj_W+*1Z8wmhIXbFguvvT}8T*joq z>^gV3!=P~sGfKN9IiEWa3_QQ`cP`;7-jjqYezy^>_#;d;u1@5Ng_x3K2>!_7?&3ZLIK^nblVykW1jBke?@UNQP`cX!tt z@fPy?_u&~9y%KFhUNI!?cwx9BiIFf!^lIJa$Sdx6LtZKJfC`Bc?c(sVKc-nk9SgmZ zGohkhil%8H_Ro(AHKwGHX%j2)j?1i`E|wpdcC%$HVhfRoNyPS5Vj+W9G`{FBy}cV! z;;u=`sF%i~#Tv$<8HX74I=u2ZZnqfIAlCxXUPq~A%vM<#1Uq($N_Iv0ys)@L}GVC$_tZ452dLf2D-%n0m*CshJZ zLQs%yC9yg7#4JoF66wsWKbPAnNzQlDnS3Ywc(3&!-Y%qz#a!|*-%88*%vrph(w_um z$XssqaZbv#6C&&vlv(pzErv*MJLsy8mg6|lX6&|Hp-9M)nZhtl+MH|p1qd>JcTpFaWQ5SChcxw zCa<4|*p2h@BwZ=zLP}RXZIpAWqVYzt$g@GaE}e<>QUo}7g}!_Wa98G;d;#Uo<6U`L z@8$-Lq&lsu(P>4JMWy#ht#@U$EU6t6$B}q1rYb#E({A$^B3j=Sl^$Ua-IA z9-(GA2KX<;5w^)Aj!>g4ALBhLQgH^Dtak)IwBE5esXE}!w5+(RrAVhN_DoOq*M}tbQ2%Mw88+KHXi&3K_PN)w;Y+*cW@jZjbUIN3uh2g(9i$l}F+|a|c zv7d#sKq@#s$4x2B7qJX>ae?gRqublN&`HEWnvl4WZ*Tyg*a#b|Z#dtPq>sYe+uC-T zgqp@UTcDZhkSe5I+S^B|q{WZU>*3-Y{NcFK-G}=YKp=}aDYWrIUy3SuWYd?xvk$5R zMN&g$DcY`pE~Q&U|H8N+ePcgKP<*?_INPWCD5+XfrJ5b^L7ZA*f{>&9dm4W!jZcEq z#t!nuEO~tNP~#)pa>l!Sf2JRH>`e9jBsSZgZf~Pvazde@QS|fZxQ?wQUczz+Vq8?j zDB7-FEysWpwL_8yUJ|24;?NjhWd9DHWU+taK%GI(wdw__4~_fk{Jv+5b$(?8Y2r7D zL!150`3Z95xJsEr$AKZ5#-KA`ixz)(j4Dc27~*x;jlm3n!+z7@T1O_7=AzWekkx9{Zxl>JR8LtE$SPi>iyNscXme&YcGXwx=>4+vEE>-#zy`-#Hh% zNXjdbixDm|SAZ9Ho8vg1k45H|mbrzc#jv7UZCP!|{hq82j7D3D%tg6qEP9XU_{$&0 zY)i12hWPe^hw;9Hq3#d*LsM59LqpPyh9ViN5A2Hz0cbzW!A`bE_fAJM+Fd{5H**1q zf6ROSElwkLIBctmtv8S z&D$LZY1lV)2&g4#rk|`YF9pDSf7x@$CCh<1Q5Rna3FFNb4=Y@hjJDb{HcVY=YU)$< zz^8{-R{|V*d({I8WqWep>=~*)Y5)3ako`4}m+(L{TkT$Bp!Y4cCxOg;PQ1Pr0PA}qal%rc7%9Vkt5Bt5z8u8{S0M_MFu#N+ zV3v+Drl^Df%*RwmgT#J&%?2gU^Nsw+|JDe zudKOc!w(u*4~{{elI>h11pA$oz{^}AHto-`%yif}mTdO;ME=-29Zv~&AN0eob^rJ= z5xbK}mZwg8(-Q<|PmDEW_fL`q7v+{>OGn{Zt||U}_Y!~pi-2FWudc`O8MShRkP7^GPo_maA*4=di98{sZ;zKsgtFGogLr&Bi_L5a zURy0}ujAYbnr)E{XpZ2EJVx^Z$q+*Oq(#V6>^~d9(oBVbZGnJk5kWHR@FFkp#Xxa_ z5UXf`f6ypB!9@U#4{Nek7Uwp_ft=J5LNeW{r>X~Xw`$gt)fZB&M0D4lWMOwg9=Lr6 zZn@iN%lq?+8@wwhXcAmaS9jmL0aY*FwYTs>U3``HY0k{Q|l;FI%3s1BcXB}sMZ0~=DWi5z@VnjM}~;JR~O2=LTcMm zgsqND$izZuma1b0_#Y*ajdOE{ky;eNnhoxu5Mi$Ewwc<9R5>1+;#>F5DB9ZwO$ zT$A(rD`HYJg%mw#rbMmT%|Yy$>kumD2SB>NA(X|GeLj+9q0HfVS3Bv+c;Zfe{xKdj z*X+VsYikKn*eZwuqn9XSJhsWm?VqMsj9rfILL9Ybubc($cI(}J?Rm1ghMNvudq=Nv zx9{K;aFI{{dFgn6DnE{*!sfM)}Ig@^88se=Yclnrd- zXj!O&Zm3pB>|4!l*=nNebO%q%2LsU?G^AYO+`K6e0mV!R?2rY?|~XDMtC*!k;G8n<=R9^j=N9h|Bpv>aGakx=!9z*c`=k37{7uyE d_wKKx9wizoCif;fOw?C=6kR&hiejfew`w{;Y`z~HzjwemzM=gE5@J~pZIwAci4)5=*M zD`zUzQn5+zrbRlikXwV~?*Sml5zubl6P``*{$IPmwUZ!9Wh*`rkoP(W{xZx7jxjf? zrJ|ZC*V<|RK19A90-7AB1NGT7XO_(Th*H^#PXy$>4w|=)ahh^_w5zprrM9l6YZ(@G z-L1Dr$aRc62kd}18SR6zCFg{Ahp|&DQcZjxOaja zocNDX2>uvtMB&En*2f5{N!CX^)Z87&kIhiYo|-J8H;G-N6`u&mdmw1#`Pn<1=NpZ!>Q-&Nq^k8wB?C3Sy~2O~HhJU%svN5W z*)(!(SvK`Uz|vh=P%2yTiGaMa+$YN=iUmZo|3@D%y^6tLxR9pI*HUm(vu zdDkV58hJLZN7&%gNayQ%iN=ikX^I?9^~|^zQ25|9S14(vYG%D&gm9TEJv>c5J>5&G z^ND?4kcX@4W~Pu9-vJ8K1KlSiUCfUpdcX4opi6shxYMJNJcLoSf`+H=Xw#0!jgzif z_!hoKK0Wr1R=}qe=PtPJD%nIrWu`?-5N$AYR6=xGU29nvn=PGIJGwGe5Jv&rByXmL zhzosy!#$#Gd?MMLF6Wc6{+)CCayBLXJ5Q4T;S5M#(HV{O{ai&}$+_nx9A!wKk00b; zpD0KlPXVOAJ2&b3ZcARm21;Jio+v#WlWb%}dSG`5g3~3B(pc=H#0UBgDc?Bm#%qr* zP)eh4<8^`3U8>}0h|(Ml_BQ6?9+C5lXh30$% zZ)yJ-cmT)k-QMucZ>N%qhhgE+VspNMcX7Lk7i6Rqbb#&;Y+#-M-c>CTs%zoAC&TC~ zSKJL2zW`l5ns3<0lB&f%oS^Ke#dt9}7n@hZ@$8z<>(IhIX@LmtFWgHIVg4RZT8K?kMh@X=zwiv3hK-o6$N?SWt0c*Cc#2-v!Z zH;WGw>3A&v7B9u4^m5a_XpTjhEnQ>PqP{SD==q;YZtiZ(L9Q0!S#>V9sTR2%P+vgL z8rB)(3S0)u*m{SeffpNTv1b*9hCS^$yA*_(tbDMQOfT*mePca!}TV~`fs&gU-wXYShdT6cD;VG>3TkY zD9DrLbmdAtQ#+%U%V*M+N@;fz{sXCQm5P(zxwD?L9yk@l8T$Eug5z~TScQM2{{dT) BvQPj3 literal 0 HcmV?d00001 diff --git a/cli/tests/resources/omop/public/FACT_RELATIONSHIP.parquet b/cli/tests/resources/omop/public/FACT_RELATIONSHIP.parquet new file mode 100644 index 0000000000000000000000000000000000000000..93b22f7b7bc26ebe5d0ddf0ead81048f4e396752 GIT binary patch literal 2167 zcmcgu%}*0S6rU|?TfVeFXWcbDR8uz`0NVly7&y!lO3{K)(b9&5Ed8K`e%Vj)VmN6G zXHR(JL_8Qwi1Fx&lYfGG;^^5J6Mb*DWeX`J80(~OX5PG=x4+--&1|~Bf*#6K8?^^u zAiacG9iO&)$UYb)RG>Eos0+eoYIcY`MnoQC0_kt?2qB&QCg@!pXD2pJe#<$iNg5?) znHs%&q+?15GJ&|5n<#J14(0%STs$i;?V1>ZanK;({+m zK8Wi)K2W*8;^K7PB5oFcfm}X!G~uGxMdxzVE`P&=bU7|g>IzV+Ru|jE#8)es1lxU1 z$Qv-g?V!%a8eml9F^)nR9*Quc*T7`i+yD9d8j%mW4aC<+8k#`UDy z6kVo7vD=D-o+e{6>B?GmKBq1#iLiIP9QHo0#(d-X2;f7g+Vc-^F9E3Ff$a|iU1^CPH2fHb!5dZ)H literal 0 HcmV?d00001 diff --git a/cli/tests/resources/omop/public/LOCATION.parquet b/cli/tests/resources/omop/public/LOCATION.parquet new file mode 100644 index 0000000000000000000000000000000000000000..49f8f3064e909be0ba3f59f5043d0156e67c9602 GIT binary patch literal 1865 zcmcIlPfrs;6rbHLrNzb&!fbZU9(rLyg5Yk;pK9V^wv@jGX%z@uNZHm>SZHkv4UHec z58%NA@kERtz=H?k$+Mrplkwoe58%P~W=p#zsWDV1%*?*uy!rj!zW2&*@T^4@I!s5? zu#pi$LWV>LkzrCoN2q#V90zI8&=f>91WmQpMOX`Kh5-jv5J{?sFl`)24*dQ2Tr@_V`5x+}EIbS~pxA#~+;2l2k#=!o?aaR$Vy}jJ>F6(Kh!(6l z2ZiGtjezKw+|{ClMgVEET4N+_V7nnI3hW0kw|=|z#(t>H6lW)2WtoB{50ZqT$b??Fu+*H#iMSZ6d1Oh6`+GF3b)1z z?mOGrBjv^iu@Dh8~*rGet)626LZai$khBM`kNSE0~|A(0u@PVrxNmo=;FfB?$V0u z#Mhs%39k(D3O-mMt|GqD66PJ5I-;qha1tb*KRR$;gK5|HW9tX`MfYsqeOTEQ^=M#y zfX6EuI(hNn9sZbT(MqoR;IF$k*ILQpA)gJjTyAO%zgdI+=#y|OZ>hqEo#*p>hhnC;ikae&nJa@Mf#VcfNXjqrmfM;m zj<#b}{&&^{DmpjCp9sC0-15OQ+g%4@gD(yI^aSRqm&^`sDqG3&?PbdRWIQrDJ`zvH zQ)Al(W85T%SDY*`g>X52)s+Hr%=X0C9DAN~?lwyI3bG0DZNGUK64 z81Wb!UYGjycuYlRJ3GoMt>3VEH#7BYf#G-IaC~HRC^hBsSxG)GsVDQ*Pc87-P)1N0 z9LwqR4T;~B{Cu_Wdd%_3>(%V!MmLkB+x=@x*Rqlo>)6NV2Tez7G)3F^18nF& zYgy{{T9){9EtB`JW&d-#*@ZQ3*6@s*{pfNvd!@heo`44 zUEovRd->2`!aqOr$@Sxp{l0eywX42hGw_l#^Bf&YMdQQM`96^+pUqQm&Qq^Z!VOyl zp26h?ZxvtT=ko=0jzp-Pu^!O&#~F)&f_Q!i)CfX|%egxuX zh^DQUecSZcvEN~#$p7cBLXx$Lz1!Y?6YTci^gkh45BK%G(}PhjZ#c3Q_r2e`^7rqa z$M7qmeMfP>_{WK>|Gj9TxNCVsM%-4R0E!grlnk89@fJ8JTXY=E1+3V))af*D-u_J% zD765JD+?XmoNnUMY$(PClR6w%P4e3&^;n7ek=>!>urZ6}obXt=&CgM)=xwvDZm&pb z370cMGt*l|d=lOUhfoU3^-!b`Q!mR#8W&~yKDK8#zAdXd4@&Z3Nqv*6Cpj`|=*+T^ z(`76> z7B~CecJGnzOA^zdrA#Yw6-yOXQh=+}3tW7>B?A}ik-8|VNL;rs$Q7vC8E-MTWH0Ti zmOB?K(6g38TBYg=D$j&QZ)w)QJ$;L|1@(=G9eiH8d>2ea665i0kZaI$%FXsbr3u3Ul33SOoOe#|rD% z*l2nT~poi{0sTHpQ0}SC}k^pFO<)SqkDVLsBnQsP8XD zIEGFN$KY~@gHwecUWx-3?(=zxk%mV@@Tx!&a0A-6ml<_bI1)+q$G0VhCPf8plEdn; zW$M0V4#?!Ji!Wg3#3P2CdAgl}!Ty11Lm#=+Ke=W#bl0(nT&JC#&LxOW<;#ue6f=z| zW||}0T^Sr+ENsK}!Ym7JuaZm!Oxo+Un-!*BUtwfo;l$u5QZW>bj*pDQvB^)Jk4{VS zIV?cS)mN7z8?)NDSCsx$?cla_S1Xwd`{>a42)q%sKaJV>YV~OKte6FP*w1aE2v)pC zcqEe)eyYDF>6h?ON%-~G_-U=#U01ucu|~Y6!f|tt0qLW)u7dQqB2{AYgCSbN@&a~7 z>r-CA{!1NK&nDl>XX>Z5hWo{$sAu?nU6MbP)P1$;ff`4a=6eP2A0ESciM~qAFi$o8 zOOJZQGaGDWcZydsC4V$J5Kl&UIV!x;lzqt8Um7Q9ouB1|s1MFKY*pI)382W$P@nVo zQCaGLzSu!uS}=>8_^gdypOI_(WPBcEvv-K}{2WWEK9ap6=~|5SLFY62cuL{S_F>!x z2WBniO6I4pdt%belF1y+k<#?A94d2^-LcQv_p&+p;xzZ)F6Q#I|79PazoaAY9G^zN z#8cZGpPAITJHOn+kwm?RQ^p*h1JwJrwcqz$R=p1!r+j&_98#_FoyGXiy)uVl)4IDi z_N>;oQRV0ZHFQ0tt=BZI3q;rBnl`EV`n48K>(`q4#XFx*(>m#?Nz>Zt4u0w`7>ulf zm=>dH&G^%rCn2t-Xj)4b#I()C*ORG^Nr=M&Ec%HLkddYa(i@`>h~grjv_#;kni^ue zHVgkc=;gGlm_(v&;eo!kU?Nn%VImgl*clEsdU|_y*M%XmD}eY`?OcTqOkCe0lr{-f z;*qpzK6>=k^(7OvxpZiqn+T;0RtA~|6{5U5Yb*Pc5*veFhFj(Xsmr*UU5bVrlH@m^1GN2I4q>ztC; zkKnt?g@D|O!OiD(Y9`puh^7)I$sl0e3y=wpPb0RtCjF=txF(2)KPFO z+_Mrj+PyN`ikR=huc_hgcr}fR7Sg^Z9lz{)-AtCU>owfkQX39+5BK$mVd{4@bnkTE zLj{+Km39nLha5@)Y_r0h{B<`L5njcLD>glnSf{ zxTmjn^t`UGDYrgqwHf+$_IXzB42OK^lLTs|X?;5WbpD$=)4VTTPf@a#QBPuN0j_({ zv*WrBO&h26Lm($s8fsbqBm6~ci2>2fy65Y2`>WB?Hp5@kyZ{sULf@pmo(esB{fkGY zln+Nno*a*lRY!-1t0N;LLzCqb_=#X}C{{CPB$X4tu8eQTe$$#XjuC2PVq=y>)vn@xm zHT_F%_E0w1!w6&B!x$};Fv1?j=)=fE7~3d$>cbd$Dy4){3e8JH2xF8H%G&Qc_ud@& z`Z^>u&Lvm(eCN;a{J!t|oj>scPmWQQ&eBAJ`YG`fGI&)Xgt*}Cf>(vt4R1HRhv4D`B$7*D_!-9(5R}s#ysP5j-QX3)IeI-C${S&0Sw_iWH zm2Z^l?Oe0GwYgF*QkUk}K2x;23ac^Co6Tj zP_N+Hm+-e=a{aG0N#f_vl8X;7l9zwX(Xjv>=;VyObQSyH1kwV|N5>7$UGB*nzlFrH zn};N8tux4lt<6HI*0h5CH$}Upuzz;5uifB+#I()e=)$gW|0hAnT%3^9j8KRz+}NtC z*oMH;+7<2vFA)E;R}$~-BwoxlOF|z;)NdSS-=IPymJQjF@+6HVs7FM7;AhpH5B~qv zk#6#iQjWcK3#9(be|+hA&rSZm;`tWq_o&8WwdfwJFHGuCyW6dYU4(t|g8)@c(f8Ti zYL=^pYxfoHYlYqNu`iAUbO+phW^;7aLwyPA60WMgv3EpP{8Z}{6xQl0{$lK;1@sRO z7=lv#gx&6yDS4rjdvm*1TCoZ0H;VQLg?-+`ZuUTSB&KZ!N6MqPCXDr~!``E)4=%Y6 z4HGg6gy0#Ugp6qPWQPm5p@ISlMh<^QF3w*&Ri@qp-Vv_P!r-VW5vsTBUvGadZqesswPyjR{qCl=`I)=dtpT zF2ara*87%-wR#&lB z$4y>jUs_P%9W=A=RtfvJF4KSfQKW2Xe^=4|sIWUH*k4XS+9epKU0Tk3cPSP5K!|Kg z#ahW#T2Sg#>h%{S)t=U7x#_IV8zwI*YT|52Md4(KW8>M#; zFFUbPpuK-cAgApL5@6+;g@L`4ScIh^heZctQ>JC9WO|j6fgg zg(Mz#77*%%9`=ouP~WavV_O>{3EZXxrqc#vwAk0P)qn5mF=p9U%bn9bURnmw-c*_h zn!i8LfA65~vj=kVJotOzL0;gV_buu<-F%+oUF3UTFAn%Nzz*&NKj1@`J+pI^J+HIx zKn^VJ!(*DCd9>^PK0^8fx{oK5uP^_|IKK32-XK0<9^;%Z!i&#OIKRRpYkZpXH6C3P zb&&|?uVE?5c^toBTVJrov8DjSU=Vbh$8OO@7|s{4+rlDrk6yqT*7+3YQ)oj2*xOLx zaWTmSEMfzHedj_ji00HZ^cGNPU{O2~EJnui)yio4+Ho2Nx z7NN25IDZ32Mz$>cND^({z#qXM3!G>0XJ7nt+4cNvs)BuZGzLjHbJ@Z(`Ci7R4!k9_ zi`B|bE}dM>hvqlH@9pgJ1o~df@@U2!Z}N*-d@cV)kUQ`e1m5yQtvDUt0N*#V>Bwk0 zRJqLOV__ryJYpny1LFzZr=rgJKxssb_@~0_g)ti*7x}>H47}+j3(q;f9Ov4Fw^N** ztL3W&j0aG_d_?2Hwwd3z5gOIDD_(|w)#*lYIklQe&#&e;Q_V~@wasIBh&Rp==k0|b z27{=K#kfdv0zn=v1hx%CIowL0OHLGK05cSBWkRVd5I>CnxXH(i zKj*l#V00wvtiLyLCc8eeGt;A4)Z(&e#)SmI%{*FnFa)XuRa~tN*lBYQ9pa)n3Wf>swa_+VE#b_zUU4*txT# VXGi<*!FBlw{}JvZWC8xt`yDIKP7eS8 literal 0 HcmV?d00001 diff --git a/cli/tests/resources/omop/public/OBSERVATION_PERIOD.parquet b/cli/tests/resources/omop/public/OBSERVATION_PERIOD.parquet new file mode 100644 index 0000000000000000000000000000000000000000..f0eddab8431a99c6075d4aa9da9bf803fae9533b GIT binary patch literal 2183 zcmcguJ#X4j6n!{=NXigJeU`0Ki6vHtd=N*drBa7$52xV(VaEcbJNNbwRnh%$RLQXxDYmM}^lkHoYyPl!uul=){>g*Pj0tJN)c z>di*Uv0C+}S*n{{k|p_*M?Upv5iM{Xv$}xN7?()REDk8^e@gv5}o$f zNi?B~mm(3m)tKnd)2`?T6GU4Lb6oP*fcB9Z!Q~vg!_AUc-c6EBsN$tagl;t^`9si^ zET8vjr|(#$YO_(boX+L0IK8L+nnATL-5B{KwW=Luioadmw~oqu277v5x?+0_=G=Z9 z)YmWnd@y64?@00NBPq9nS%JVALQ1P=k0)Ug0eV3nntXLBUWz+ zsu6=uS-OY87@2>KH+h2CdVHPc2c#mU5{v?cUNOJnRP^kgnX-GuTSs%vVv3`Vs@Ee{nCG3=2Z;!3cV%2dL%dJ+k7w+O)6<_}0 XmDQ!lQbgSiF*f>Rs*kZP`~>+6((J0K literal 0 HcmV?d00001 diff --git a/cli/tests/resources/omop/public/PERSON.parquet b/cli/tests/resources/omop/public/PERSON.parquet new file mode 100644 index 0000000000000000000000000000000000000000..ae693885e0fb0302ecc507e0848168ee9591bf73 GIT binary patch literal 5420 zcmcgw-D?|15Z{v(S$389BR=(%Dv6uMiy=-V%W)E=33yMk{1GL#RYzYGj4a9Whh(X; zEX95*eJCM>68aPGVX;Qd=QiT+6S{Eu}=0-C}!Lu zBBBW)bixNU{f6LT2HH0o|g#8Y4<2Gxj4!y#-3zdKHX9i;4c-qC@P; zq*qhv;#@MnnoJi;8q~Xj{k}kNw$NWp09MyjqqiYgx9HYLl?V|8>!o`mLhAq-zc&Iv z2>JEXkve*x}i*-)@ zEYQ!}=oVy>!jVDWHgg6kY2XwQZDJ>85aC&V&H_*@Jv>tg{odvjiKEX#9Fi@S*f3%^ zXTL~q+b+Qj2aJM0HB;T<*=>^ECHF4`zmuVIdY{dRpI`-D{NdX#8dR568-ea9W4IIQ zJY8B#)khsp(X5955D)uYDp5+8mU3w={#yclagcs>5UHxu*ThN#s(4Wb(N!oyboh|i zjm#G{Bm*$LafBbgR>e1U9O8t7r>~CT`4Z1A}O>rrJ>| zneho;@K2=$g6(Rz5c<<$z1lm9iDbIA*6#@Ry8`{An?CM_T1R~iweB&@2X-B1Pr9_Y zx|CchZ8XOEP^7oS9b+{tq5`bAC)UoS#Bl@r-@rP|QqXHNQ|b2ltLIaXy+-K82@ym^d1Kibsd?iJFX zy*k^H5m3$UsZ`Wo$LZ7KJI>M_#N_5%IzwH&vK#XA;|copiDvvbUUUu<$+DJQOy?5f zF|giesyeqQ?fOFk5!+#$al=()H5(SJTQJbDf53qTL^Z2*Ur+FMmvaD$43;|HZ(KGF z{fM!4DN~R(Tsf4k>LOJnhLVA;5VST*?LHE8A}Zc*j5|+#tshY}%?o9EeU#M6c8kO5 zc+s&3r^h`#?a}=@e$jn96a9~458by@$g3VK_V6%3_jm)}qYan7(jH28V_T%VvD49g z*q!LBZ#een9d-+~+Mjgx5cuXG*;ptv6}hNvKhmRi7hb2BpD{KAH(pm5t1vmk0*qyt zFT-WB%-B_Y^D#DopU~Dmp>+WzZkWago(@t9gI!mgG2GhT+=QWG-Nr%g)8{PiR#6r$y{JL9`*z} z{`w9+=6@V0?yETxFMQaU0)t4r&r7ZHI!f zQN%tjXOpYf%JWtDG(Y3h=UuMz%U+iYV=$_a2yA16KluZd$jhfyj|JTEXs9sHb2Wnn2&oA5hJ)~`)*lF8pQ*4FELG@{Wkr8&J@EPjy~9zD zHy8Cx7LuzWjsf!taBx>68vdE9Jg7WgRr%&eR>ua*^Dx(_$3Lao_pymhc;)%#SNP+q z^!PG2!W;(vHjFxA=pXnQgMPdw*o|#GN`2PlKdRR2(->FiDr@uA`~?tD{25%oAayY~ ziu3rQ*GDa0q*AWLqm!GEH(G4~H5J4QdU&d#0wurM<;z6sEhLKXt*1+;lZC?RM6sB! j^p@e*NBC0OJ2-U4eZ~#POE|{9{69GgzdKCB@1*|%$Ibyq@dcT+AqE>e#>R2f zLl0FHMO775QI$j01B!Zxq8_RqqNv9z%B?;1+Dnu}#3_fWs-mjz&3Zk%W=Yxxvcj`F zf8YDw_ukuCG0&r;)Jc8xScJML=_91~BL^X*1%9pYbHa}jIzW|SnD3?SwB4&Xaa3{Q zIyZUG)PsSJW4P7jXlZ>+*aL@`ws=mF_Fj*2q*y9wnRKO;St@8+rBupnXqlyKnzp!I z?z;~64-PiNTrUFm3-_-*vx!4YxGC5mOTsWn}mJml&lXG zGo{VKhQV{&0aV$Kt?YRVFqIG}gY1!)Gu>7u$nQI6$(451MqeCNoH&obvS(p>hp@le zCDQ?^uT}tQi)Ab6>+S5|oF-ITx)U^HzAa&>+6WXRRw>HfP5UI*(=Hf%LlK+^VcUI@kx9$UMtZ4{A{X3A z_T(^obl3}7Kz^d^7KMOFEDVV3y|O+(D_+`BPj1)cN@}B=E!RY)q9WH7_Vmb} ziu{MTfAyBh<*n>ywrq1wpvB)j?7n9|EdDX*rCoK$Oo87lR7!9%ZKZM*y>ij$gF*Jq z;GRbJFYw>KEmatl$yn!PTan)%W!H`#WCA?&Nu6jT8Es{{;bS}R>?M=udCw`Lv~Oyc zGWitsLT(JT?D<)s&&uOh@7o@7`c%_VGSn5xK@m_V*f70D@D+j;Jv0#c;;?rZ%S7M4 z_sdV8H(?lW2D1rM{utA?X6%ZY&<-HkC@tMyzfB1c+64}UdFR%A2;H;D^`8oL9NJ%BMJA z<-t`krmCEu!>1tUVZ1}%xI^!K94Wvy7=&eWRah3(U^~AcmMuU(6h;dxd=eh3upK6# zUxNo8Mk6>>Z_kI$`2A;~H#r5X0ZNb%kJ#nvMQqB;foL(A&*ik-_|@gHSZy&7TS@1q zHq&vO4^Ew}ijyi})8XB~w136^kcQrBb2l+k*duWeXd=(Xo>wCr3Ox@U{6b|H$)BLgwK=jsF0!oDvZL literal 0 HcmV?d00001 diff --git a/cli/tests/resources/omop/public/SPECIMEN.parquet b/cli/tests/resources/omop/public/SPECIMEN.parquet new file mode 100644 index 0000000000000000000000000000000000000000..9a7dec7397c4d9ad19310f1b6e2418806d4791ad GIT binary patch literal 4873 zcmcgwO>7%Q6dt>q)Jc<&q*-?zmMRitghX4%4uJ;I!}vEzT@#1If0t;p>o`sw$Box9 z$*JN{aR8|lgg9{E&_fR#P{k1;1U;Y%RXrgg^@I>es6ql&oOy4&_O92Piqa;lcz55M zneTn?Z)TN6wiFNraabHnz=Mnu;*rPa3CY4wB;u&(J;R54#a>wwP%8;I?j}Jl929eeJ071?KrC26}3>+TXdUA|HMOoI}UVBEE}>T-Hvz$ zt~$jNwUU5$1T-ZpRfy+<6DC!uO;z8k7FKP1o8f)fPw(^tUlY&<@5pyTe5utw z;Qn79ebYn8hwh*C?|Zt#>}icip4MD2;-Izq0%3nNB#VO4Wg~50u*-DlH(ta)xV-mW z^w&Xpb5NEX5D&29=oIu~0_qsI2=rk{95r`_*91@=m#|CVjouoO#eRdC=Sp3v=p}uR z6I%A>>BC`qZ&-#Uj%GS?4S3!0LR*k)ykjy<3EtZ64Y~g6q?zmI>Xnl2oZ>s9^s`Y& zv58|tJ!G2idQF;=Qc?8nt&+N0E9nQ6~Xz#X6KbD_`y?dukd*hu|;%NBBY5M4egADr~&Jd}$ zrfw9r6mcB9o^;#J>B)ga8S>6YtVd&0W1d7!*=f^njvTnh!f6ghySJ^$vBdi$@)>MK zBKE_S_WscXv^TP0rZXbuR^A*wxJ-#S05+V9W|AaiWPvlfQ*0f5OhSxlUco2(9vsq# z5j~s_nM3*bIOLCoqd4G?^x*yfXXzoHfAk2>EP52@41G372Yvd}fIhV*_c3$4I%7C|}Dfa&3Rw6M&gFg1npd0e!Pw z4Ue0iOV^d`k^=@~dy4=vvpMVcBi(3>lMEw4?g&RiSHa(XHJ;v7GJ*11d3JwoI=z<* zrq{KtSWU~|IL=RD+!#wPxA4Nwc$Z;P=(3VcmUHj}AE&_IfEt+HTno-@=Ch%xY_N=z z*!<$_&3LgW&s4l6r{VRkek{a`W?|WoqY>_Z$QDm592R;<%f#whygXIlo`l1O{V==K zUVqU!eyHup$7P6L8L;#`$0|ZyvN*4AB#l?m$R|>XYRNPt3+o4@mcKQeX7M+NvN|0q z=YoKnnOWE7%8h*5<~vS`?E?}o_on}`WkZe{`RAm#D>$8nu+=*F+v2&|iMJ5x#OsPD zuct;nyRZ-Sv#w;&UgQ!ivg9`sHRP7RyuK{{@>aC^i#X8eK(@%hBu_2UXS`mp*!eib zS;cts;I*Ah6}i*)evF`YF4@RulGU|Lx}FU#ZD{co+>Zc1htH0SNE20M=SsbxpH-{X ov&zoSc4J}}{y9?Gu1o}`&rO}1@;!i0xBv1-hE5Q&2>%587s*=0S^xk5 literal 0 HcmV?d00001 diff --git a/cli/tests/resources/omop/public/VISIT_DETAIL.parquet b/cli/tests/resources/omop/public/VISIT_DETAIL.parquet new file mode 100644 index 0000000000000000000000000000000000000000..da20bd662a52d035bd75a55b9ffa10a3c043faaf GIT binary patch literal 3228 zcmbtX%Wvak5cg(xyA6$mSa3+B2=!7+2<;|K6|q6u!#K}&OSds>UYE$VomU;l}eEioXqltoOfzIm)q`y@o+lwng8=J!2ITs7t`Cm zn0%Fbvl9UE?XCwAHnq`sX!y(g@1p?Yo3LjbQRI=B(w0O_ZQhxD#ZW~eCB5ke|U(R~;3!NmW_aIu}S($xks zAH%zK5ZZW^*6q%($3f&&r;9}h>vg|)B2dsAA3=~+jUI5P335TZi{f+F$@Q|TY(jIP z*mGa)?WN7-wD&hu$lhF+bS}!p6Ynu1II3PLgX*-AuD0b`1w_{#3XEyh zojZ6Kj1R9!r^;fNFw|l;E!WRX{Jd<|rLJ82l3_yO^K3Htj0{DL2H?b<7jlerj!4=0 zpaF=N^s*4M->gaI#XxPV1N~aAOYLTA7g#d zWD_4>JtSQ%70rtx<1}{s128_hF-Z%NL{XyBiY#@@^<}9nNnOOj__Bw};7ljo;bBZg zg;azO%lAB#^7Th^VIc537 zH3x5nFG*n6&0IIvgJ8F$shw67c9N!PyR2*CmNc1A(%dBdNSc_Y ztu6a85JW^}zht0@43Q}!GT4x@AAX>S_{$$dWQYuZi0sGwz#jvL?|W{Nlib|y1D0NT zd++<6bDsOW&-?Cb}YL^dNCJ3N4uMOI{22pA!2TmRpW6JiBQf5NkwF}5leY)8hz zYGAAq`kqBFwrh-21t422SLvQJ0b}|}8~w)&gawig*O0JX?haubKv;4m#tDfAIoU4R zQ7Cr(P+xMf?WQ+fE&#cELg70G6^C9Sl(68f zCKODMTYKt+Qs3mY|0Tz8`!&8XX(zM$*xW!)7XA9*ZV%Y*{h8tC1cj%2rT>Yqu z2BF&p>H!Q^)289Z>y`#BmI=*nn3bAq(+NOi8-#F&a_l1^S2YHuF&Se0te`YD*XOE& z;*gv!=e3re2wQo0Yvo~!Zt>Ztbw?Dsr6+}!@9wk}J2Gw2F1i)F-h`axC_&w!J7fI^1|2%8oF`!f@ktgt0Z|%d%0%vdo|5KA(@V zqxj}$Y#g6pSD#>c(B*+)YzW;)!qCkZhT$xT_9S-qAH^?3Spb@{V2|+cE#Sxie}x8k zof>qz0f&W8aAydeHEMu|e1ef;$~#$zPfsi*Vv&p*9L}V?!vm@OWNvyeF*iFF%^_0^ z?RX7u+!znx4e;XL;gz^Ix~7f}7Zccr`2y~Y+s$8Jz~BCG#rF9k+>5L+(CU@Mmv}5v zPEJQW$TP7tu{s+a-k2SXt^;;P&5tgtu@lfMjN^Fxo_pdux;PuldDLKZaoU?$pY;x8 zn)!~5H}YjC+^d|wkIf-7%uWMep5M54bUgtafpc!!n_I*_kgF%%?q~4D$CDn|sMrBN zC+*{*^H=P|7!LETC8j4fX1zn1WQ_QO9y?z-s~U z7827wPnDZG^|V$mEWqPgtJ>iL|L6pQ)YT8~HI{(-$pJ=mB{4d@7>8KZV6L2i5pz>e zG#E}+%Lmpp3#gMoCV=$>md~$|)a!Q)+G>0gu_4G!WKIo2e#c5MlIzEgRq-qQxaf8K zVA4VV3ctZg57apDM4YH685@AQ#w*KU!Up;L)EowU{o^{mdVRS*Ree)~k-5|o)M#+j zgF3+chg8S$S-D1P@mK1H*G65w8t|Iy(eJ77spKDvK8))L#F4^yc=BsS6gHl(0nA5L zAMfx&9OhCgAH})K>kRNf(kxV;wmxz4yx!RJMCmOiOE0aY%Lh`$;(=tTR9Nd>g}+$m d3roENg9kkaJ None: + """Create instance of OMOP file helper.""" + self.export_dir = root_dir / "exports" + + @staticmethod + def _get_slugs( + project_name: str, extract_datetime: datetime.datetime + ) -> tuple[str, str]: + """Convert project name and datetime to slugs for writing to filesystem.""" + project_slug = slugify.slugify(project_name) + extract_time_slug = slugify.slugify(extract_datetime.isoformat()) + return project_slug, extract_time_slug + + def copy_to_exports( + self, + omop_dir: pathlib.Path, + project_name: str, + extract_datetime: datetime.datetime, + ) -> str: + """ + Copy public omop directory as the latest extract for the project. + + Creates directories if they don't already exist. + :param omop_dir: parent path for omop export, with a "public" subdirectory + :param project_name: name of the project + :param extract_datetime: datetime that the OMOP ES extract was run + :raises FileNotFoundError: if there is no public subdirectory in `omop_dir` + :returns str: the project slug, so this can be registered for export to the DSH + """ + public_input = omop_dir / "public" + if not public_input.exists(): + msg = f"Could not find public directory in input {omop_dir}" + raise FileNotFoundError(msg) + + # Make directory for exports if they don't exist + project_slug, extract_time_slug = self._get_slugs( + project_name, extract_datetime + ) + export_base = self.export_dir / project_slug + public_output = OmopExtract._mkdir( + export_base / "all_extracts" / "omop" / extract_time_slug / "public" + ) + + # Copy extract files, overwriting if it exists + shutil.copytree(public_input, public_output, dirs_exist_ok=True) + # Make the latest export dir if it doesn't exist + latest_parent_dir = self._mkdir(export_base / "latest" / "omop") + # Symlink this extract to the latest directory + latest_public = latest_parent_dir / "public" + if latest_public.exists(): + latest_public.unlink() + + latest_public.symlink_to(public_output, target_is_directory=True) + return project_slug + + @staticmethod + def _mkdir(directory: pathlib.Path) -> pathlib.Path: + directory.mkdir(parents=True, exist_ok=True) + return directory From 5b00e66e8d7b2529467c40bad89feb1eb39777c8 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Wed, 13 Dec 2023 11:57:48 +0000 Subject: [PATCH 2/5] Document OMOP ES file exporting --- pixl_core/README.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pixl_core/README.md b/pixl_core/README.md index fced850ed..4317ab1d4 100644 --- a/pixl_core/README.md +++ b/pixl_core/README.md @@ -51,3 +51,32 @@ The client of choice for RabbitMQ at this point in time is [pika](https://pika.r asynchronous way of transferring messages. The former is geared towards high data throughput whereas the latter is geared towards stability. The asynchronous mode of transferring messages is a lot more complex as it is based on the [asyncio event loop](https://docs.python.org/3/library/asyncio-eventloop.html). + + +### OMOP ES files + +Public parquet exports from OMOP ES that should be transferred outside the hospital are copied to the `exports` directory at the repository base. + +Within this directory each project has a directory, with all extracts run stored in `all_extracts` and the `latest` directory +contains a symlink to the most extract. This symlinking means that during the export stage it is clear which export should be sent. + +``` +└── project-1 + ├── all_extracts + │ └── omop + │ ├── 2020-06-10t18-00-00 + │ │ └── public + │ └── 2020-07-10t18-00-00 + │ └── public + └── latest + └── omop + └── public -> ../../../ all_extracts / omop / 2020-07-10t18-00-00 / public +└── project-2 + ├── all_extracts + │ └── omop + │ └── 2023-12-13t16-22-40 + │ └── public + └── latest + └── omop + └── public -> ../../../ all_extracts / omop / 2023-12-13t16-22-40 / public +``` \ No newline at end of file From 86c1bcb66a946cdf8ed1b52ffc44ee95034b4d9b Mon Sep 17 00:00:00 2001 From: Milan Malfait Date: Wed, 13 Dec 2023 12:03:59 +0000 Subject: [PATCH 3/5] Add copyright header --- cli/tests/test_copy_omop.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cli/tests/test_copy_omop.py b/cli/tests/test_copy_omop.py index 1959e3fae..bc064b04f 100644 --- a/cli/tests/test_copy_omop.py +++ b/cli/tests/test_copy_omop.py @@ -1,3 +1,16 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Test copying of OMOP ES data for later export.""" import datetime From 1084b530c34afb440b6d8ff9bf10095502faa5bf Mon Sep 17 00:00:00 2001 From: Milan Malfait Date: Wed, 13 Dec 2023 12:13:04 +0000 Subject: [PATCH 4/5] Add more copyright headers --- cli/tests/conftest.py | 14 ++++++++++++++ pixl_core/src/core/omop.py | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py index 3da3052a7..e809c0677 100644 --- a/cli/tests/conftest.py +++ b/cli/tests/conftest.py @@ -1,3 +1,17 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """CLI testing fixtures.""" import pathlib diff --git a/pixl_core/src/core/omop.py b/pixl_core/src/core/omop.py index 76a4a915e..e49e80d99 100644 --- a/pixl_core/src/core/omop.py +++ b/pixl_core/src/core/omop.py @@ -1,3 +1,17 @@ +# Copyright (c) University College London Hospitals NHS Foundation Trust +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """Processing of OMOP parquet files.""" import datetime import pathlib From 8671fbd6b31bc0b33ac097c923db57696cdfebb0 Mon Sep 17 00:00:00 2001 From: Stef Piatek Date: Wed, 13 Dec 2023 12:22:03 +0000 Subject: [PATCH 5/5] Update documentation Co-authored-by: Milan Malfait --- cli/tests/conftest.py | 2 +- pixl_core/README.md | 2 +- pixl_core/src/core/omop.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/tests/conftest.py b/cli/tests/conftest.py index e809c0677..254115b38 100644 --- a/cli/tests/conftest.py +++ b/cli/tests/conftest.py @@ -21,7 +21,7 @@ @pytest.fixture() def omop_files(tmp_path_factory: pytest.TempPathFactory) -> OmopExtract: - """Create an OmopFiles instance using a temporary directory""" + """Create an OmopExtract instance using a temporary directory""" export_dir = tmp_path_factory.mktemp("repo_base") return OmopExtract(export_dir) diff --git a/pixl_core/README.md b/pixl_core/README.md index 4317ab1d4..20356cbe0 100644 --- a/pixl_core/README.md +++ b/pixl_core/README.md @@ -58,7 +58,7 @@ The asynchronous mode of transferring messages is a lot more complex as it is ba Public parquet exports from OMOP ES that should be transferred outside the hospital are copied to the `exports` directory at the repository base. Within this directory each project has a directory, with all extracts run stored in `all_extracts` and the `latest` directory -contains a symlink to the most extract. This symlinking means that during the export stage it is clear which export should be sent. +contains a symlink to the most recent extract. This symlinking means that during the export stage it is clear which export should be sent. ``` └── project-1 diff --git a/pixl_core/src/core/omop.py b/pixl_core/src/core/omop.py index e49e80d99..dc09cb327 100644 --- a/pixl_core/src/core/omop.py +++ b/pixl_core/src/core/omop.py @@ -26,7 +26,7 @@ class OmopExtract: """Processing Omop extracts on the filesystem.""" def __init__(self, root_dir: pathlib.Path = root_from_install) -> None: - """Create instance of OMOP file helper.""" + """Create instance of OMOPExtract helper.""" self.export_dir = root_dir / "exports" @staticmethod