From ac88f98f58324dcf0c7acd403b8b6199742a3a9e Mon Sep 17 00:00:00 2001 From: Mareh Date: Wed, 10 Jun 2026 22:57:36 +0200 Subject: [PATCH] Finish week 6 assignment --- AI_ASSIST.md | 65 +++++++++++++++++++++++++++-- Dockerfile | 8 ++-- README.md | 6 +++ docs/execution_history.png | Bin 0 -> 14083 bytes docs/execution_history.txt | 3 ++ requirements.txt | 2 + src/pipeline.py | 83 +++++++++++++++++++++++++++++++++---- 7 files changed, 151 insertions(+), 16 deletions(-) create mode 100644 docs/execution_history.png create mode 100644 docs/execution_history.txt diff --git a/AI_ASSIST.md b/AI_ASSIST.md index 9b31b94..933cae4 100644 --- a/AI_ASSIST.md +++ b/AI_ASSIST.md @@ -8,18 +8,75 @@ -TODO: paste your prompt here. +python pipeline.py +POSTGRES_URL is not set. +Retrieve it from Key Vault using the CLI, then export it before running: + +    export POSTGRES_URL="$(az keyvault secret show --vault-name kv-hyf-data --name postgres-url --query value -o tsv)" + +Traceback (most recent call last): +  File "C:\Users\Beheerder\c55-data-week-6\src\pipeline.py", line 170, in +    run() +  File "C:\Users\Beheerder\c55-data-week-6\src\pipeline.py", line 152, in run +    config = get_config() +             ^^^^^^^^^^^^ +  File "C:\Users\Beheerder\c55-data-week-6\src\pipeline.py", line 53, in get_config +    raise RuntimeError("missing POSTGRES_URL") +RuntimeError: missing POSTGRES_URL +((.venv) )  +why i got this error i already add the url to .env ## The code or suggestion it returned -```text -TODO: paste the AI output here. +```You got this error because even though you saved your URL inside the `.env` file, your terminal hasn't actually read that file into its active memory yet. + +Python's `os.environ.get()` function doesn't know how to read a `.env` file automatically—it can only look at variables that are actively loaded into your terminal session's environment memory. + +Since you are using **Git Bash**, you need to run the specific commands that read your `.env` file and export those secrets into your terminal's memory. + +--- + +### How to fix it (Load your variables) + +Make sure you are in your project folder, your `(.venv)` is active, and run this exact command line in your Git Bash terminal: + +```bash +set -a && source .env && set +a + +``` + +### What this just did: + +* **`source .env`** told Git Bash to open your `.env` file and read your lines. +* **`set -a`** and **`set +a`** told your terminal to automatically export those lines into its active memory space so that any Python script run afterward can see them. + +--- + +### Test it again! + +To make sure your terminal successfully loaded the variables before running your script, you can print them to the screen using `echo`: + +```bash +echo $POSTGRES_URL + +``` + +*(If you see your connection string print out in the terminal, it worked perfectly!)* + +Now, try running your pipeline module again: + +```bash +python -m src.pipeline + +``` + +It should pass right through the `get_config()` check and start executing your cloud pipeline steps! ``` ## What I changed after reviewing it -TODO: describe your review here. +I got another error with the connection string. I discovered that I had entered the URLs for POSTGRES_URL and AZURE_STORAGE_CONNECTION_STRING in my .env file without quotation marks. The AI explained that using double quotation marks ("") is important because it ensures the strings are correctly passed to Python, preventing issues with special characters and protecting critical account details. diff --git a/Dockerfile b/Dockerfile index ee3c978..085e87d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,10 @@ FROM python:3.11-slim WORKDIR /app # TODO Task 4: copy requirements.txt (must appear before any COPY src command) - +COPY requirements.txt . # TODO Task 4: install dependencies with pip - +RUN pip install --no-cache-dir -r requirements.txt # TODO Task 4: copy the src/ folder - +COPY src/ src # TODO Task 4: set the CMD to run the pipeline (python -m src.pipeline) -CMD ["python", "-c", "raise SystemExit('Dockerfile not finished: Task 4 still pending')"] +CMD ["python", "-m", "src.pipeline"] diff --git a/README.md b/README.md index fa94600..72e296d 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,12 @@ After deploying the Container App Job and triggering a run, capture proof: your screenshot using a Markdown image link. The grader looks for the `## Verification` heading and a `![alt](docs/your-file.png)` reference pointing at the image you committed. +## Verification + +The pipeline has been successfully deployed as an Azure Container App Job and executed. +Below is the proof of the successful run from the Azure Portal execution history: + +![Azure Job Execution Success](docs/execution_history.png) ## Check your score locally diff --git a/docs/execution_history.png b/docs/execution_history.png new file mode 100644 index 0000000000000000000000000000000000000000..21979d53769593b3d07e37a62f7efc42d722488d GIT binary patch literal 14083 zcmdVBXH-*N+btZ$iXtl5P{4+W2uPO_KoO+(A~hn?dkMX%w;&(^snUB52~t7{grEp0 zy(aX~OG1YbAVAK}{k+fj{W|~8yT^d+Hr8JIDs#^3T1mK;#%m@) zJ27d^=>r0ZeOG?7&$4}YF)P)xM?bvr^5ScWQF-E-oE%_Y}WTI!9Q$XN|)t>g2%apJ@AfA zpYhw9e`Zezd-SABQ8FZiGbI*1UPb`hDPTJPZXa7+N-zBPpea9UO6I=@FGBwp?k_|z zuHCqHZTVAIYQk1K?*SPPLBLOd|zN3ThGF2&eQpFe-bC7H5jXD8@YS~GnIroBk|Mi`UW_P-+Q$*=i?*whzbSbff$0^D=!aS85SfJ zBz=w4*NlSA%`dOjQ*95biY47<{=rlINf>OQNA_mV8$}C??Ee1#apa2`#pr&(Ks&e+ zeWEKhbB7FLFQDvvCWS;yA7&s3HhMfKx8@P<^X(-D2=q&Nc?6vQyQa+ktKh?CzR@bX zIFs~e709s)tH`lUkz*!y$yYYrY2OY9lUTT6B;dtaS=h~T!afcrMOtpw&@@u85Yv@6=bdFN0dhB4f{8^?uB1#gn^KZmYKlu&Cn-v}(QQ^U~WBajP zMXUGkZFh>kv9a7(#N8{npDykDQnx@l-_5cOeUf1aZit`ZQr+2=av03AkSD1=(Oa9u zplgDIp-a6lVp;7fm&?Oh$j!_p_ohDWHjN=_N`T>s9+Zg#r+ET;v^K&P ze$q)CjG?rY7HTLcC>U2eCU>Vx6>8_chU?_3;M5yRbPHi?#DUlJmsE2swa1kfACO6K z7eeE%!&eTlKFa#mfD@F*zCgFbk3C{t;w{y9x@{)^^NKWe(Dal`xY62mg* z(c_mAu}OtkS|AMVy`C+#R@;moEevJ4T?@0Uah}iz95euZ)pm*f8XFt2rZEjnja2-4 zGg~ErcNt{}VR|m}I`9O4aL^~}P>H_or%x{k15#6F0Yv4Aeo3B8;6V{ElL9M@ z9~n1f2yq&{4hre`J5p&wE(=C#li5Up^y7^e@kSQ(d@r`K2E%P^db}ncT3H2_xp=SY zSXlg3tXd@m%RvIk2|*H+NT~kj{aHi}(fixC0;|=r8pBjEd}`u}Yx!pNUMVt`1@vK( z;^I(N4k_>6Stev;6vW3Ho@|`vhkyK7d2qPh1Kk?@?4LEIs;3w4TsMDxpkkpb6)qnJ zE3lIDHBLfnaBHUZems2v+c!F0$Q7$k~0zA1b0 z=}fUjvjYMiXg^+C{zyHgbXnN?iOjK^nOO}oB`nNgD^Any;A^tqPI;SN<$}K7=I0w# z7Ds;6{mtg+_``Ny`XjDCH7(M2!Q}gqiN*FqC2Xz)(9vRL&kjML@ke9AYtyrOrEn5X zKj3f{>E#RzRxy$TIP;4ts?l+>!RYnYmM4vYIo@X{r0e>;BLTP)xIEz~i8qSL`wYZz?Wos;H_Cn3W3xNK^tLG`4rQ_uqg7OdER{I(`)r7FL_L z0bF*EStg4#R^`Qe8w9F_>lMeA=pjlV-5EpSENi(#X-?bYc@LHQzt{F5r}4+5hR>= zJ#9Xd>cK=Bg=ShTC(ZQN>r#)VMvzgg=Th%I@=~AKr?X6h1Hv|F9c$(OHqPKu>_v_| zU}8A3lQ9VZ%%uyjHyk<=`G|w%pMu)|_65sO)v(xNOz0){9N?+-L!uJM+*!qFcc$%i z9QhK$wC7Bq}Y7Zn4a(l6dwA7c2(~pg=RtX}0zbNqw z@}^eHyQOt0I=EJ0tlDXN)Y;hA-$m0YXB&e<1{#KkpGP;UI~+|j*VwL(dP6s+UZ@AG z2;zlEMF&8>@&Hz+lMm8mz*wFmKu-2>JRhj=g@JxRBb#JlEclvEqn;&>HDc z3?>h<>b{nO-yW^jfA#8u)9{bukKj8B%g=)g0k>ylq>HMqj!hKpt~w7kEM|&#+4&sq zxpjS)=|Tko29^v2_~R+B3%O-PLvjEgFN)ObKXBh&G0M+(($DKEpi!~3N+kacu3`ap zNzXwSUK(z_=$`2)rkTS#%52oW`t2*Y-ANRn*j;4dtJ{u#h^ME^O1i zw7-}!uv$Y?)~;5X2%=_I1szcf+ySn>J)V>Yu(|!t(sQS=>aNw7zn13y#?d1UbF@Ct z;JZCU?Uq8h&7SE>J}TuoyPW;$Eb8rX0MpIqTHcsoxZRXKo{Vjy$6BbUT0$~q4Pmeky0*@FFO4M zb%`%Pyyo=f&a$%H=2d2sIkW(%2%@d69W+Kh!En}oZzGILS+Kl3Mckzk?YTDdWGw09 z$CK)rGU~U*oqK!3IQqvIGjsuX_S{*@PvyL|k?%Z#j;I=`0?_>G{4eOiR)+v|ZCoO7 zYuU!n8dn*c7fhovplthIdK@2ntACBgH)aOo;^Me@#KjGP$EFB+6c9r{7b`?wPXYVG z6r*9f2~YG(73z_>-Ydh|bgi(AJ=Qmb1fs0L+h50ekLAxY+Y{q?h&W9BnM-$yX4|4k z%Y`;i#KrM?k6eqtM1|`EAS~*;^$<*+dtXB+DXVTbWEVCjeSuUfju`%=v!Kl!)EM*qMc=%%qRDyO{3UED}X>U8X2L>F+WP9 zuAvz;0g6&a7^4PD^m$ju>$YPvxKFe!JmTV!m4&cyx572j;R$Jn6=F`Kk0p?s83FDU zlR;FcqhksdAEyb<52Oa+w&GbG1RrrB1oFdq6ABvdNc(8jdj-N%M+tc)`fy{zq{!I8 zBnM=j;`H?N=q`bFW`5l0N1=j0bl)A!!C3~}V}J1V7Mt+Gm7|SjW;XGlG>1!9ZOr^= z7)k>!ACNA5-(?|X*y9vG3egH{LmE;IWA3|hpmJ5q6%tD)@N`LU9K5Wt#HI0g6m9ID z_TD^5!svnGS1k$$I6fw(2!|N6rL0ZRHJU?xOw47jhmY*1nqC}?5i+z%IHxLa7v#*$ z%+Ay&c@f{OxPzlEab6$_!Y*;>0V>Ws_(=5g5CVR~+%#|L`%R`>LV65mE)7;YjaBgq z4^(l2Kv~0xn#lq>hg|0~UP_pI53N1YR8cvo2e6zK2z3+ZIt9SOu@6xW1oH|19j#Ai zm=3E+;^VyrtD`kZ4*|YVp`Z47_0RkSSPFoHipq+?Dmz`KT9*)jmJlNz$H*}k>;#(U zGV9BiFWW@~t(!yf#OW&!)INOTX6NAtZdyg}f?cOVf)03IGqH(iEq0sf4`8F!B~L4y{dI|4vb9)CQ|ChAvpK(2osneD8qZc#$2a#q{e>Z!_ zP3rCDNf^xXNM%y1=J%}gEF42q*h2+wjWn(I@AC>YQVZ_Mhtj9-NgU(}q^#Be*@!gC z&257b_kFj`BpTXa?1dBc-l>ZBZeC+$T~ad7U=w#PWfQf30|ZlA8Yh=BnTW4JZY`m$ z@yma{J&^R9`vSej|0C1geX!W_tb*_NyCcOt6M-UutrwY&J0K!f?X1fEK(cZvw-49p zW(IAA)Ag1@WKt0X>tAKNc@?_6Wutq`00?5PDBj|D^88-8rDWhjm9Zo-K;f}+V6g!r zg@`XJ45aS%^@%F2PShLHbv2L{wj-(na9|V>`yE>_b;U7V+U=za2BiW>P9)kBZ|3cb zP(6M6^uaN8UBBMTiRtDuEx_5mbKIHzSwBCiuZMS-G3bV)b_P)%v;rgYE+)<6MX8P=q{y|Hl>%k7#i>ga>W$P|ar{3vG7wSI+E zF{i=bZ5-RlItE-tmQG`nwg?mI_J+vtHxfsBR%y5ztov5JR028z%L@AH&kMv-?zUl>2ebaF*Y2kTBk1S z7M8-L8JVM9Cup-X6&gjPw7Vlq2P7BQBMp+cZ59xPOYy>_U*Zs603yNuWOwyJ(_$Si z1(X-(gwAy!d=jb~???s=^BqIWNvf|Rug|-KVUd=aUe&v+V}<-?$EA``Vr?RgUr2|k z3}fIQd~5`GgdQIWnVDr8>GeJcRl1&6bNmY9ltuclEf?1I$c_X9$K!x3qq&QcYE#y9U@z0gcKDPdmMiK^O`(LDuZKY|H5Sxf=xz9mkR-* z1m8IHMY#~nZ;LtW7$1ZcV8@*6yAG}boO}fUDqwyjvbaH!VcDcH*5Xr7pNo|p{^+0}Ut8&;jZ?wk#-c9ao^9TAF(Dg~j?LX0$x8{=So#sQvgb8vPs4qn@;* z;J(7lz!Exl_vN&(iH>GO-VC9GufnS;y22-Ke7SNk7Z*()vB@+N_c&i5Y0#{Cyb7TG zwG08h+14o3LMF|#UX$awYYdsoT)$XH0ZJUyHaE}MU7K()_TKt^ezmJhKUXp8W(USN!}=aKS{=H(-Vv$Uzpqc2>zpbu2* z^!xWD*NHx=PLxPrf#8he(vjN`=&q!(+{74Dv$I#oMkjFeRWh! zN~&W+WnE@}j)nbi$3U&y>`)m2Z0~VImb6{AyZr@_@Fhl-)(5*|E(r-_jKh9k?(2kv zg#0s1Y(*B$p#^x*&$%vfLwnkBAd(jL&p%5k4#;m0)5%vJ@|H<@3F?=bo6+U}Pe2XY z^l;Qr=ZHga6UcpulW~Ulx>59-D}2U}fz)uoH0=G8XkP>D9hwkhzW}1j& zvQ$hU_V|+dP48eqk}8fu;N20&W_x?Z(2GAJ4gm+cCc`4Xo&p*)jG5j2$m>^XSp86b z9JeMhPE*DUQvk@##qM-nI&}ixLzpC$0L7=n1R8~Oa^2_MOm0ffSS*1fg^tdpH^&A| z)*F^RjNSdk;(u9 z0gXFx5=`p2bT6DmG;maJ0#7qX7?TS|Q4Z_voTFKdV8e|W1p)j0E!IGc6myF_<-Nn_pp9T_dl-vKzw_^VW6lOdhAX+a$XJzdAkMlw}=z)q*hqqtARrG zq26qH5mK@Z>y^Hjt1f8et$u^kI&A^v)u})^)rAh9o5yqvOn4IzaQmVDK_1^i>-0w=0t;~=%Uub)q87h=F#8KCqR!*Pq;nJi!BBPs zigdknX@#(EJPC|~+1XN!fWdGd7|!l=kGVJiomJvsaonVj zFxk=+L$iElnPEp7tZ^bAkeRhQiD}Q>UJ?Q+=|Cs5*cf$EllC#f4G_=>lX~1xH^Lla zOjs2=TOC+@`+C7*X67RLQkp2~*3t6r7|?$WEKNn%v658Zy!lR-YK;d()pW3lG-q2& zi9i@Lr-7GOwQ-H}EHu;2?z#6`nwZPvoIw&Gt#+0ySw!u&-m`=K^mH@)RDiC{5R>?L zmZ6T0DT}fR2`IZ4fF_K20h(Uv&>Tkf{{9sJ)W@cA`3zIb@89Bd_!6sIPQ4Jw>E5mN)+#;v> ztI-w~Y~o&NfU}B#<^lmXbmNt__BWuxQUK{WDw%HP-2EQPoRmcG$b|oX)s;g2iYp6- z^mL`eUof0qn)>+(UGD-}Q2Thn&iCK<0fDBjx&eJ4%*L;mnC;y$Nw1afzyiTgj+JWE z9SI#ki!GjDHa)lYAG3l$m)|RO{C8h2=3XuH|BUj#XaBr}?~Knk=)M1$IH=b1Gq56b zW&G^_oVfcXCh@=fA$R5f;~wDJ|9kr^I(m9~@6FDEK>RLtMja?8my(L@#@=tj{#?pX zxVVAOa2LARtx8hiVtM0#)-p|B(mm;VTDij<8SI!@&|K(I3RgoG_vulp)3Y_EPwk(e zTcMVfZjLF&^212C(T&xYq<5Kq$Ihjp;|)#doBx{0^!?nwcW9+FH9mCO^U6*P7VQ52 zuAQO)>ujC*RssV0ZwYqIH5U>RG2{Qe`s(Mk@g8MuSrZ5!CH+zDPW5lz_UmPyq%Fqy zjGM(pzePwh9j}my6uy?l`g4R&uT5sIS?ysGNZu$ebD+4?-RDR;At$ldwrht|kWmm_ zD%H3muUBR*jIW4IblWelu{(5xUc;&pV{&{x1<~R5_X2WC zMr*A<{%xe^si6nghQt-Dn8)_M#pMLKji?OQs@_U=5Xmb}vI1#c>aR$orc2|C%)6{p zMx7!R2`TSV)F>Ovck-=H_xlsv%~4nWw9Y?KKR@|#RDy?SQ^YX8sdX;IdpZ0=l~m&H zUvyOdYf=b=}UUyRI!F;*m zTwW0*dL+UX_bDn|M+e{J(tRRcO(R0bB=D`XHx5>HstfSP?c!3OMX$B7JZ!D*teMqH ze}Rvk2D3v}Cr8@t{QOYLg^eH!)rK6?bYCwXOYJJ1+0qx$Lh&v>k9)y9&1e<(QgnP4-blxZQoUWr@%)%PPE;FuMo7Vx z)3C^tFY_&S^HS}Hvmtlm(*pJrsWT3cU@l)}@=O^HQr-G05EMXhZr#kWtcbDhmPm2zn)&7##+ zhdU}VT4%vO)$GLI@wt5!6qkLYL%l{yc0LHl)V*7HAoT*|8NghduxW<2DVJ8I7EDI%h9v7KhdR~^iFljLD3sy?0d(INg=$aBVTsHzn>Cf-BT= z%xzak!KNE`lA}>lq%USPppe zndf?O+sBoAn1!4y?y=&5sOY{lEy=kt@`@f6gPx|Qbs9(OH9hw-aO*7_b!ybb%}T!+ z9@%oLmPikwd>ocp*fSg6mYt3Z_yLQIwS`1QZ~MGSl6oE;ZigKYoi0pEtHHrl%L>ID3<0_<$ z>$^$5x*sxfxoX6kMy-v?Zm3xqhb@>8dLaIfDMANaHb!DUD%ZmNQx}5xLf%?)ChWm@ zV&2FcLP-0^>_)+n3r+!e?psLeky#u%e{w@lv=slBWA{d_>&l5UW9bu}FB-q#DtJsr zaNAfz*xhQp+-GKqR209kir_@&B5cDzyMN8G4HbN=J*iS0A<^ViVU@09g`wHyY$WO| zcxOCu4>|c5A2j(aP4jRSu9|XnrXt~)%g3Cj4&&_?|3RRBsb7%fku+#;Us&VSTE4B5 zKYVlB8!uvdrCm$0N%q0(%VWme3|Ex``BnETWS?#fb;xENm@!@CJ|GmSJMg)W)$x#w4kB;n6Lb z8Ym$)Xpo~#Y}({PeuFM!4TmPlL2Z5dX)t!-(KQ!gpZf8B-jO6T$$&EyE@g+;Pf#|S zFaijR|6=kF-axjjx=`OwKExa)LBQ`aMoni8h&q^4((Mt&HV zEdj>yjMhbkQ7Dh5jw)`odtXctI^+89O)6Gv{jrIaLld5E7VJ-|Ra1?-@Q+C`k4B<5 zefV4*DIcyn7PDEM*ZCfr;&&%yo>rU3Z5-;u&xlXsz`0)}s1{(j<-JYv(81!9RgM86 zAG|6>xQ09B@1+yU{ih~r6s_xZBO^MFUwv&wWoR#eMlF_cA=>2^otQQD=nyYAVC^!YzqX{RXbTAZ@Nt>4 z2Vb$ckWd@%gYMWfciWyv$5=r^F8pFU?9fpxV7=m8_iOh?Ow}2LM@2%ut-|X!oX2Ne zgJ-x-+r(|S@^}2+5nTjW?-tc1Ja=D_Pz07fkwxOpS|(MyVnz zypuMkTdlff*!UF9Qrn>oj)?nDO%xutFOcpjr|77!zD*Po%4HqkgR`>V{TI+=K_3Pe zL*Bqt>k>%(;69G9P1@xqfEzIw=C`a@FuUOAu4;dMv)Q~Xok(or{$gu7e@Z-5g!0yW z%Z_Rbv1fXu^1T=? zQVk%n_73j?9IU)ACH%4<)jj?#rFiz#p07|1N8Fls!Fk4^c6}-J<%E@vQbJG5ylkz@ z9dm-!$E%JRiB`4~3o|UfaTdx2XKF2bY?--pA9H;b%y`^ibyr#CyUjUAG?Q+H@TL9j zo#V+<4UpSR^Wy9!`vn&1-4865z?PapBDS-aL2;ZAIV`8!3rj^lFnY^I^q1k76If>}Mk7i|R*&*D)1o@Ho!ATZ% zAIR}ZeUW^d5fQ3TDpMZz`4}I&LfZ)EFzF)5^dX?O#g1h89O@<~1?AmO?Y$SX{s2O&uCAqKVMylTTOe zL8}&T7d>crJ}&h2Cab)-h4=jWuXGIkpZ(m-`h#bJ?t0(S>2MurknLV&psG`mEVzUB zoD1tgQLEIHK`=RVP}!q&Ieh#hOWwk-&j}gB`W`~LqK!`^*(ms;>Cv1fr)kDDKNQqD1CKXlopAHNJd4+3Nl`=nVs{_}D z{Y>)#60hc*M@vPQM z`;9b1tCY#gYy;`u-s2ty*(Wl>dde&p%M6{=8?XhAzWc_rTr?W>wx)7=ndpj%llscS zy(Q=QTEf=YrJAT8vj6<)^jKFQ?6PrvPLBSWQMQ#}2e%|-GT4ywUk*)fUdbO#*#lrZ zT0bkZ&3d@NH(HWy&Mlcr@etf+Uh|(i&cmhK{c%n!G zuwDlw8O0pch*ZXyGnjZLPu={f+mB|BPdGYml3{!llNE0R;G`Mz{5V60b_OSr-*epK zpza2_ET=;NOlZpw*k@8amn!#vrso5opOwVhU;Dg2_6<#?zsSU9 z%`5$Pdj;f%AzgH-HJ7VOG>ll^TyxsvCAB0Af6*;jhYzmQ%|{T#n_AoMH6(~S}b#<}v>GHEa$s7p#c)|EaJLoj#-m!tF;b>%>Oqw_Rz#o*rz_1J zlwjJ0u{yICnPo0+GZ>F~M0-4wv)58so#V4L*Irq+jj;Y)oc5c!fzxGxS z%bj-F+Vww9H}>LQgcVFjJnOcGOsdpFa4d=VqpC$!VKLh^p7$%5LY3BUVSK&Zg_EzU z#CqI=ul!T9m1fYE(bf}esh<D}xDn7q*NWdW0XeEdd%7rIAP5p1CvIcbu?pp0b&edyYMa z4F<>NkWS3Sj{R{GIV9P?-Y~`C#HnbZYN}c6ow~JjyPWoe)ZOEQ9j4;^a}BHS`Et$sS&nuND6zS+S$C_W@n}V#{oz)Z{SO<19Mqc= zvvZW%_zKhY(ieC$*mlnQNLmx=lRB2Vd>#Yi1LR2AI>$W6qRtQ4%IMYgLOga66!`YV ziX73ysGT_^8cp>0U4Wg{8Fk0F%Q%YD_`*?zqq{qD&4^j+0^cCHZ3&!Y(#ePs|H-oP zl?R!aH0Bf1`Q9UC*OK#B`JC6!XHH1yB||tXvBAwM83XIj-G}`-u4tK59|-NFG501y zRmhS#gPc@O03)f5E|q$DauZ~*RrEY`}-s62sBV%-B z=Y~z{NA3@4x#>JAopB>l?aq>VvzEe7J@WE+e!9u54-XufmZq_CJ@42W<2d6dLkXVH z3BsQ|$I(rCHM@S_u!H+M=|e&pNs#t0Jj(}GKn#r*v0Kf_9KUx)#z|$>zaf(QKiw35 zFw-bt?Yv5k?~kOTRYk`?%XTCC|3bUVovPerOZC43(j;l_Tts1ed?8Y?xO!(DWE6^rN zA?vnCtbl)L+Nav^Q<|dLR-ms!EKfPeksYyTO#DYV_p~>YpJ-rahr<%af(KK|tTM8O z7lV|#ZW|Thy#W3Qg` z!RDaeuXNAc3*p#;ivxiB%qYv!Htwlr4bzFYi(=hrL0p!c?M_d*Q8sq{TR`JMl=Jb6 z*6FBkDTI>uarY9U{4?Zj(Dx{jZZyItL)c&M$go(1ZWXjbd~tKwtU=)421F7EqchN9 zk&4V&m6uOvzK4ED?_1fOGqKkso%#xLorW@U=44=RJDA=10erwUra$bA_IUNYSkDgJ z@rUj9hI$2;p?IGWgD}r@++_~QuKX}!xo%&p^2;Ms7)Setr;Im&xdR#*)}xIEu!DsP zeOEp0-KXy{x6anQCq*oQdIxEZOLaG|_Z_D0?wE)So@d537d^KN599DZuEj=&9XQBBOx>@kJu3MnXqu(!?kEHCMZJx5K&tcD5qnhu{zN z+_AA;cz#kHRqR6&=NDL|x1Dmw%x;g%(M5U>a(wIhR1gs(wm1;;Jl8^L5FtEaM{HRupVCiS$Jsfzhpi>~v7d;BK59P=aT zJPD2UA77qPwLCu5@Aq%rkqnhM_cHd*gU2cdz8{s8fNyL9%LY$0{Bj=jPS4uy z%{R-G3RpVsi{|`n3%b~OLmK4j_LTN6O)Q|k7d>1^cj!HFfgk+zUp|54+_wm_2z{GCHZI*Ihv zZr<~q~r|CFeN;LdX0-);mG_VUJA)^0Q+lqc&Vc?VD1qG zvmq}eP{6lIOalMhh!;{nU;Rxb3ZtWF4?6eg|GsA6x%{Yy{$ngqRwvBCITjqK)oK+Z zS$O}-c-b>p6VmP5YWumBLT*L-L1PJPJ*>!fv;LPfRx(L&OvEDW+{*tf5Snz JzI^-f{{Y1!P`&^F literal 0 HcmV?d00001 diff --git a/docs/execution_history.txt b/docs/execution_history.txt new file mode 100644 index 0000000..ac1f32d --- /dev/null +++ b/docs/execution_history.txt @@ -0,0 +1,3 @@ +Name StartTime Status +--------------------------- ------------------------- --------- +mareh-aboghanem-job-u6ikgff 2026-06-10T20:14:29+00:00 Succeeded diff --git a/requirements.txt b/requirements.txt index b5d18d7..81c09fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,3 +11,5 @@ # TODO: pin psycopg2-binary (uncomment and add a version) # psycopg2-binary== +azure-storage-blob==12.30.0 +psycopg2-binary==2.9.12 \ No newline at end of file diff --git a/src/pipeline.py b/src/pipeline.py index ef1fc8a..5c2bc98 100644 --- a/src/pipeline.py +++ b/src/pipeline.py @@ -5,7 +5,6 @@ Database for PostgreSQL. When you finish the assignment it will run as a Container App Job triggered from the Azure Portal or the CLI. -Replace every `raise NotImplementedError` below with a real implementation. Reference chapters: - Blob upload: Data Track/Week 6/week_6__3_azure_blob_storage.md @@ -16,13 +15,18 @@ import logging import os from datetime import date +import sys +import json +from contextlib import closing +import psycopg2 +from azure.storage.blob import BlobServiceClient logging.basicConfig(level=logging.INFO, format="%(levelname)s %(message)s") logger = logging.getLogger(__name__) # TASK 3 hint: quiet the Azure SDK so its DEBUG output does not drown your own # pipeline logs. The right call lives in Chapter 5 (Viewing logs). - +logging.getLogger("azure").setLevel(logging.WARNING) def get_config() -> dict: """Return configuration read from environment variables. @@ -37,9 +41,29 @@ def get_config() -> dict: Raise RuntimeError with a clear message if a required variable is missing. """ - raise NotImplementedError( - "Task 3: read POSTGRES_URL and AZURE_STORAGE_CONNECTION_STRING from os.environ" - ) + conn_postgres=os.environ.get("POSTGRES_URL") + if not conn_postgres: + logging.info( + "POSTGRES_URL is not set.\n" + "Retrieve it from Key Vault using the CLI, then export it before running:\n\n" + " export POSTGRES_URL=\"$(az keyvault secret show --vault-name kv-hyf-data --name postgres-url --query value -o tsv)\"\n", + file=sys.stderr, + ) + raise RuntimeError("missing POSTGRES_URL") + conn=os.environ.get("AZURE_STORAGE_CONNECTION_STRING") + if not conn: + logging.info( + "AZURE_STORAGE_CONNECTION_STRING is not set.\n" + "Retrieve it from Key Vault using the CLI, then export it before running:\n\n" + " export AZURE_STORAGE_CONNECTION_STRING=\"$(az keyvault secret show --vault-name kv-hyf-data --name storage-connection-string --query value -o tsv)\"\n", + file=sys.stderr, + ) + raise RuntimeError("missing AZURE_STORAGE_CONNECTION_STRING") + return { + "postgres_url": conn_postgres, + "azure_storage_connection_string": conn, + "source_name": os.environ.get("SOURCE_NAME", "weather"), + } def fetch_records() -> list[dict]: @@ -49,7 +73,13 @@ def fetch_records() -> list[dict]: one dict with a stable key set (for example: station, timestamp, temperature_c, humidity_pct). """ - raise NotImplementedError("Task 3: return a list of at least one record") + mock_record = { + "station": "Amsterdam", + "timestamp": "2024-06-01T12:00:00Z", + "temperature_c": 20.5, + "humidity_pct": 60, + } + return [mock_record] def upload_raw_to_blob(records: list[dict], blob_conn_str: str, source: str) -> str: @@ -62,7 +92,12 @@ def upload_raw_to_blob(records: list[dict], blob_conn_str: str, source: str) -> teacher has pre-created it). Overwrite if the blob already exists so same-day reruns succeed. """ - raise NotImplementedError("Task 1 + Task 3: upload records to blob storage") + blob_service_client = BlobServiceClient.from_connection_string(blob_conn_str) + container_client = blob_service_client.get_container_client("raw") + blob_name = f"raw/{source}/{date.today().isoformat()}.json" + json_data = json.dumps(records) + container_client.upload_blob(blob_name, data=json_data, overwrite=True) + return blob_name def write_to_postgres(records: list[dict], postgres_url: str) -> int: @@ -78,7 +113,39 @@ def write_to_postgres(records: list[dict], postgres_url: str) -> int: See Chapter 4 for the connection-and-cursor pattern this is based on. """ - raise NotImplementedError("Task 2 + Task 3: insert rows into Azure Postgres") + # its already added sslmode=require to the connection string, so we can just use it as is + with closing(psycopg2.connect(postgres_url)) as conn: + # with psycopg2.connect(postgres_url)as conn: + with conn.cursor() as cur: + cur.execute("CREATE SCHEMA IF NOT EXISTS dev_mareh;") + cur.execute("SET search_path TO dev_mareh;") + cur.execute(""" + CREATE TABLE IF NOT EXISTS weather_readings ( + station VARCHAR(50), + timestamp TIMESTAMPTZ, + temperature_c FLOAT, + humidity_pct INT, + PRIMARY KEY (station, timestamp) + ); + """) + for each_record in records: + cur.execute( + """ + INSERT INTO weather_readings (station, timestamp, temperature_c, humidity_pct) + VALUES (%s, %s, %s, %s) + ON CONFLICT (station, timestamp) DO UPDATE + SET temperature_c = EXCLUDED.temperature_c, + humidity_pct = EXCLUDED.humidity_pct; + """, + ( + each_record["station"], + each_record["timestamp"], + each_record["temperature_c"], + each_record["humidity_pct"], + ), + ) + conn.commit() + return len(records) def run() -> None: