From 158cb63d54d5e64ed6999312aa454072f2b6739b Mon Sep 17 00:00:00 2001 From: "peter.fong" Date: Sat, 29 Nov 2025 12:58:41 +0000 Subject: [PATCH] bookscraper new --- bookscraper/app.py | 53 ++++ bookscraper/init.sh | 44 +++ .../output/合成召唤/piaotian/cover.jpg | Bin 0 -> 11043 bytes bookscraper/requirements.txt | 5 + bookscraper/scraper/__init__.py | 0 bookscraper/scraper/book_scraper.py | 269 ++++++++++++++++++ bookscraper/scraper/logger.py | 27 ++ bookscraper/scraper/sites.py | 11 + bookscraper/scraper/utils.py | 22 ++ bookscraper/static/.keep | 0 bookscraper/templates/index.html | 22 ++ bookscraper/templates/result.html | 18 ++ bookscraper/text_replacements.txt | 79 +++++ delfland/azure devops/toc_gen.py | 53 ++++ delfland/azure devops/toc_gen.txt | 36 +++ delfland/init.sh | 47 +++ delfland/po-quest/.gitignore | 3 + delfland/po-quest/app.py | 17 ++ delfland/po-quest/config.py | 7 + delfland/po-quest/database.py | 6 + delfland/po-quest/extensions.py | 3 + delfland/po-quest/forms.py | 14 + delfland/po-quest/models.py | 20 ++ delfland/po-quest/requirements.txt | 4 + delfland/po-quest/routes/__init__.py | 0 delfland/po-quest/routes/admin.py | 94 ++++++ delfland/po-quest/routes/main.py | 10 + delfland/po-quest/templates/admin.html | 27 ++ delfland/po-quest/templates/base.html | 26 ++ delfland/po-quest/templates/edit_choice.html | 13 + .../po-quest/templates/edit_question.html | 13 + delfland/po-quest/templates/index.html | 13 + delfland/webhooktest/Dockerfile | 10 + delfland/webhooktest/README.md | 0 delfland/webhooktest/app/__init__.py | 0 delfland/webhooktest/app/logger.py | 8 + delfland/webhooktest/app/main.py | 7 + delfland/webhooktest/app/routes.py | 85 ++++++ delfland/webhooktest/app/storage.py | 9 + delfland/webhooktest/app/templates/send.html | 10 + delfland/webhooktest/app/templates/trace.html | 28 ++ delfland/webhooktest/docker-compose.yml | 19 ++ delfland/webhooktest/init.sh | 19 ++ delfland/webhooktest/requirements.txt | 5 + solarforecast/README.md | 0 solarforecast/backend/db.py | 97 +++++++ solarforecast/backend/fetch_ned_data.py | 52 ++++ solarforecast/requirements.txt | 3 + 48 files changed, 1308 insertions(+) create mode 100644 bookscraper/app.py create mode 100644 bookscraper/init.sh create mode 100644 bookscraper/output/合成召唤/piaotian/cover.jpg create mode 100644 bookscraper/requirements.txt create mode 100644 bookscraper/scraper/__init__.py create mode 100644 bookscraper/scraper/book_scraper.py create mode 100644 bookscraper/scraper/logger.py create mode 100644 bookscraper/scraper/sites.py create mode 100644 bookscraper/scraper/utils.py create mode 100644 bookscraper/static/.keep create mode 100644 bookscraper/templates/index.html create mode 100644 bookscraper/templates/result.html create mode 100644 bookscraper/text_replacements.txt create mode 100644 delfland/azure devops/toc_gen.py create mode 100644 delfland/azure devops/toc_gen.txt create mode 100644 delfland/init.sh create mode 100644 delfland/po-quest/.gitignore create mode 100644 delfland/po-quest/app.py create mode 100644 delfland/po-quest/config.py create mode 100644 delfland/po-quest/database.py create mode 100644 delfland/po-quest/extensions.py create mode 100644 delfland/po-quest/forms.py create mode 100644 delfland/po-quest/models.py create mode 100644 delfland/po-quest/requirements.txt create mode 100644 delfland/po-quest/routes/__init__.py create mode 100644 delfland/po-quest/routes/admin.py create mode 100644 delfland/po-quest/routes/main.py create mode 100644 delfland/po-quest/templates/admin.html create mode 100644 delfland/po-quest/templates/base.html create mode 100644 delfland/po-quest/templates/edit_choice.html create mode 100644 delfland/po-quest/templates/edit_question.html create mode 100644 delfland/po-quest/templates/index.html create mode 100644 delfland/webhooktest/Dockerfile create mode 100644 delfland/webhooktest/README.md create mode 100644 delfland/webhooktest/app/__init__.py create mode 100644 delfland/webhooktest/app/logger.py create mode 100644 delfland/webhooktest/app/main.py create mode 100644 delfland/webhooktest/app/routes.py create mode 100644 delfland/webhooktest/app/storage.py create mode 100644 delfland/webhooktest/app/templates/send.html create mode 100644 delfland/webhooktest/app/templates/trace.html create mode 100644 delfland/webhooktest/docker-compose.yml create mode 100644 delfland/webhooktest/init.sh create mode 100644 delfland/webhooktest/requirements.txt create mode 100644 solarforecast/README.md create mode 100644 solarforecast/backend/db.py create mode 100644 solarforecast/backend/fetch_ned_data.py create mode 100644 solarforecast/requirements.txt diff --git a/bookscraper/app.py b/bookscraper/app.py new file mode 100644 index 0000000..4f6d9a6 --- /dev/null +++ b/bookscraper/app.py @@ -0,0 +1,53 @@ +from flask import Flask, request, render_template_string +from scraper.book_scraper import BookScraper +from scraper.sites import BookSite +import sys +import os + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +app = Flask(__name__) + + +# --- GET: toon formulier --- +@app.route("/", methods=["GET"]) +def index(): + return render_template_string(""" + + +

BookScraper

+
+
+
+ +
+ + + """) + + +# --- POST: scraper uitvoeren --- +@app.route("/", methods=["POST"]) +def run_scraper(): + url = request.form.get("url") + + site = BookSite() + scraper = BookScraper(site, url) + result = scraper.execute() + + return render_template_string(""" + + +

Scrape result: {{title}}

+

Debug output:

+ 
+{{debug}}
+
+

Terug

+ + + """, title=result["title"], debug=result["debug"]) + + +if __name__ == "__main__": + app.run(debug=True) diff --git a/bookscraper/init.sh b/bookscraper/init.sh new file mode 100644 index 0000000..ec08bef --- /dev/null +++ b/bookscraper/init.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +set -e + +echo "📂 Creating Flask BookScraper structure in current directory..." + +# --- Create folders --- +mkdir -p scraper +mkdir -p templates +mkdir -p static + +# --- Create empty files --- + +touch app.py + +touch scraper/__init__.py +touch scraper/scraper.py +touch scraper/sites.py +touch scraper/utils.py + +touch templates/index.html +touch templates/result.html + +touch static/.keep # empty placeholder to keep folder under git + +# --- Optional: auto-create requirements file --- +cat < requirements.txt +flask +requests +beautifulsoup4 +lxml +pillow +EOF + +echo "🎉 Structure created successfully!" + +# Show structure +echo +if command -v tree >/dev/null 2>&1; then + tree . +else + echo "Install 'tree' for pretty output. Current structure:" + ls -R . +fi diff --git a/bookscraper/output/合成召唤/piaotian/cover.jpg b/bookscraper/output/合成召唤/piaotian/cover.jpg new file mode 100644 index 0000000000000000000000000000000000000000..733afb469ca947d06c8b260784213d2628e2c738 GIT binary patch literal 11043 zcmbW6WmFt6*XM`g6faPqI74xlB7-{=DDG{6qEpTHXK3i?&oIz2F)=XyjsEhlAAmuENy^ACi}ga= z0-MQ|OyFx`9uBiybr-qLqGICm3W`d~DynL_ zdin;2M#d&qpR8?c?d%=gJv_au&7 zf9-#R{U2N;|G1uEV4!1Q|A!0hnfJenPJ)5S$d5%TtBq~p`hrQ|D-M}lVqSF@F0-J{ zDY>QFBpwA&XoKbKKWP6&_J0Q!^8ZElzrg-C*CGHP9qpg<&`AK&fHPvrBZCq8nj&k1 z-fyEB)Wu9@)O=q!WYR_96|$!ui`gl6;XT!h^PMe)jcJU^4NZkQ8i+LAgrZ*0jJAx;6)k zQMqh~9{Y@ZWx{Ost&w6U-=<9Z;QD!QZ3WD^BS)e`Jo8R-oYv za*-uTvL__>w`!6K)=8-pf@{~;0(mjEH(Uf%FVWk)Ol-fR_UNoHLxu!IWiq!PlocPq zC6fpV4RUz}g_}?G0{c}fbWx9!Co@yDiiKVgNJgPh_}X{gnM?YzN|tw%vgwJpR*E6o z1UoxJ^#c4%+fR6eoSJKDQ?Mw10-mw0o$UejrklM7_S)4$it;Dx%0^H+qrQt&L8yMg ztW{&Z;OpVku%}b>8l%U#xqSz*OkQX={0AZ($5cIFOUkt8#}hWqoV0iGK{9D^|8UAB z2`%gi`@qytH+lg7bafY%BoJ}2!q(h~tZOA-u`9SIPl7$IGuL5t2|Tm^u0d-CRT?_VqZOg@v73#Kg?Y5eQ1e>&#q3sx0TqC7^OSu zzM&(S*}Nw|O{`|lq0{W2pL@Rf0e5Zx&6K?R3ry@C7g38N^EC(;f|q7o_#^@~b9l;g zz__S4QqXu)nfvhf;TnIj9PTGF{mfmt5!H{GoFj z(4&(uK)dfh1h4Q&(H%5zb9-;BXs-HXQINC%y65K-4+H5srfhvsupH^{>xl`j`(QY? zaNgz^A>ovPjR|_zY}jVLc73tQCnoJt6-<7Ay#cY^)K0fJMYAZ}e)4C8V7Ucci(YlB z>Y@@8ESC{cTJX#zjN9W2uX{1i-t*lzE<`IaetyYj@nOG7PmB;zGh{l}X$voN_aG-# zq~^-nkTI?(puBJw{jZ$R<;%*#QKp!4p1Om84nfCqk*Rl9BmE@}Z5{^V=5;C0sr&Mj zPrvqk39L((JU40e@~TnBJBM+EsPv!mQerR9zKS(8^t2)v0uJ*r@r4plqJzG;4&Gc> z&NzWbt%|al7s9ccz8EU6F*Mg$5Nx_7Sw&o1EK^s!E^@VJ8aQUn1j0wgZb{LyzEbTu zmuSI6mJCvNxn1JESzLhXtazbc~2}G z&!|8CTJWa8`9@1xUdCw5I>l<3o}G$bnB2>;TJ_sGNyHB2Z@V8vgnLDPbKShSw3d^? z&uf^bVT1N8PS~^+L$Dc8Ic_H_Rh4D!7k86Mp%dJFsLs*?2f9#^3(Me6dK+hcWfQZT z3&PwX3%0;*`h}!kFF@^#d0|N?!LHduL192T{)h0uZK%wgcT0V!^3As(W)N8D4$SVryZC^;>=Uim*yc#L61d{|`aV|8B9~k1G+y2; zIsA7~i{t~hFAMv01JJ|pVQj54h%i3V>80)rE5VnB=hXC2KEI3QnJ0-`U*{^hxkVQh zVHT=OvL9TG~(h%X(q8p39y|OHUr5a)P21;=A831N1?v}=4j{0CoY8J$W%;C!?$}qyR}ss zzg$focf7m2gt*_-+aH-@r-5Janq*_jeSclq*Q)o`61Wf6ne-)>W1K?3&KmKcLVon! zy|S*QcsM3g|RGceI`DM+_w(U@x?&Tr-n@pD`q27`Qm1?p=pj@TzcTUfrUXaF0Fk` zS>M8l_RwMZodg4WbvI_3@T)=TB3Z(s$_mW3y4aMgmoagi##!IpkF~uV9dSV4v3BBE zG?tS>v87GQnCzj|HOmNw7foSREv{G0O?R{EH#s_15DN;woU2rzxMDP_mB^FtRoun& zGK4roWm5eJ2R?6qSNIMDA?uqg07Ass$1RqoN{>YLZSftqLX6M7WfLm5<}ZPMo`s=$ zlTPcMi1;B%)tHZgOi7Q7kOk^RIm-+ERCOnPsUKJLYM6PLu*0I}$^=}zQ=%W2`DH8p z(PbhzxT9p9XNp4^msR*a(?-(|(vG&yo=iRI=-3TKzGYa=UuY>0CfEJuti-ge&IqgB z%O6;h>_1%qUvT_87IbDRczma!f!ALE(dS~@h!rEzEHv<`rZGgK zLhLpP*N|iOfcil^=vyF_I+s_4GkL3$(Ux{_Onc63*!mW$kRKVi&m%9w&ws9IPl`&4 z32_$UK;iYmTve!0<%Ei3NDKJp3Pv9_^;po8{P9aCS-c$Pzh!a5I!?p$pgJ2xR+M92 zyd@37$(y^UaoKR;oXWWlyTpLLR5Eh2yP0gdZu}X5o88FF6O^N40E+ z8LL0D9yI!;k@T@L6r0C}$L+V}B7%)-`?>_*?&58YsVYrB z$60b#AZ-P1fKtNz#WC;uIY-uPVNM{w>93jB)9H!M;l{!DYuF&JfJk!s{WrP(X&K8%RRT{JN_ zGapFqfxS$_=SzNtdio2{ou<{&0n06^&1{5tH79!j15ZHwbcKXyRWe2l(Yr<|G+yNI z56CCKqWCphqLlhsUGkL;oQkl|^NTZ}{Xt}mQ(t-|0uyLQ~^I&KH# z=DPXNK)Wj7wM53b;vwhaWu`3bftUkA4Q9uW{w+9uka1APN+}Za^Jj^8b+|@4^5BW+ zv~4BrdNd_z#UxAsXO3#C0og7rZ#zcD0+w(bh2xw98V%{(N0QkIMqN z`1J)Dq(V|0yAHocr@%xbF-+&APDaM5d>M>Mb}8cLINE|f{JqeCg;>|W2{KK zy$USRD>31n9HKO*(n%N4JfRq>lDdtZg}b;r#)8Fzt}_d=m$RF&FMh8RuZ4i33~|Yr zS>k1La>LzBa-%*U59FM$<7Az&a!K2Nt=|CIc8h<#qtLU%vwjTZK5oocUM9Rl)F0KfN6Rou*_9Fo`Lz) zkB-3Sv$k`h3~0CQpGON%PeMNjFgrUot7qy%jZ>RXSc4^f;2AaAp`SjM4#9j%?;hfU zy4oqydc`|~p>b1;%^PQvep`p&*#HZ0$$mRk!|78dhO0)YT7nXZThAx!c$keHbzsN( z8kVBBRk5j0TKRUA$y!12QcZvqFKP*+K=j zEs`-d9QvH@7yi|Lz#FjDS7a3m^n8=nt#69&=h_lS5AjEK=2H4>Kdmr?N_;RES5^L6 zvowFGC&tBl>n%sk59Ht%hklCb(DAoubnE!2y{f4;|6e%_HD_3nn4`69AM-xNg9-@w zHK9ph6f>X3#s-hMc~hI?%*DZ_2Cw;)uqjM@)ztU$j|sbF!k!nl_Iuz#y)7-DhW|}? zq3l)%rM;T0>eocw$I6d$5r^88o>UWXk%8_jIK-`0i9)8B(F811t9gW3MvlDEQN?j`cG|{H8$zs;)}TQ9)w?@t!kZ|KVqQ z$;QQw6wTj7?%*A^=htjs1f1hU&zuN2HzV zv@IKkY>|*08QfKNicV4#-`k?iM!eTdEPMkk*b!)-kVZ4oFVjROE`r`0_)dS4-O@m}QxSs%y{zzY)=oA(b>A?ffR0zwb$3iP+1z$)zHqDI z5{V`jvy;o>yPL~%2;^!E!n41dEw!WGaGT6o)>E9WK=tQMn?N}+dHkBooWJN^i4r|i zt)P#hcWkZ6{*)WEXw;*x%Um`-)u=_1;^=pkjQRE?M5?%r`hJtH#M2STnCe!fstCyK zYb~~?Ha3rQGdc&G=>1^^XBmwcB43vbgE{fvXLINuXje@YX??$#LWto`1k_myjmj0! z;585z4k=8+Ok=$u5&Q-+n?*5R*NR-N3aT zOVQTp4b(l@)od@eRn~j+QNOUJA^Y-(H%Str^6z80vuSeWAu+5ksZIF>Q!59J%1X3( zh_z&5LHf0_jSy)LqX@4*KRJ=Db`?cFTB=hi7c>mHLybhw?A(vqIcsARN9$^RYm&3r zXtplk7n`2hONZ)gG2cZ1gim{PSm$zlW$jc^cq;`%EiSW7cEeGWT7zzmK)sx8FnZ ze1}-LWgrP_vhbpxu9BBU7EsP~D+taFojkLM_0<-ndh->|@%O&~yFuEz-U}>1W>D~6ncQUHWw^TuNtG~Hp$6$}I7SZpw``7M1j~1hm?WQ4 zwwVc-_aR>`dKk-?1VBLR91a`g#9)#86LY@pxfA^1$k+#^_hO(7Dg{!vNuV&v5&y6_l_2iNiGs3d1X~0jdYUusGRl?x z(fdYh79zBQ)Gkbi?^mqeUBHWyHhl>Jw1L5f2t~Twb!QwU?&F8drN%CI#>ZWi?c_(c z?u#*REqSvk%Fj}?ie2u4Uy;XF^)t2*!Sf0}@}AWYrU3s-c=4uJQ|h+OpxEmsf4Fwn zmGJU-3ipDkEO?{zs<3*Bhi)aII%75Hjnqdo6h8Skpa;2gFTXt!S$sb)Z1ad; zk33#^q<=K0O73R$S<;oq^y*cv3bDD1;U8%qk2HnKo6U^s&0<_les(8jK!<+rTd>j? zpt7xY1<(`BUgRpGf~YI(?Gwyr)5*5%4)^6XZ=bvr}SH(kh=DZ;mzel?&7V?6s1^v9-rQM((nWhzR z1DkL%+ZQh^ihl*ZV{;Mf_GG>g*514>QzYvNoE0eMu#V42P#m?3vLaFOsBYR-#=cF?wK!_s@)kBWuzIdh3IqE?OF;mkaX} z=V{tFHny02sV3_fTlOkgrs}nZ$fj(M^DW+`G@2dEYR}=O)G7jJIAjZ5^hnnZ)mNf3 z#9o`v+OC z+yS>GM2jD8bN{pSzC(APv;EZhy03OK=or}Ek*gT4_D>vLWYQ$Gr-pNswE+F*ThYVx ziXygZ#bvaCQM1+wkZj4CgO|3vD`7P4a7|}z6_o4^w$tz2zFJ2Q(~1=( zB#NSrxYsRw(X@1n1O1sBWvIcGIn(A^k2o~{+_E(ktKtJzOTkhO!fhR?iy+_j>~L##ved!LSYAd} zC=(w)zH)iIE()x9jAZHCh=V;X&`;=1zlBaFP^RPjSpvm7JM?BAQ>QD2Rlcic9SL}t zKSB_ykmpoR0%O2S*;9k8tVu5)z!d(4W>9Mkx21S3S@YPcu%suIiRTr0e*xT7{2TBfYVfnx3l3IC{XDCI)C zW_y)gon0=Tf#L=!uA5^moAx@omc0#&`SZz1I@a=mr^P5h3y+_^RVV8n(lyJQG~L%= zh&*Fg;eB3C($!T!V(?Bq1vSicMo{iiH0Q9SWH7cy|!G4O|R;A9$A*I)7ag$*z?BUh-sIZT^wt4WzE$%afdRi4~?ad4zPPQy3VbEfMtG2x~S z0$!tfq@i|vx9%$WE=ytnLibS6XgX^e;U#Hij`t$`umaKC^5IM(p4#sHCvdr`oc3#t zu|l`7xQN#L!`CXA%yW>N7Sk${_rOr|>CfCA*Rc&~{Pt-5bhBF-qpg<43bq`$=y!&G z-clm(c2YY`6MSrC>SsJenbO`i=1A)7s+{m_H0tpcDswzb9Ea&5?xhOI-VGL*2xr*O zJ?lG;_U#_Vi0vU3z#ZdX);TbNCfj6~zF&0UeVr;a)mncafFPw||EQ!TtyZwnAE}YP z-C}<;j*++|Tf5_#K+c_KlZye6OZ@mpS2eO>gLLUe2(xoUJJzKj;rUELjQOk%bmTV!`5-rl%3 z9RB8;fG=zDiH;@~y9Z_L@x!qg*QT&(QZAQEk^rMt_X*t!M=r$K)44&DtAxKB?8M3; zl6u9`?Fn+dom1aXXU{*qLNw^4!=02DZQKBuYstavYZ^F#HZ>S=42i5z2w!NRwFk?#{5IHl$kQ18SNf+-jt1=8`N{ri=uYT(-{(?%?jS&^1bDMdw?tP=n^=iiBkGGmDYwn`NqNZAk_iU`g1=%c>b?v%m*JHFO?lx+s1wpJ>4)^eaE8aTsX0emn zO4Lr^T@#Vr?wV6BF51_VnmfGFj*Q0%sLn0XHOiqmt#7CRS8RSD50M%Z#K!>mY`f0m zZI!8=K=<0h`uwQ_C;Rvf{?vO(QB8af5ZlPK0BI|ob+`LMVnS!Oky$seFGW$8>fef& z1uDac15=g8b)Q!JTd4l!9mL6p=~O_5(dHWP2Z=N8-t3~`USh||0=J~RgZ=}PvHZeR zXk=z&_b@H{`}fxeQYGMWtt!o9^6~7LEo{;8tIFrG4BZ$>=7OATo5EB<{WXE0IpvKG zNVj2o{q)IY_jNiA1N21#?nzvJ1L#9>x1-IJ5(MI=qOG;a&SRG~`}3keO0`08@H11L zxnu@*x|35Kzj8L-!dBT%p{Ow*>T?2JJ$xG5M`q-;qqhe{?MBP#rgwo zb47RY)?f6kS{ijmSxU!U{k)6;t?!r{J1_$E3P<7k*)LIeY+H|AOTZs8m-&7M56Pw* zm^jU&=F;2rz8n^(JQuMlc~#BK!o`K2avKsXhR0dfr2dO4(6sLOy^s;TU>V6?uz;BG z;;L`b7=^&7&~LIpN&ed)j$NDmi8M_@F7Bzd(R;nfV>cS>d_4EYJL^dD>Ao_2H3&cv zO<(z=xK0;NoRFCt1j{}N@CJP;(L|#3J8A$91h{adgKVI3-nMH+O8xOL_j*qyah&-s zv4bUHqoy%`>V%L^{B+y%~+;I%UnmOZpAHx)&2M z#hUhB=RWJ6z;&XsBB;uYrS$5Xn|K`QzOvz_Cwt0OH%*$ot4w%EiUCKlahyqGN?6ZXBFk_kUtiKhLPz>+ zchuf=|6)U4!giwZ!_KIOJEhS|?HTk3lNQ2M>36|SLeEQjwTKJT*((`+r+~)$^@RBOH z=&)Vy2I)056df+e03_A)7kLRx$@@2rU=ovet?Eo9+UIR6UdlVTZj-0Nx6;hi0V4GLaCstA{HZ zW`69G&hUIHy{9C0U~wvn@OY&qODgHy*Xt%pnIx{;PzI==_a!DW@35a0G$Y=-UptBV zVgA8j*5ukdgcpIy{xX>f>Kft~s=rm!+B#R#nig4M3l{v&BBG zQkw35K99mn8mnZs6s26xJ@_-1>w`7X_FI9ZwX=&EJ(v(n>(&syWYWBfwRE2n!?x_E zO8Z#B#@Ui4_D_jxhoQIg>%rGPl`F4&)NgC!=sBmHS^sp0H`MrD)tMjL72UhBt3<~f zH_W*dnT!q8Q@qLT%gcDD$}9?+hIRNnrweHs>rruy@np@gAL9@4cKwPhRsbfsV7vM2 z36SoQgcz0}!@|}S48MOHTlP)z)&~)lm3{~q(~rBNCHpdY>;2$xXf)E#NVL^SHfQS7 zZrB1B z`pq^+lfrH@6U6?my`b4qN-+-5E1>dc;IMn?feltX^|2)a-OEJGD&A(v3orOn4)&I! zl&+MQ^<##8oO;j87~1d-L+L4)qBD)p2ZV&}52A+4chcGVcF5PVtRo`;Ityw3%YNNk zyfj}t5|-yncv*h|xiEsbo)PZ6S0nGr#?Ruk_L4QA8{1~3zk(3rkC<{K=mPzOf5cH1 z$$tUJGOxE-ducac-W@>h-8M!a7dYTP#lFS-rszVZhU{=OV|R($%DOi?^8JRW(xWBhU` zZgbOkZj`%kos1apz~M~dSM*X2jB)6*VbKM)80 zXQ=uK?Q!GDf98Qr>b3Q4bkE7hfEK&_l&UozO|)`qFT~uU$6{7uzONuc7pH51$=kW~ z#7K%Y#I_;zT}Y36qZ5YrR-68W=Zd}~#n7zKQ6uv!5wi2BEzw`6nw*?#?$)ok2~$=v zM;P{O46DpM9{Ek1KRa-%tH_mZCxN37A?I6W4WYklmcA6OSNsJGOSbunj?;4d1w?~( zkV22IU#U9(;1?RTeuZ;$EIM2big!@MO1#+_%8pwwppaz-ITHViq&U*+z8m$V7FqIy z7!zrIw>qs7$dMReocWxqnWl-z0e&}8e=9rEhi$L48n?$7X#NX*gz zL0b%f-_41Mxrb7&zG;*Fqm#Fe{1;zNt~l7hD|H&)2aF(-y*Zt_3}XhRCIi!Y1S2!v zn-Hgi!mJi0NcP-UcPiN8@sKR#w5U-+nJb9nn$wYj^J9li4TR+# zKtS60S3?X!Y`1b)9QbTWQ)TFqmS68GiQ6*H=bL^^X!t?g);UZiLn-y0`YW^_<%!sAh1vxRgY!nI04ARw3$r*HXepzzcb zi0?mVT`I)Qj^VhE#u@@Be_8+B-}wthvP|Ga=ZUW=Pmbl!n9>coU>V5^1ycXC*T`NN zA?iY}mK5a9-Q#>gf$j3qHFsRt=5Si48ObgDiY&t3@pwDL$JXX;gxE@>K1E#6@4?qK z-x*b%wj!vu?Ud7)j%8G_QJyEi9qJ}i>sBZ%{GGUkkm1`YYD;U+qtGa~Urgu1Od#f} z*GZZO)E`b`*IHi&I>kFBsZ#1$&(JQ^|73Jq=o~2Heqb%{&P(BSek>FwIz}VZg8JTv z{mW;EjH|TQOgj^#o}4&wvs{4ExW@U@+!Ba_c;ov@jqDB6L|k9k)FVerL1$gU!!opXk(9?C1+mBS<&=hrEU|vqJPVS%Bhp8rMtMM z=VwV6cy2n#sMu^CiQ7n&f)b+*+%Ni+u0`^O&h(j&&8~X3tkYH5c3qS8YBXNRIu9XA z^nt@%Uh!YT?7gW0V)`$@?IVkMdP($8C%#tOoC4uU*CL1UA9gk0fxb+Tg|`~~#thae zFB6Ei*2BeqH^Jl08*;8UK|)StoX`f}A@;Fns*T=ew$2?UTv49IYl6(?(x&_h(Rubq zG%Cz7+)E$X4i+H=u0`oi#n9bf(+0L~!=Rhk=N#7>?V(7BmwEs3-qOWifZ*SS{{ibs Be0l%? literal 0 HcmV?d00001 diff --git a/bookscraper/requirements.txt b/bookscraper/requirements.txt new file mode 100644 index 0000000..c51da14 --- /dev/null +++ b/bookscraper/requirements.txt @@ -0,0 +1,5 @@ +flask +requests +beautifulsoup4 +lxml +pillow diff --git a/bookscraper/scraper/__init__.py b/bookscraper/scraper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bookscraper/scraper/book_scraper.py b/bookscraper/scraper/book_scraper.py new file mode 100644 index 0000000..56b4f25 --- /dev/null +++ b/bookscraper/scraper/book_scraper.py @@ -0,0 +1,269 @@ +import requests +import os +import time +from pathlib import Path +from bs4 import BeautifulSoup +from urllib.parse import urljoin, urlparse +from PIL import Image +from io import BytesIO +from dotenv import load_dotenv + +from scraper.logger import setup_logger, LOG_BUFFER +from scraper.utils import clean_text, load_replacements + +load_dotenv() +logger = setup_logger() + + +class Chapter: + def __init__(self, number, title, url): + self.number = number + self.title = title + self.url = url + self.text = "" + + +class BookScraper: + def __init__(self, site, url): + self.site = site + self.url = url + + self.book_title = "" + self.book_author = "" + self.book_description = "" + self.cover_url = "" + + self.chapters = [] + self.chapter_base = None + self.base_path = None + + # ENV settings + self.DRY_RUN = os.getenv("DRY_RUN", "0") == "1" + self.TEST_CHAPTER_LIMIT = int(os.getenv("TEST_CHAPTER_LIMIT", "10")) + self.MAX_VOL_SIZE = int(os.getenv("MAX_VOL_SIZE", "1500")) + self.MAX_DL_PER_SEC = int(os.getenv("MAX_DL_PER_SEC", "2")) + + # Load text replacements + self.replacements = load_replacements("replacements.txt") + + # ----------------------------------------------------- + def execute(self): + LOG_BUFFER.seek(0) + LOG_BUFFER.truncate(0) + + logger.debug("Starting scraper for %s", self.url) + soup = self.get_document(self.url) + + self.parse_title(soup) + self.parse_author(soup) + self.parse_description(soup) + self.parse_cover(soup) + self.prepare_output_folder() + + chapter_page = self.get_chapter_page(soup) + self.parse_chapter_links(chapter_page) + + if self.DRY_RUN: + logger.debug( + "DRY RUN → downloading only first %s chapters", self.TEST_CHAPTER_LIMIT) + self.get_some_chapters(self.TEST_CHAPTER_LIMIT) + else: + self.get_all_chapters() + self.split_into_volumes() + + return { + "title": self.book_title, + "debug": LOG_BUFFER.getvalue() + } + + # ----------------------------------------------------- + # NETWORK + # ----------------------------------------------------- + def get_document(self, url): + logger.debug("GET %s", url) + time.sleep(1 / max(1, self.MAX_DL_PER_SEC)) + + resp = requests.get( + url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10) + resp.encoding = self.site.encoding + + logger.debug("HTTP %s for %s", resp.status_code, url) + return BeautifulSoup(resp.text, "lxml") + + # ----------------------------------------------------- + # BASIC PARSERS (piaotia structure) + # ----------------------------------------------------- + def parse_title(self, soup): + h1 = soup.find("h1") + if h1: + self.book_title = h1.get_text(strip=True) + else: + self.book_title = "UnknownTitle" + logger.debug("Book title: %s", self.book_title) + + def parse_author(self, soup): + td = soup.find("td", string=lambda t: t and "作" in t and "者" in t) + if td: + raw = td.get_text(strip=True) + if "：" in raw: + self.book_author = raw.split("：", 1)[1].strip() + else: + self.book_author = "UnknownAuthor" + else: + self.book_author = "UnknownAuthor" + logger.debug("Book author: %s", self.book_author) + + def parse_description(self, soup): + span = soup.find("span", string=lambda t: t and "内容简介" in t) + if not span: + self.book_description = "" + return + + parts = [] + for sib in span.next_siblings: + if getattr(sib, "name", None) == "span": + break + txt = sib.get_text(strip=True) if not isinstance( + sib, str) else sib.strip() + if txt: + parts.append(txt) + + self.book_description = "\n".join(parts) + logger.debug("Description parsed (%s chars)", + len(self.book_description)) + + def parse_cover(self, soup): + selector = ( + "html > body > div:nth-of-type(6) > div:nth-of-type(2) > div > table " + "> tr:nth-of-type(4) > td:nth-of-type(1) > table > tr:nth-of-type(1) " + "> td:nth-of-type(2) > a:nth-of-type(1) > img" + ) + img = soup.select_one(selector) + if img: + self.cover_url = urljoin(self.site.root, img.get("src")) + else: + logger.debug("Cover not found!") + logger.debug("Cover URL = %s", self.cover_url) + + # ----------------------------------------------------- + def prepare_output_folder(self): + output_root = os.getenv("OUTPUT_DIR", "./output") + self.base_path = Path(output_root) / self.book_title / self.site.name + self.base_path.mkdir(parents=True, exist_ok=True) + logger.debug("Output directory: %s", self.base_path) + + if self.cover_url: + self.save_image(self.cover_url, self.base_path / "cover.jpg") + + def save_image(self, url, path): + logger.debug("Downloading cover: %s", url) + resp = requests.get( + url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10) + if resp.status_code == 200: + img = Image.open(BytesIO(resp.content)) + img.save(path) + logger.debug("Cover saved to %s", path) + + # ----------------------------------------------------- + # CHAPTER PAGE + # ----------------------------------------------------- + def get_chapter_page(self, soup): + node = soup.select_one( + "html > body > div:nth-of-type(6) > div:nth-of-type(2) > div > table") + link = node.select_one("a") + href = link.get("href") + chapter_url = urljoin(self.site.root, href) + + parsed = urlparse(chapter_url) + base = parsed.path.rsplit("/", 1)[0] + "/" + self.chapter_base = f"{parsed.scheme}://{parsed.netloc}{base}" + + logger.debug("Chapter index URL = %s", chapter_url) + logger.debug("CHAPTER_BASE = %s", self.chapter_base) + + return self.get_document(chapter_url) + + def parse_chapter_links(self, soup): + container = soup.select_one("div.centent") + links = container.select("ul li a[href]") + + for i, a in enumerate(links, 1): + href = a.get("href") + if not href.endswith(".html"): + continue + + abs_url = urljoin(self.chapter_base, href) + title = a.get_text(strip=True) + self.chapters.append(Chapter(i, title, abs_url)) + + logger.debug("Total chapters: %s", len(self.chapters)) + + # ----------------------------------------------------- + # DOWNLOAD CHAPTERS + # ----------------------------------------------------- + def get_all_chapters(self): + for ch in self.chapters: + ch.text = self.fetch_chapter(ch) + logger.debug("CH %s length = %s", ch.number, len(ch.text)) + + def get_some_chapters(self, limit): + for ch in self.chapters[:limit]: + ch.text = self.fetch_chapter(ch) + filename = self.base_path / f"{ch.number:05d}_{ch.title}.txt" + filename.write_text(ch.text, encoding="utf-8") + logger.debug("Saved test chapter: %s", filename) + + def fetch_chapter(self, ch): + soup = self.get_document(ch.url) + text = self.parse_chapter_text(soup) + return clean_text(text, self.replacements) + + def parse_chapter_text(self, soup): + body = soup.body + h1 = body.find("h1") + + parts = [] + collecting = False + + for sib in h1.next_siblings: + if getattr(sib, "get", None) and sib.get("class") == ["bottomlink"]: + break + if getattr(sib, "get", None) and sib.get("class") == ["toplink"]: + continue + if getattr(sib, "name", None) in ["script", "style"]: + continue + + if not collecting: + if getattr(sib, "name", None) == "br": + collecting = True + continue + + txt = sib.strip() if isinstance(sib, str) else sib.get_text("\n", strip=True) + if txt: + parts.append(txt) + + return "\n".join(parts).strip() + + # ----------------------------------------------------- + # SPLIT VOLUMES + # ----------------------------------------------------- + def split_into_volumes(self): + logger.debug( + "Splitting into volumes (max %s chapters per volume)", self.MAX_VOL_SIZE) + + chapters = len(self.chapters) + volume = 1 + index = 0 + + while index < chapters: + chunk = self.chapters[index:index + self.MAX_VOL_SIZE] + volume_dir = self.base_path / f"v{volume}" + volume_dir.mkdir(exist_ok=True) + + for ch in chunk: + filename = volume_dir / f"{ch.number:05d}_{ch.title}.txt" + filename.write_text(ch.text, encoding="utf-8") + + logger.debug("Volume %s saved (%s chapters)", volume, len(chunk)) + volume += 1 + index += self.MAX_VOL_SIZE diff --git a/bookscraper/scraper/logger.py b/bookscraper/scraper/logger.py new file mode 100644 index 0000000..f70d0d5 --- /dev/null +++ b/bookscraper/scraper/logger.py @@ -0,0 +1,27 @@ +# scraper/logger.py +import logging +from io import StringIO + +# In-memory buffer returned to web UI +LOG_BUFFER = StringIO() + + +def setup_logger(): + logger = logging.getLogger("bookscraper") + logger.setLevel(logging.DEBUG) + logger.handlers = [] # voorkomen dubbele handlers bij reload + + # Console handler + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + ch.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + + # Buffer handler for returning to UI + mh = logging.StreamHandler(LOG_BUFFER) + mh.setLevel(logging.DEBUG) + mh.setFormatter(logging.Formatter("[%(levelname)s] %(message)s")) + + logger.addHandler(ch) + logger.addHandler(mh) + + return logger diff --git a/bookscraper/scraper/sites.py b/bookscraper/scraper/sites.py new file mode 100644 index 0000000..89d3451 --- /dev/null +++ b/bookscraper/scraper/sites.py @@ -0,0 +1,11 @@ +class BookSite: + def __init__(self): + self.name = "piaotian" + self.root = "https://www.ptwxz.com" + self.chapter_list_selector = "div.centent" + self.encoding = "gb2312" + self.replacements = { + "  ": "\n", + "手机用户请访问http://m.piaotian.net": "", + "(新飘天文学www.piaotian.cc )": "", + } diff --git a/bookscraper/scraper/utils.py b/bookscraper/scraper/utils.py new file mode 100644 index 0000000..dfe9c2a --- /dev/null +++ b/bookscraper/scraper/utils.py @@ -0,0 +1,22 @@ +import os + +# scraper/utils.py + + +def load_replacements(path): + repl = {} + if not path or not os.path.exists(path): + return repl + + with open(path, encoding="utf-8") as f: + for line in f: + if "=>" in line: + k, v = line.strip().split("=>", 1) + repl[k.strip()] = v.strip() + return repl + + +def clean_text(text, repl_dict): + for src, tgt in repl_dict.items(): + text = text.replace(src, tgt) + return text diff --git a/bookscraper/static/.keep b/bookscraper/static/.keep new file mode 100644 index 0000000..e69de29 diff --git a/bookscraper/templates/index.html b/bookscraper/templates/index.html new file mode 100644 index 0000000..03526d9 --- /dev/null +++ b/bookscraper/templates/index.html @@ -0,0 +1,22 @@ + + + + Book Scraper + + + +

Book Scraper

+ +{% if error %} +

{{ error }}

+{% endif %} + +
+

+ +

+ +
+ + + diff --git a/bookscraper/templates/result.html b/bookscraper/templates/result.html new file mode 100644 index 0000000..a1e0a56 --- /dev/null +++ b/bookscraper/templates/result.html @@ -0,0 +1,18 @@ + + + + Scrape Done + + + +

Scrape Complete

+ +

Book title: {{ title }}

+ +

Output folder:

+
{{ basepath }}
+ +Scrape another book + + + diff --git a/bookscraper/text_replacements.txt b/bookscraper/text_replacements.txt new file mode 100644 index 0000000..73c2339 --- /dev/null +++ b/bookscraper/text_replacements.txt @@ -0,0 +1,79 @@ +# ---------- BASIC HTML CLEANUP ---------- + = +\xa0= +\u3000= +\r= +\t= +\ufeff= +
= +
= +
= + +# dubbele spaties weg + = + +# ---------- WEBSITE NOISE ---------- +飘天文学= +飘天文学网= +小说阅读网= +阅读更多小说最新章节请返回飘天文学网首页= +返回飘天文学网首页= +永久地址：www.piaotia.com= +www.piaotia.com= +piaotia.com= +piaotian.com= +www.piaotian.com= +www.piaotian.net= + +# ---------- NAVIGATION ---------- +上一章= +下一章= +返回目录= +返回书页= +上一页= +下一页= +章节目录= +加入书架= +加入书签= +推荐本书= +我的书架= + +（快捷键 ←）= +（快捷键 →）= +快捷键= +←= +→= + +# ---------- COPYRIGHT / DISCLAIMER ---------- +重要声明= +版权= +All rights reserved= +Copyright= +Copyright ©= +版权所有= +本小说来自互联网资源，如果侵犯您的权益请联系我们= +本站立场无关= +均由网友发表或上传= + +# ---------- COMMON NOISE ---------- +广告= +广告位= +手机阅读请访问= +章节错误请点击举报= +举报原因如下= +章节错误= +报错= +错误章节= + +# ---------- ASCII CLEANUP ---------- +“=" +”=" +‘=' +’=' +—=- +–=- +…=... + +# ---------- KNOWN GARBAGE STRINGS ---------- + = += diff --git a/delfland/azure devops/toc_gen.py b/delfland/azure devops/toc_gen.py new file mode 100644 index 0000000..9cb7601 --- /dev/null +++ b/delfland/azure devops/toc_gen.py @@ -0,0 +1,53 @@ +import requests +from requests.auth import HTTPBasicAuth +import os +import argparse + +# Invoerparameters +parser = argparse.ArgumentParser( + description='Genereer TOC uit Azure DevOps Wiki-structuur') +parser.add_argument('--max-depth', type=int, default=3, + help='Maximale diepte van de TOC (standaard: 3)') +args = parser.parse_args() + +# Configuratie +organization = "hhdelfland" +project = "Delfland.EAM_OBS_beheer" +wiki = "Delfland.EAM_OBS_beheer.wiki" +pat = os.getenv( + "AZURE_PAT") or "14S2VfW2iYhpYHC90zL64JHVy9Fst10qIbg2Dw5erzPT3FH8x6J9JQQJ99BEACAAAAA3TYdMAAASAZDO43sH" + +url = f"https://dev.azure.com/{organization}/{project}/_apis/wiki/wikis/{wiki}/pages?api-version=7.1-preview.1&recursionLevel=full" +auth = HTTPBasicAuth('', pat) + +# API-aanroep +response = requests.get(url, auth=auth) + +# Functie om TOC te genereren met max diepte + + +def generate_toc(pages, depth=1, max_depth=3): + if depth > max_depth: + return "" + + toc = "" + for page in sorted(pages, key=lambda p: p.get("order", 0)): + title = page["path"].split("/")[-1].replace("-", " ") + link = page["path"].strip("/").replace(" ", "%20") + ".md" + indent = " " * (depth - 1) + toc += f"{indent}- [{title}]({link})\n" + if "subPages" in page and page["subPages"]: + toc += generate_toc(page["subPages"], depth + 1, max_depth) + return toc + + +# Resultaat tonen +if response.status_code == 200: + data = response.json() + toc = generate_toc(data.get("subPages", []), + depth=5, max_depth=args.max_depth) + print("# Wiki TOC\n") + print(toc) +else: + print("Fout bij ophalen wiki-pagina's:", response.status_code) + print(response.text) diff --git a/delfland/azure devops/toc_gen.txt b/delfland/azure devops/toc_gen.txt new file mode 100644 index 0000000..8c4feb8 --- /dev/null +++ b/delfland/azure devops/toc_gen.txt @@ -0,0 +1,36 @@ +import requests +from requests.auth import HTTPBasicAuth +import os + +organization = "hhdelfland" +project = "Delfland.EAM_OBS_beheer" +wiki = "Delfland.EAM_OBS_beheer.wiki" +pat = os.getenv( + "AZURE_PAT") or "PLAK JE ACCESS TOKEN HIER TUSSEN DE AANHALINGSTEKENS" + +url = f"https://dev.azure.com/{organization}/{project}/_apis/wiki/wikis/{wiki}/pages?api-version=7.1-preview.1&recursionLevel=full" +auth = HTTPBasicAuth('', pat) + +response = requests.get(url, auth=auth) + + +def generate_toc(pages, indent=0): + toc = "" + for page in pages: + title = page["path"].split("/")[-1] + url = page.get("remoteUrl") + if url: + toc += " " * indent + f"- [{title}]({url})\n" + if "subPages" in page: + toc += generate_toc(page["subPages"], indent + 1) + return toc + + +if response.status_code == 200: + data = response.json() + toc = generate_toc(data.get("subPages", [])) + print("# Wiki TOC\n") + print(toc) +else: + print("Fout bij ophalen wiki-pagina's:", response.status_code) + print(response.text) diff --git a/delfland/init.sh b/delfland/init.sh new file mode 100644 index 0000000..4b6721c --- /dev/null +++ b/delfland/init.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# Projectnaam +PROJECT_NAME="po-quest" + +# Maak de projectmappen aan +echo "📁 Maken van projectstructuur..." +mkdir -p $PROJECT_NAME +mkdir -p $PROJECT_NAME/routes +mkdir -p $PROJECT_NAME/templates +mkdir -p $PROJECT_NAME/static/css +mkdir -p $PROJECT_NAME/static/js + +# Maak bestanden aan in de hoofdmap +touch $PROJECT_NAME/app.py +touch $PROJECT_NAME/config.py +touch $PROJECT_NAME/extensions.py +touch $PROJECT_NAME/models.py +touch $PROJECT_NAME/forms.py +touch $PROJECT_NAME/database.py +touch $PROJECT_NAME/requirements.txt +touch $PROJECT_NAME/.gitignore + +# Maak de bestanden in de routes-map +touch $PROJECT_NAME/routes/__init__.py +touch $PROJECT_NAME/routes/main.py +touch $PROJECT_NAME/routes/admin.py + +# Maak de HTML templates aan +touch $PROJECT_NAME/templates/base.html +touch $PROJECT_NAME/templates/index.html +touch $PROJECT_NAME/templates/admin.html +touch $PROJECT_NAME/templates/edit_question.html +touch $PROJECT_NAME/templates/edit_choice.html + +# Schrijf de benodigde pakketten naar requirements.txt +echo "flask" > $PROJECT_NAME/requirements.txt +echo "flask_sqlalchemy" >> $PROJECT_NAME/requirements.txt +echo "flask_wtf" >> $PROJECT_NAME/requirements.txt +echo "wtforms" >> $PROJECT_NAME/requirements.txt + +# Maak een .gitignore bestand +echo "__pycache__/" > $PROJECT_NAME/.gitignore +echo "*.sqlite3" >> $PROJECT_NAME/.gitignore +echo "*.db" >> $PROJECT_NAME/.gitignore + +echo "✅ Setup voltooid! Je kunt starten met ontwikkelen in de map '$PROJECT_NAME'." diff --git a/delfland/po-quest/.gitignore b/delfland/po-quest/.gitignore new file mode 100644 index 0000000..c57a982 --- /dev/null +++ b/delfland/po-quest/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.sqlite3 +*.db diff --git a/delfland/po-quest/app.py b/delfland/po-quest/app.py new file mode 100644 index 0000000..dae6874 --- /dev/null +++ b/delfland/po-quest/app.py @@ -0,0 +1,17 @@ +from flask import Flask +from extensions import db +from routes.admin import admin_bp + +app = Flask(__name__) +app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///po_quest.db' +app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False +app.secret_key = 'supersecretkey' + +# Initialiseer de database +db.init_app(app) + +# Registreer de blueprint +app.register_blueprint(admin_bp) + +if __name__ == "__main__": + app.run(debug=True) diff --git a/delfland/po-quest/config.py b/delfland/po-quest/config.py new file mode 100644 index 0000000..0bd98d9 --- /dev/null +++ b/delfland/po-quest/config.py @@ -0,0 +1,7 @@ +import os + + +class Config: + SECRET_KEY = os.getenv("SECRET_KEY", "supersecretkey") + SQLALCHEMY_DATABASE_URI = "sqlite:///database.db" + SQLALCHEMY_TRACK_MODIFICATIONS = False diff --git a/delfland/po-quest/database.py b/delfland/po-quest/database.py new file mode 100644 index 0000000..11912df --- /dev/null +++ b/delfland/po-quest/database.py @@ -0,0 +1,6 @@ +from app import app +from extensions import db + +with app.app_context(): + db.create_all() + print("✅ Database is aangemaakt!") diff --git a/delfland/po-quest/extensions.py b/delfland/po-quest/extensions.py new file mode 100644 index 0000000..f0b13d6 --- /dev/null +++ b/delfland/po-quest/extensions.py @@ -0,0 +1,3 @@ +from flask_sqlalchemy import SQLAlchemy + +db = SQLAlchemy() diff --git a/delfland/po-quest/forms.py b/delfland/po-quest/forms.py new file mode 100644 index 0000000..e20ebd1 --- /dev/null +++ b/delfland/po-quest/forms.py @@ -0,0 +1,14 @@ +# forms.py +from flask_wtf import FlaskForm +from wtforms import StringField, SelectField +from wtforms.validators import DataRequired +from models import Question # Zorg ervoor dat Question correct geïmporteerd wordt + + +class QuestionForm(FlaskForm): + text = StringField('Vraag', validators=[DataRequired()]) + + +class ChoiceForm(FlaskForm): + text = StringField('Keuze', validators=[DataRequired()]) + next_question = SelectField('Volgende vraag', coerce=int) diff --git a/delfland/po-quest/models.py b/delfland/po-quest/models.py new file mode 100644 index 0000000..83f8a34 --- /dev/null +++ b/delfland/po-quest/models.py @@ -0,0 +1,20 @@ +from extensions import db + + +class Question(db.Model): + id = db.Column(db.Integer, primary_key=True) + text = db.Column(db.String(255), nullable=False) + choices = db.relationship('Choice', backref='question', lazy=True) + + +class Choice(db.Model): + id = db.Column(db.Integer, primary_key=True) + text = db.Column(db.String(255), nullable=False) + question_id = db.Column(db.Integer, db.ForeignKey( + 'question.id'), nullable=False) + + # Nieuwe kolom voor de volgende vraag + next_question_id = db.Column( + db.Integer, db.ForeignKey('question.id'), nullable=True) + next_question = db.relationship('Question', foreign_keys=[ + next_question_id], backref='previous_choices') diff --git a/delfland/po-quest/requirements.txt b/delfland/po-quest/requirements.txt new file mode 100644 index 0000000..0a535c7 --- /dev/null +++ b/delfland/po-quest/requirements.txt @@ -0,0 +1,4 @@ +flask +flask_sqlalchemy +flask_wtf +wtforms diff --git a/delfland/po-quest/routes/__init__.py b/delfland/po-quest/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/delfland/po-quest/routes/admin.py b/delfland/po-quest/routes/admin.py new file mode 100644 index 0000000..83e6895 --- /dev/null +++ b/delfland/po-quest/routes/admin.py @@ -0,0 +1,94 @@ +from flask import Blueprint, render_template, request, redirect, url_for +from flask_sqlalchemy import SQLAlchemy +from models import db, Question, Choice +from forms import ChoiceForm, QuestionForm + +# Maak de blueprint aan +admin_bp = Blueprint('admin', __name__) + +# Route voor het admin-dashboard + + +@admin_bp.route('/') +def admin_dashboard(): + # Haal alle vragen en keuzes op uit de database + questions = Question.query.all() + return render_template('admin_dashboard.html', questions=questions) + +# Route voor het toevoegen van een vraag + + +@admin_bp.route('/add_question', methods=['GET', 'POST']) +def add_question(): + form = QuestionForm() + if form.validate_on_submit(): + question_text = form.text.data + new_question = Question(text=question_text) + db.session.add(new_question) + db.session.commit() + return redirect(url_for('admin.admin_dashboard')) + return render_template('add_question.html', form=form) + +# Route voor het bewerken van een vraag + + +@admin_bp.route('/edit_question/', methods=['GET', 'POST']) +def edit_question(question_id): + question = Question.query.get_or_404(question_id) + form = QuestionForm(obj=question) + if form.validate_on_submit(): + question.text = form.text.data + db.session.commit() + return redirect(url_for('admin.admin_dashboard')) + return render_template('edit_question.html', form=form, question=question) + +# Route voor het verwijderen van een vraag + + +@admin_bp.route('/delete_question/', methods=['POST']) +def delete_question(question_id): + question = Question.query.get_or_404(question_id) + db.session.delete(question) + db.session.commit() + return redirect(url_for('admin.admin_dashboard')) + +# Route voor het toevoegen van een keuze aan een vraag + + +@admin_bp.route('/add_choice/', methods=['GET', 'POST']) +def add_choice(question_id): + form = ChoiceForm() + question = Question.query.get_or_404(question_id) + if form.validate_on_submit(): + choice_text = form.text.data + next_question_id = form.next_question.data if form.next_question.data else None + new_choice = Choice( + text=choice_text, question_id=question.id, next_question_id=next_question_id) + db.session.add(new_choice) + db.session.commit() + return redirect(url_for('admin.edit_question', question_id=question.id)) + return render_template('add_choice.html', form=form, question=question) + +# Route voor het bewerken van een keuze + + +@admin_bp.route('/edit_choice/', methods=['GET', 'POST']) +def edit_choice(choice_id): + choice = Choice.query.get_or_404(choice_id) + form = ChoiceForm(obj=choice) + if form.validate_on_submit(): + choice.text = form.text.data + choice.next_question_id = form.next_question.data if form.next_question.data else None + db.session.commit() + return redirect(url_for('admin.edit_question', question_id=choice.question_id)) + return render_template('edit_choice.html', form=form, choice=choice) + +# Route voor het verwijderen van een keuze + + +@admin_bp.route('/delete_choice/', methods=['POST']) +def delete_choice(choice_id): + choice = Choice.query.get_or_404(choice_id) + db.session.delete(choice) + db.session.commit() + return redirect(url_for('admin.edit_question', question_id=choice.question_id)) diff --git a/delfland/po-quest/routes/main.py b/delfland/po-quest/routes/main.py new file mode 100644 index 0000000..4d145db --- /dev/null +++ b/delfland/po-quest/routes/main.py @@ -0,0 +1,10 @@ +from flask import Blueprint, render_template +from models import Question + +main_bp = Blueprint("main", __name__) + + +@main_bp.route("/") +def index(): + first_question = Question.query.first() + return render_template("index.html", question=first_question) diff --git a/delfland/po-quest/templates/admin.html b/delfland/po-quest/templates/admin.html new file mode 100644 index 0000000..1a21b42 --- /dev/null +++ b/delfland/po-quest/templates/admin.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} + +{% block content %} +

Beheer Vragenlijst

+ Nieuwe Vraag Toevoegen + +{% endblock %} diff --git a/delfland/po-quest/templates/base.html b/delfland/po-quest/templates/base.html new file mode 100644 index 0000000..538b382 --- /dev/null +++ b/delfland/po-quest/templates/base.html @@ -0,0 +1,26 @@ + + + + + + Vragenlijst + + + +
+

Vragenlijst App

+ +
+ +
+ {% block content %}{% endblock %} +
+ +
+

© 2025 Mijn App

+
+ + diff --git a/delfland/po-quest/templates/edit_choice.html b/delfland/po-quest/templates/edit_choice.html new file mode 100644 index 0000000..68da7e0 --- /dev/null +++ b/delfland/po-quest/templates/edit_choice.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block content %} +

Keuze Bewerken

+
+ {{ form.hidden_tag() }} +
+ + {{ form.text() }} +
+ +
+{% endblock %} diff --git a/delfland/po-quest/templates/edit_question.html b/delfland/po-quest/templates/edit_question.html new file mode 100644 index 0000000..312a51f --- /dev/null +++ b/delfland/po-quest/templates/edit_question.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block content %} +

Vraag Bewerken

+
+ {{ form.hidden_tag() }} +
+ + {{ form.text() }} +
+ +
+{% endblock %} diff --git a/delfland/po-quest/templates/index.html b/delfland/po-quest/templates/index.html new file mode 100644 index 0000000..003501d --- /dev/null +++ b/delfland/po-quest/templates/index.html @@ -0,0 +1,13 @@ +{% extends "base.html" %} + +{% block content %} +

{{ question.text }}

+
+ {% for choice in question.choices %} +
+ {% endfor %} + +
+{% endblock %} diff --git a/delfland/webhooktest/Dockerfile b/delfland/webhooktest/Dockerfile new file mode 100644 index 0000000..f22491f --- /dev/null +++ b/delfland/webhooktest/Dockerfile @@ -0,0 +1,10 @@ +FROM python:3.11-slim + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ app/ + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/delfland/webhooktest/README.md b/delfland/webhooktest/README.md new file mode 100644 index 0000000..e69de29 diff --git a/delfland/webhooktest/app/__init__.py b/delfland/webhooktest/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/delfland/webhooktest/app/logger.py b/delfland/webhooktest/app/logger.py new file mode 100644 index 0000000..1066cdc --- /dev/null +++ b/delfland/webhooktest/app/logger.py @@ -0,0 +1,8 @@ +import logging + + +def setup_logging(): + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + ) diff --git a/delfland/webhooktest/app/main.py b/delfland/webhooktest/app/main.py new file mode 100644 index 0000000..db852ac --- /dev/null +++ b/delfland/webhooktest/app/main.py @@ -0,0 +1,7 @@ +from fastapi import FastAPI +from .routes import router +from .logger import setup_logging + +app = FastAPI() +setup_logging() +app.include_router(router) diff --git a/delfland/webhooktest/app/routes.py b/delfland/webhooktest/app/routes.py new file mode 100644 index 0000000..5ec0d17 --- /dev/null +++ b/delfland/webhooktest/app/routes.py @@ -0,0 +1,85 @@ +from fastapi import APIRouter, Request, Form, WebSocket, WebSocketDisconnect +from fastapi.responses import HTMLResponse +from fastapi.templating import Jinja2Templates +from fastapi.security import APIKeyHeader +from dotenv import load_dotenv +import os +import logging +from .storage import store_message, get_all_messages + +load_dotenv() + +API_KEY = os.getenv("API_KEY") +api_key_header = APIKeyHeader(name="x-api-key", auto_error=False) + +router = APIRouter() +templates = Jinja2Templates(directory="app/templates") + +# Live WebSocket connecties +active_connections = [] + + +async def validate_api_key(api_key: str = None): + if api_key != API_KEY: + raise Exception("Unauthorized") + + +async def broadcast(msg: str): + for connection in active_connections: + try: + await connection.send_text(msg) + except: + continue + + +@router.api_route("/webhook", methods=["GET", "POST"]) +async def webhook(request: Request, api_key: str = api_key_header): + await validate_api_key(api_key) + body = await request.body() + log_entry = { + "method": request.method, + "headers": dict(request.headers), + "body": body.decode() + } + logging.info(f"Webhook received: {log_entry}") + store_message(log_entry) + await broadcast(str(log_entry)) # stuur live update via websocket + return {"status": "ok"} + + +@router.get("/trace", response_class=HTMLResponse) +async def trace_page(request: Request): + return templates.TemplateResponse("trace.html", { + "request": request, + "messages": get_all_messages() + }) + + +@router.websocket("/ws/trace") +async def websocket_endpoint(websocket: WebSocket): + await websocket.accept() + active_connections.append(websocket) + try: + while True: + await websocket.receive_text() # optioneel: keep-alive ping + except WebSocketDisconnect: + active_connections.remove(websocket) + + +@router.get("/send", response_class=HTMLResponse) +async def send_form(request: Request): + return templates.TemplateResponse("send.html", {"request": request}) + + +@router.post("/send") +async def send_post(request: Request, url: str = Form(...), body: str = Form(...), key: str = Form(...)): + import httpx + headers = {"x-api-key": key} + try: + response = httpx.post(url, content=body, headers=headers) + return { + "status_code": response.status_code, + "response": response.text + } + except Exception as e: + return {"error": str(e)} diff --git a/delfland/webhooktest/app/storage.py b/delfland/webhooktest/app/storage.py new file mode 100644 index 0000000..bb94a35 --- /dev/null +++ b/delfland/webhooktest/app/storage.py @@ -0,0 +1,9 @@ +messages = [] + + +def store_message(msg: dict): + messages.append(msg) + + +def get_all_messages(): + return list(messages) diff --git a/delfland/webhooktest/app/templates/send.html b/delfland/webhooktest/app/templates/send.html new file mode 100644 index 0000000..416dff2 --- /dev/null +++ b/delfland/webhooktest/app/templates/send.html @@ -0,0 +1,10 @@ + + +

Webhookbericht versturen

+
+
+
+
+ +
+ diff --git a/delfland/webhooktest/app/templates/trace.html b/delfland/webhooktest/app/templates/trace.html new file mode 100644 index 0000000..5a4d274 --- /dev/null +++ b/delfland/webhooktest/app/templates/trace.html @@ -0,0 +1,28 @@ + + + + Live Webhook Trace + + + +

Ontvangen Webhookberichten (live)

+
    + {% for msg in messages %} + 
  • {{ msg }}
    • + {% endfor %} +
+ + + + diff --git a/delfland/webhooktest/docker-compose.yml b/delfland/webhooktest/docker-compose.yml new file mode 100644 index 0000000..576863d --- /dev/null +++ b/delfland/webhooktest/docker-compose.yml @@ -0,0 +1,19 @@ +# docker compose -f docker-compose.yml up -d --build + +version: "3.3" +services: + webhooktest: + restart: always + container_name: webhooktest + environment: + - PUID=1000 + - PGID=1000 + - TZ=Europe/Amsterdam + ports: + - target: 8000 + published: 5905 + protocol: tcp + build: + dockerfile: ./Dockerfile + env_file: + - .env diff --git a/delfland/webhooktest/init.sh b/delfland/webhooktest/init.sh new file mode 100644 index 0000000..781e388 --- /dev/null +++ b/delfland/webhooktest/init.sh @@ -0,0 +1,19 @@ +# Maak hoofddirectory aan +mkdir -p app/templates + +# Ga naar projectmap + +# Maak lege bestanden aan in de root +touch .env Dockerfile requirements.txt README.md + +# Ga naar de app-map +cd app + +# Maak Python-bestanden aan +touch __init__.py main.py logger.py routes.py storage.py + +# Ga naar templates-map +cd templates + +# Maak de HTML-templates aan +touch trace.html send.html diff --git a/delfland/webhooktest/requirements.txt b/delfland/webhooktest/requirements.txt new file mode 100644 index 0000000..15d538f --- /dev/null +++ b/delfland/webhooktest/requirements.txt @@ -0,0 +1,5 @@ +fastapi +uvicorn +python-dotenv +jinja2 +python-multipart \ No newline at end of file diff --git a/solarforecast/README.md b/solarforecast/README.md new file mode 100644 index 0000000..e69de29 diff --git a/solarforecast/backend/db.py b/solarforecast/backend/db.py new file mode 100644 index 0000000..0e9ee6a --- /dev/null +++ b/solarforecast/backend/db.py @@ -0,0 +1,97 @@ +import os +import mysql.connector +from dotenv import load_dotenv +from datetime import datetime + +load_dotenv(dotenv_path="../.env") + +db_config = { + "host": os.getenv("MYSQL_HOST"), + "port": int(os.getenv("MYSQL_PORT")), + "user": os.getenv("MYSQL_USER"), + "password": os.getenv("MYSQL_PASSWORD"), + "database": os.getenv("MYSQL_DATABASE"), +} + + +def parse_datetime_safe(dt_string): + try: + return datetime.fromisoformat(dt_string.replace("Z", "").replace("+00:00", "")) + except Exception: + return None + + +def parse_float_safe(value): + try: + return float(value) + except Exception: + return None + + +def init_db(): + """Creëer ruwe datatabel met unieke forecast_time.""" + conn = mysql.connector.connect(**db_config) + cursor = conn.cursor() + cursor.execute(""" + CREATE TABLE IF NOT EXISTS solar_raw_data ( + id INT AUTO_INCREMENT PRIMARY KEY, + forecast_time DATETIME NOT NULL UNIQUE, + valid_to DATETIME, + capacity FLOAT, + volume FLOAT, + percentage FLOAT, + emission FLOAT, + emission_factor FLOAT, + last_update DATETIME + ) + """) + conn.commit() + cursor.close() + conn.close() + + +def insert_forecast_records(records): + """Voegt ruwe forecast records toe, met veilige parsing en validatie.""" + conn = mysql.connector.connect(**db_config) + cursor = conn.cursor() + + # Haal reeds bekende timestamps op + cursor.execute("SELECT forecast_time FROM solar_raw_data") + existing_timestamps = set(row[0] for row in cursor.fetchall()) + + new_rows = 0 + for record in records: + forecast_time = parse_datetime_safe(record.get("validfrom", "")) + if not forecast_time or forecast_time in existing_timestamps: + continue + + try: + cursor.execute(""" + INSERT INTO solar_raw_data ( + forecast_time, + valid_to, + capacity, + volume, + percentage, + emission, + emission_factor, + last_update + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s) + """, ( + forecast_time, + parse_datetime_safe(record.get("validto")), + parse_float_safe(record.get("capacity")), + parse_float_safe(record.get("volume")), + parse_float_safe(record.get("percentage")), + parse_float_safe(record.get("emission")), + parse_float_safe(record.get("emissionfactor")), + parse_datetime_safe(record.get("lastupdate")) + )) + new_rows += 1 + except Exception as e: + print(f"❌ Fout bij record: {e}\nRecord: {record}") + + conn.commit() + cursor.close() + conn.close() + print(f"✅ {new_rows} nieuwe records opgeslagen.") diff --git a/solarforecast/backend/fetch_ned_data.py b/solarforecast/backend/fetch_ned_data.py new file mode 100644 index 0000000..8454923 --- /dev/null +++ b/solarforecast/backend/fetch_ned_data.py @@ -0,0 +1,52 @@ +import os +import requests +from dotenv import load_dotenv +from datetime import datetime, timedelta +from db import init_db, insert_forecast_records + +load_dotenv(dotenv_path="../.env") + +NED_API_KEY = os.getenv("NED_API_KEY") + + +def fetch_forecast(): + url = "https://api.ned.nl/v1/utilizations" + headers = { + "X-AUTH-TOKEN": NED_API_KEY, + "Accept": "application/ld+json" + } + + today = datetime.utcnow().date() + tomorrow = today + timedelta(days=1) + + params = { + "point": 9, + "type": 2, + "granularity": 3, + "granularitytimezone": 1, + "classification": 2, + "activity": 1, + "validfrom[after]": today.strftime("%Y-%m-%d"), + "validfrom[strictly_before]": tomorrow.strftime("%Y-%m-%d") + } + + response = requests.get(url, headers=headers, params=params) + response.raise_for_status() + return response.json() + + +def main(): + print("📡 Ophalen zonneproductievoorspelling van NED.nl") + data = fetch_forecast() + records = data.get("hydra:member", []) + print(records) + print(f"Gevonden records: {len(records)}") + + init_db() + insert_forecast_records(records) + + print("✅ Data opgeslagen in database") + + +if __name__ == "__main__": + main() diff --git a/solarforecast/requirements.txt b/solarforecast/requirements.txt new file mode 100644 index 0000000..09f2bbe --- /dev/null +++ b/solarforecast/requirements.txt @@ -0,0 +1,3 @@ +requests +python-dotenv +mysql-connector-python