From e53b8bd227728d23760d673eab305cc0467bb0e0 Mon Sep 17 00:00:00 2001 From: Ankita Hegde Date: Sun, 29 Oct 2023 19:07:25 +0530 Subject: [PATCH] assignment of session4 --- myworld1/breakpoint_demo.py | 16 ++ myworld1/db.sqlite3 | Bin 0 -> 135168 bytes myworld1/docker-compose.yml | 63 ++++++++ myworld1/dockerfiles/Dockerfile | 14 ++ myworld1/manage.py | 22 +++ myworld1/members/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 169 bytes .../members/__pycache__/admin.cpython-310.pyc | Bin 0 -> 2401 bytes .../members/__pycache__/apps.cpython-310.pyc | Bin 0 -> 2844 bytes .../__pycache__/models.cpython-310.pyc | Bin 0 -> 2741 bytes .../members/__pycache__/tasks.cpython-310.pyc | Bin 0 -> 2290 bytes .../members/__pycache__/urls.cpython-310.pyc | Bin 0 -> 706 bytes .../members/__pycache__/views.cpython-310.pyc | Bin 0 -> 3194 bytes myworld1/members/admin.py | 60 ++++++++ myworld1/members/apps.py | 108 ++++++++++++++ myworld1/members/migrations/0001_initial.py | 22 +++ .../0002_students_delete_members.py | 28 ++++ myworld1/members/migrations/0003_blog.py | 24 +++ .../members/migrations/0004_blog_content.py | 19 +++ myworld1/members/migrations/0005_blog_path.py | 19 +++ .../migrations/0006_alter_blog_path.py | 18 +++ ...og_author_alter_blog_blog_time_and_more.py | 38 +++++ ...og_author_alter_blog_blog_time_and_more.py | 48 ++++++ .../migrations/0009_alter_blog_path.py | 18 +++ ...og_author_alter_blog_blog_time_and_more.py | 38 +++++ ...og_author_alter_blog_blog_time_and_more.py | 49 ++++++ .../migrations/0012_blog_recommended.py | 19 +++ .../migrations/0013_job_jobstats_joblogs.py | 48 ++++++ myworld1/members/migrations/__init__.py | 0 .../__pycache__/0001_initial.cpython-310.pyc | Bin 0 -> 709 bytes ...02_students_delete_members.cpython-310.pyc | Bin 0 -> 940 bytes .../__pycache__/0003_blog.cpython-310.pyc | Bin 0 -> 868 bytes .../0004_blog_content.cpython-310.pyc | Bin 0 -> 596 bytes .../0005_blog_path.cpython-310.pyc | Bin 0 -> 598 bytes .../0006_alter_blog_path.cpython-310.pyc | Bin 0 -> 578 bytes ...er_blog_blog_time_and_more.cpython-310.pyc | Bin 0 -> 859 bytes ...er_blog_blog_time_and_more.cpython-310.pyc | Bin 0 -> 994 bytes .../0009_alter_blog_path.cpython-310.pyc | Bin 0 -> 616 bytes ...er_blog_blog_time_and_more.cpython-310.pyc | Bin 0 -> 859 bytes ...er_blog_blog_time_and_more.cpython-310.pyc | Bin 0 -> 976 bytes .../0012_blog_recommended.cpython-310.pyc | Bin 0 -> 647 bytes .../0013_job_jobstats_joblogs.cpython-310.pyc | Bin 0 -> 1629 bytes .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 180 bytes myworld1/members/models.py | 64 ++++++++ myworld1/members/tasks.py | 83 +++++++++++ myworld1/members/templates/.myfirst.html.swp | Bin 0 -> 12288 bytes myworld1/members/templates/myfirst.html | 9 ++ myworld1/members/tests.py | 3 + myworld1/members/urls.py | 20 +++ myworld1/members/views.py | 104 +++++++++++++ myworld1/myworld1/__init__.py | 5 + .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 238 bytes .../__pycache__/celery.cpython-310.pyc | Bin 0 -> 733 bytes .../__pycache__/settings.cpython-310.pyc | Bin 0 -> 2769 bytes .../myworld1/__pycache__/urls.cpython-310.pyc | Bin 0 -> 1018 bytes .../myworld1/__pycache__/wsgi.cpython-310.pyc | Bin 0 -> 575 bytes myworld1/myworld1/asgi.py | 16 ++ myworld1/myworld1/celery.py | 35 +++++ myworld1/myworld1/settings.py | 141 ++++++++++++++++++ myworld1/myworld1/urls.py | 23 +++ myworld1/myworld1/wsgi.py | 16 ++ myworld1/scrapped_data.html | 37 +++++ myworld1/web_scrapper.py | 95 ++++++++++++ test_celery/__init__.py | 0 .../__pycache__/__init__.cpython-310.pyc | Bin 0 -> 164 bytes .../__pycache__/celery.cpython-310.pyc | Bin 0 -> 395 bytes .../__pycache__/run_tasks.cpython-310.pyc | Bin 0 -> 428 bytes test_celery/__pycache__/tasks.cpython-310.pyc | Bin 0 -> 502 bytes test_celery/celery.py | 9 ++ test_celery/run_tasks.py | 13 ++ test_celery/tasks.py | 11 ++ 71 files changed, 1355 insertions(+) create mode 100644 myworld1/breakpoint_demo.py create mode 100644 myworld1/db.sqlite3 create mode 100644 myworld1/docker-compose.yml create mode 100644 myworld1/dockerfiles/Dockerfile create mode 100755 myworld1/manage.py create mode 100644 myworld1/members/__init__.py create mode 100644 myworld1/members/__pycache__/__init__.cpython-310.pyc create mode 100644 myworld1/members/__pycache__/admin.cpython-310.pyc create mode 100644 myworld1/members/__pycache__/apps.cpython-310.pyc create mode 100644 myworld1/members/__pycache__/models.cpython-310.pyc create mode 100644 myworld1/members/__pycache__/tasks.cpython-310.pyc create mode 100644 myworld1/members/__pycache__/urls.cpython-310.pyc create mode 100644 myworld1/members/__pycache__/views.cpython-310.pyc create mode 100644 myworld1/members/admin.py create mode 100644 myworld1/members/apps.py create mode 100644 myworld1/members/migrations/0001_initial.py create mode 100644 myworld1/members/migrations/0002_students_delete_members.py create mode 100644 myworld1/members/migrations/0003_blog.py create mode 100644 myworld1/members/migrations/0004_blog_content.py create mode 100644 myworld1/members/migrations/0005_blog_path.py create mode 100644 myworld1/members/migrations/0006_alter_blog_path.py create mode 100644 myworld1/members/migrations/0007_alter_blog_author_alter_blog_blog_time_and_more.py create mode 100644 myworld1/members/migrations/0008_alter_blog_author_alter_blog_blog_time_and_more.py create mode 100644 myworld1/members/migrations/0009_alter_blog_path.py create mode 100644 myworld1/members/migrations/0010_alter_blog_author_alter_blog_blog_time_and_more.py create mode 100644 myworld1/members/migrations/0011_alter_blog_author_alter_blog_blog_time_and_more.py create mode 100644 myworld1/members/migrations/0012_blog_recommended.py create mode 100644 myworld1/members/migrations/0013_job_jobstats_joblogs.py create mode 100644 myworld1/members/migrations/__init__.py create mode 100644 myworld1/members/migrations/__pycache__/0001_initial.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0002_students_delete_members.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0003_blog.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0004_blog_content.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0005_blog_path.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0006_alter_blog_path.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0007_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0008_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0009_alter_blog_path.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0010_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0011_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0012_blog_recommended.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/0013_job_jobstats_joblogs.cpython-310.pyc create mode 100644 myworld1/members/migrations/__pycache__/__init__.cpython-310.pyc create mode 100644 myworld1/members/models.py create mode 100644 myworld1/members/tasks.py create mode 100644 myworld1/members/templates/.myfirst.html.swp create mode 100644 myworld1/members/templates/myfirst.html create mode 100644 myworld1/members/tests.py create mode 100644 myworld1/members/urls.py create mode 100644 myworld1/members/views.py create mode 100644 myworld1/myworld1/__init__.py create mode 100644 myworld1/myworld1/__pycache__/__init__.cpython-310.pyc create mode 100644 myworld1/myworld1/__pycache__/celery.cpython-310.pyc create mode 100644 myworld1/myworld1/__pycache__/settings.cpython-310.pyc create mode 100644 myworld1/myworld1/__pycache__/urls.cpython-310.pyc create mode 100644 myworld1/myworld1/__pycache__/wsgi.cpython-310.pyc create mode 100644 myworld1/myworld1/asgi.py create mode 100644 myworld1/myworld1/celery.py create mode 100644 myworld1/myworld1/settings.py create mode 100644 myworld1/myworld1/urls.py create mode 100644 myworld1/myworld1/wsgi.py create mode 100644 myworld1/scrapped_data.html create mode 100644 myworld1/web_scrapper.py create mode 100644 test_celery/__init__.py create mode 100644 test_celery/__pycache__/__init__.cpython-310.pyc create mode 100644 test_celery/__pycache__/celery.cpython-310.pyc create mode 100644 test_celery/__pycache__/run_tasks.cpython-310.pyc create mode 100644 test_celery/__pycache__/tasks.cpython-310.pyc create mode 100644 test_celery/celery.py create mode 100644 test_celery/run_tasks.py create mode 100644 test_celery/tasks.py diff --git a/myworld1/breakpoint_demo.py b/myworld1/breakpoint_demo.py new file mode 100644 index 0000000..6153b5d --- /dev/null +++ b/myworld1/breakpoint_demo.py @@ -0,0 +1,16 @@ +breakpoint() + +print("This is line number 1") + +a = 1 +b = 2 + +sum1 = a + b + +print(f"The sum is {sum}") + +for i in range(5): + sum1 += 1 + +print(f"The sum at the end is {sum1}") + diff --git a/myworld1/db.sqlite3 b/myworld1/db.sqlite3 new file mode 100644 index 0000000000000000000000000000000000000000..48761c3c738c239ef6b46519d7bab74fac4692c4 GIT binary patch literal 135168 zcmeI5TWlLwddFu-N;EBzN0uxzj^vnLt^6y2iTZLwRR+ufp#&gOH?HQYdSIngDQML|NEi@AQFtK` z5CkDe|BuoCjn{MZjjORie~Y&FyS)tx)3-j=!;H!InT@~xYd3T32LTWO0T2KI5C8!X z009sH0T2KI5cuc`3=es!)1kgmf&3ZyJjswT-{1Ov)_30f1Me5TTi$U0+x>6$-|0Wr z_tn1F`>s5>SSEgprA4K`*5AiXlmotmcZjYEH{5k`kJd z!c$5lAVp>)u~|7b9g-5Um=YT75l&4c4c#zurQ#l$!m-&g2ZRL`v8T zl-CYY9_lDAg+t0kFLiXQ;V5o*#PICPh5CsqvZTD&PyM{o@DnrrY-;%&3TpUool+6{@VlCg+)L30p8FC^Xl3wsoAC0<}!ge1`Hl=d4YeC|P zs2q*X9ibi?buESL9`u5i%kSCbLeufMB88)u-PB2=@k))xYqhn5w3>KSiH75sdwYb5 zOG!IMMv?L@ls21qRmPV)?S?lVR$`&}LQjw2rp6sgx(JnaX`$&sw@l`=+{* z*RpDQwUkSDuk5%?`(iBVqOK@q)0JYmGG)vPnKm zewl0^+94eU0|5{K0T2KI5C8!X009sH0T2Lz|3d;pF6X4k?(S_dxlymjIp*Ozd9=g2gO`MdwVzx{%9OG!_Ff zhK^FFZn7nicgSCmZ;~&OUn5)Ovu$U>CJ2B42!H?xfB*=900@8p2!H?xfWV^=7;=q_ zh~~l~^OyU$>(q$YSU6<4h`v$%L;r%P` z?|5JLE_=`PzuW)y{@?C@*#D{iPxPPYdxvJk3j{y_1V8`;KmY_l00cnbhd^L_v_}vp zrreu3eTyz)=g%z6X~h8B4w&0dk5HSS?9S+Uojw*|v#RfD+C5pfORv(|#TL8zo`zlE zB(*y;>1M0vZO-Us!(#jiuBw--GH+^{s?6=D{rqgbrYf72sj6XjqHbrZve}uc%sqi& zuBw-*vKcW|=H}#aYB3e-;i_zRR83{(z5rWmK0_Y^5#{H4OnGhQro@&E^cfJA!PME7 z!PIKc5Twt4h?9{XCb-R-iME@L(g!@m)6clLa(3I-l*~2)^x+L!A1+gB4+=dDhOS=`#kjFFBY-YcDfpLajY#=>r6wXJjt6ao63H zN_PbKg9+#8GXkE;h$+-=XUeoJ>9YaU(iCa8G$l4I$LT`>o{1SI&~C=$HEr1Z|ETwC z0{JeT{r^)M1Nas4Iw_F5R$2e=F5fcVktCpb1roCd@Oe7{>Fv%;&#orm8!1a*tmW1av?QSyQSYPhvuKR z|M9`-Y&bLCVQ~;?2eNL`BJ+S8^9Ja^dC;X=-tPDyd(&ox2;J zE98vLYcDSumzQ3;zqoo~p){u~oR6$*U8pUutw^(SxiWWMo-MCjGS;$-SF%?U#?pGF zaDD0g%h!$7R6@S9azR!up1-($^J1vBem8rgzOp?sB_#rqweUR@jYS@u5jmxrEM*HT zwXKAn-;PFBH&aF>et*SS-K=D`R#KZk@v)?IeyuQ@tIplfV~N$xrE8aOOZT$t(U&f- z=HtbUDJ@*PCvDCzts3cgdA)X_jglnK%HiomG$AJ>w*KEi-V(@LPA1p`0w4eaAOHd&00JNY0w4eaAOHe~jR5Zd4_l3*JRkr9 zAOHd&00JNY0w4eaAOHd&&`H4S{+@8u^*Mo@_5F?K8@+$*-Rgg%_ju1;*XR1)Yy5Nk zz_BF$g77ur`L_Ih@ASUjwhNme00JQJp$P1Ja?tHxpAvVBTrs0>YqjdCS~K*Dnyr*- zWuyK+m#J^6^i3ihib!cGt=5XUjdt^)xtsIJ<@vzv#f2NU=K~9i7v^6IjJI3u<8(Z* zwAh(xJaBHD3(IB3pAC$2c~sAMaA)?I+n-H{JDOF3`q8ppDdY?zS1KB+^_~?X9!^Kn z2}S41I*bR+QrP8p813TGCcYyJD?*D7Dt>an?SF-ezqXh7x+khTB`Z=owUX$r``CeW zcR1TmawEfd@PX9l_D@ZT4~ME+Dqp8(Yb@>c+;8exPM*Iy-@Qg@1qO1(s-D#=fyJfe zz~b$zR|D5?E?i6Aycc+J{$3z?dwFSLkvh6Izqm~6z0m|ZbKRW3IDd0~ac=&W?Nkgo zYb1dS?C4D{WMOe(c_DfA>OFJk`~`b^I)GyipbT9a+t0*4Zle>Lkq`+nKGSmwbT6OuLCyC^q&Uz`d2>fE|X` z_Fq5NHU2@x?Fqi|N{x|yS zbw{s%a#Fm!E;5epg&7<26yg>ZvV`T_(0=6GizEg zTT%kD|`Q- z|6h;3ee_2?zj@@_M+STUovs5YIDX%G&-q7=pAz42%nAP`NINemZWdm7P!BB^^g>Fn z7^?ZZZF8(O4$^k?k)j|gxr$NUJKB-g_OuA@9GBhxTS0Lr!Q+f(bjHy6q@1d6mva?e zWswq9i=?AUG#Y9(37DlaXX089cFkW86t)_&(%a5OXWgPRZhtT+K6u6+Win;8<~`cV zwe=u#+$hYte&31;g(WLwWS`EW;iHB^)CfQfzvdD zXf$42;@fJgI8|=l?vtl`{qv`H%WQ?6<>uQXJI5y7{;@G}r(nfSd8n0}@OPvYkaggZ z+FWaMlOuMH^_nN~gYlg8^s?bPcBM>PoajR9)w!%7}nlEK@MSHZ7PN33b zIYTvSG`3woeX3N-+glmiyFcm1DfX~rEw-TL@|H*?5^fR6gjBVamGMBk2zo?IS96;* znA(1bj|gkDRd)@f(v$L`N!>$OaOdneomii2nOL`C{c5^IC!LE`wR*p-vpEfAB^-nHI=ZVD+iZnp&H6*zY3WFGYs~GxPsey-O~hDPI!=%-Wpq|*Rg$$- zC>GJIIT$NfuPM6A?)NGiuIQ<*ZJ=rXNDZKrTGP{22G0C|Rlg{jyDNH` zPNFuhs!CgbW!I!qQC-Pv+3|o;(DHe@J}FQu8o4aJ95t_^fw{}`b1$;aV7LFo^8qQy zhQd`kZ>iId>V~0Z=>@L+V_rKnYk*?wpFXO=snL2Y?~N5!_A@j;zL==@AN0&3v)N7i zClAwkaOaAjj%rD9XT!RtaVb2phW7SFdGbms6RNAV4bes@!}`k{xmJjz5NPf(pI%ZMAtI1Z5>B@xlbOr80|g}QUEg#9W%Cu=_(IT zQhYFTQ0*;+SLLu4O+`bB^&x}%ZZFOgW^uZF-so!I96$Fs+XpEq?QbofXvv3;yZtxl zXc=iMIREHxSSwBvu}CBl35TtgRJS;>TMg~zpj~g>j!3m00ck)1V8`;KmY_l00cnb<4=Iy|BK}N0{y`Y1V8`;KmY_l00ck)1V8`;KmY_l z;P4S}IYp1Ne*gczK;A!m1w@HJ00ck)1V8`;KmY_l00ck)1V8`;1i|U_c)S#Zn`{c? zJLK=kSI8fdH_01x6E6?|0T2KI5C8!X009sH0T2KI5CDM>O<>UN5=KT^zcg>C*Cm`9 zF~5n;Fn^EqxP*x_<~MV&?}ajK@B3XYVR)E*O#u5!1Fw^A`Pml#>;8idx;e%+t?wZ? zCb|UwDgHGk^ebcNS4~y!v-^LM{E0w+@B#r4009sH0T2KI5C8!X009sH0T6hM3CxNf zft>VAik_rg%hZb1+K;_Du4OXxi>hY>uYUT|uQ(M+2~A1iDJ2q+qO(e5HX=<+N-P?Z z>+ApD70A1faRDI>1V8`;KmY_l00ck)1V8`;KmY_l;1LLToF_f?NCCV5caSd&g`kf>ZJCDIoeWh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o6v9KO;XkRX;H= zJF_HFzdXMvyEr4iK;I>?B+)f5Ju@#gwJ0+$T{j#mVWgj12~?DmVyK^+nwykbRIDE# gpP83g5+AQuPxX)`3EM60A*1YdTEz|_Oj+**X(?(5$ho}jcnl02cc?&hIrDhv7ZN7z?t)-@e zzukxDgd?YW0}*5}v2U6GkIX2gho` zMWLj2uVm#^J3o{oT|;rToJt*F=U635;<_4$n*yZ3Za66+PgH!O@@!0QH$QTG8Qa+o z3SmWzTanFKl7dumk>o+Tp+}Wh(UNXBg88J0_~|(;gOmkGLc9T^F$#MBb*Vd6hUc@L7<9PhT6C+6sAr4 z$wHl!fcS`UndhS5f@^o4C_-fqcN)Sul&Zy1Z|_`GB$aHVx3HG zKd>;Nj&Nd^yC^;GMLzeV8m~n`#r>$x8=`LPb+m_QZ({OW+B=#dt;Xbf^*Pv~lKc+a z8HL5!L+dV^u}AElt!!@JwSTv`^M`Yv5r2lKoqm5XD@T&JIr!$r!JT_gFYG6~V`Ppz z*c&aBDvQ0{6A+Vwz0+{-&`jJPB!`b3?N{v8_WU}~ErBXmifp2tax{atQL}*-0i_!^ zQ*kRcJ_yV+3bo0$+3B`9N`GzEo++1~dNmzjd=;M?xDt?Yl+ge)RQ1B5jLS((8OB1Z zrW7UYqHpM<=-KcI^*^OPGx}#RBm9VSgqgg6LLq93al{LeL!`L#n|l zcX;jRAwd%batz5lEv8uFZUR%;a}7XHNAKE^3gCV0VPGHP4}3)8-+BRqlmm<6P1H+@ z?gBflR{)_la;`0kUa&qBd`9a(C-DV|T@qiCpwka~4WJ7I&Xag$rPdvZY)32w#+uGm zS{AyIWLbGDR>@Stt2B|o&|6q8yezWY@gx-)mzC>vxTa^y;1W*nV3V{>XZ3 z73B?Rd-#CF`gkTAbwf;AO5R{V3wPF=SG*-Ne2gX{t%Ic1K9W%_BpjephpJRbwxsF# z4bH~mR|Ux!uOvfJlg=KCYDD{Zetn3!elCIJ556MtH3`aq*9XzvcO%}zN2xN*BBX3dUbaYw8?>!&C zA=ee$a|7pfm{f;^aCMu~XxQ6{+I*H2)3Q(ZHKZe;x}f)qi%bIDBjAysLI(Ut#ex8Z z4wtJIfU`{XbygacX0lHYuPII4_>o+hN5nik=2ACwHRu4vf!7;8T{-|U#g|XOe}%7u k{pz_qB>W`Z!AE8V4x0D}0BTW`SaU=W`ZR? z|MME0_{0ZUPEwx-Bmg-sX;6tyOIqjX!g_vf>6Z36rkL6ncxjjJxm`M?RpQbv(Xn%Zk@XO)=fJ%V zawZeD-Q%FFSg?r35=MF|lCwFWpwW|zZ}%pg!4x$N{oWUQfBgL5uy^x4vE6%542Spk zjt(m9=)orj+wP4%eR#9qU)Z3DAO8BCf`ce$lBKEgCJQbyu4F!^`AF(d6vmtu*xHmgP>h8lWcc=zuh>`#UOvG z+A*aO&mKoIGk1PV+oub_K^_iJ6TN}1{}jH!=-n6m7eJmsd-Wl}UKv%wK!RCUU=oR* zzpvpr2E(?NR%xBv6kov1Wg{6AzIE9omuLTr32$HKHzu@4#k)s*kxpVcOQ${1q6Ehl zy*NpF@`UwdJWiM>t{;8&`Dh?jQUX4B+N`mP#JNMR;3-4FG+_uWDTD zv;uDerb1cm$RZ>Dmp~I806y}NR@?O#Yo)yn#MdHW69_sQsz?A8zEm2;5D}oV(z!rO zS3(4wI@Dg_rB_<~s&q?hg+P^vzXLd^d5KEz+@qSiCTlt$PFQH$dM;0HReNbY)U)F(wf#UFepadYRc7dq-rgdJwu$B$jr2)G%VfTk6 zKv?+@_z?dRMtaqk>s5ZWYT2UoWmvXAP5@_W%b;wpKpt>DDcf4^sk02zhT&*bYfc+4 z>4w@|OX`#zkkVT7ILi$)4quJCvRQ`ZM%gI4D@SJ#+AhQM7VTW1732vR!`vF{0#8NT zEH~EJmUhyzZnRTB`@p!-l~K_>cp`Z`k+UrA2^n(;>Ehanl(`u0?&`3Ih=G7l2N|F4 z7L5}*PX;D=@*JKytd5Uu<8+!0=y)*CC`-iPV0*c z5Jf706J&IeXuKIkXNx#l&vYp}ju(lHz`L{P*vN47+$Gf+CF;z}2LMi?37tfJc&B`1 zXz-N7*T1S7>(l8cj$>dVelDI)vV3}r_h6_nXT%N^E8&4tMuGW;5rBoiyh9{#?Jw-KW7bD+!IxHFH~4pSPL! z#7Uyc@g!zV96iv%mYOnNsPgC^Y;|bs)+BqF=DgTddq))Wo+If`H%EJ7JmTfZ#_0H= zzJu0BE0-buvi1@DOY{$k9h`x-yKuX;GnS=Fi)_Mmm}hkt`*|hLcMdJNg4HiBuS+4+ z`U4^Fl?T~~ua{%R*C%{33ghymPC6ZA<$@nH8fwd4b+W&dmurwroPQ(M;R|E3)Yq zpmprXG1gPGAlh7^CHEoDro~gAxzFqbL;zVlLx2E%pJ2zY`NXZs5zLVzH2t<&QxPnh z>&L9c+K;WI!y*70Yi~0byGQ>4dx=!EX6?H?%S+z9nkjzt4QZmjG<|y( z(IHfp<_Dy_ISb#1(HbeoM$CZ8PrKQ#Jf@jtByvMXU2e> z5a30jv=9IGgakAANK7WYU>t59Wa^C z7hjvnA1axR)NC0Wck8g5cgimISx6fB5uNd42oNUN7sG0cMV$GZW?zPwwXx4j!gLlB z<||tADTHnr|0^y8aM|?a+b|_TWhz9{E<`GJQWF9B8LeGd625?yj(<`TIw%J811%g$ z)>CvixhYLZG)mh-yShAF}|X5Vp=MqoukCEZ0RK zK(w}0rb?9XP!uUHKCZZYpZqgb;oP=RSzZIbBc=u0mTR!iR_Af9LFR0>dac5HEQ&c= z$uG%iDhS69^^Xh0D_Fb0dSPk!0jnyQiB9I=FR?y%7S_6TT}VFOEq>>ZOz3s9zp>Td zxOz=~PcNg|p?#pNrey_rTam{V(8=l?>gkD>hBoqRT2F;?F%!Opbseo{0+oehE`SMS zJ;j8R%i$`l^M&CJObEaPVg;3g_S_W_u>eUQ>L!Bq2QpWFhj-2>o4S<9JzLV@X7TlD zF;avWrSUAl)T$WZqk)Qn`~YImi_eO$G?0)d@<)gy#LIE1gHbx%M!-xV)-JZw@lL%> zmC7ddJSsYKmw>rp&(qs}g7_J&q|cTBUj|f7FW5#ufKMUY&x&2K#)j1;@D8Tw#%=m^ qGq7eX}c@9DL|5JM-qfH}7kHZzu8nDuUAh1aG6R`vE+oM_nxSa60{f zVbL|D{zEd5#{DSSlMayEJmjggfqIzirJ(EIr{Ro`c4yIcGMfU!oQ5o=JJP1FI17io zt4U)piitGha6*Tgax#BbVOCXn0AKnqNH`kfu_iFVCpg!*E;OQXV{Dw7XGQ)eMy?C} z)H*{Laa-ubB-Vx8J~2v((8f+~z*xym7^}Gjqnkt4x$_2Rhet5`2xte5$UXo{Aoi)GS+b6628^J?n`8n=Z$Zr?>3>H{@DpH;Ffnx|LD${9We z{PI=OhDwyws#u;=S42xu!Y^vpFy{5BLng4Z}1VrC~sqweW^V18=?)By97;_c)R(3z5NXS^7gN7<+Y>^@geGV zRFi4bgUsD})Oqss@%K+Rx3?jjvf;+ia8?R^HcMp{X2$y@*6w6UHQjthK zP7bA!hA%0zLBbXaF^x$Xvs0dIoJjYRbCnLGFJ?65DRY#*!67t217thoQXj=!I^e6X z*sP*7^cf#YCrXB4G|IqhL}Td;lbF*OjF`%EL+P1QrgobtXQQcfxvD{lC01AR%I2#f zoho!l9dx>@5Ic=_rKLbLB7v=D$T#pWP^FedTI`mxFwba2WUb8Tyu1)2iwd}fvN8?# zXg`IkLYf7Vr9W5e50e=LXcrS@aB0AyvX-)3gM6Tj0PV7NhJp!e7hxB;=nq0BUB&H} zP-5z8f$G6Q1Wthth*fIz)gOp$fav4x9s|tv4ik0&xE<17I^wi9p_2h+X^)5LK?==K zbdn4DX$C`0TgDE)g01;$U;|gNhpU>R9C#P7R(wtDXdT>ymHKL$p(#BNFKab@K~t~N zg%9}Udkdt>u7f)uxpA@j3_HJRAG=L<69Ne8lOu>YB44Zx0TJwosoPXqlZ4PHEihT0 zxPoGdy4y-YAEaNEc&aOS!gYP#*CmDwX!}g+KkYnVpDTSs5%Ux&(uRJluxZRgMm}G*7GH{{R3qdp!UE literal 0 HcmV?d00001 diff --git a/myworld1/members/__pycache__/urls.cpython-310.pyc b/myworld1/members/__pycache__/urls.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76a22ae170659c10f733a723c711cc4390d70273 GIT binary patch literal 706 zcmb7=y-ve06ou`kY15`HP%!`-h^0%jS5<|00G3J&7E+YdR+^I7k?m5Nm6>;dk%i|$ zSz+nQ$iy`jh^QceEg$LA{W|tF`^Y} zC6&}77Df!yDzj!)YSY@S1yl#J?`rqxrUK-Ys8OQ7XEvd(PThYylZ6-)6NSt5^mm4Cy{L#4sxIa%dn9FjKM6scX#L+4%OX^ zvS1p#kFD}Gw7b`>aLpV>aVJ+>~`B0 zo`3vpApX*`tbfvA^YJjajiw@IS={0*w+a?BMzo#Vg%dl5c5=7yV$aZS?iWEEfcChb zhea!H8GeBE?YM2|Fz*ym45iT1%DY7`?iqShwq45>Tk+QaDev)4-Y>4j*Ub7T9~9g1 zwxPSQdneuj-Q!!2taw-SM8y02+PVFK#Rq)*k;S(~@X&e4;yrM7_%1lR#EGx-J$@ZB zH~0;H6W^Qs)?I7#$}{rQh*iE6b1o#P=+{~=J`!qC&Xu5d-Ml=)0ykYONKYUh zXqiq)CX|9_QcU%f{$Gvg6%UD#*vh#A>tZ}k|pqA6A zPzopOL#bY>*rQqX?C((Yl*zAtendZv4iqp1}v(| zKg@CNI(U%$!Uql-3XVkeUJepFsea?&~m zrt|4BaFk}b;7Ei7L;`-DOtU{C-O1O%d+JwhlfcXYRjoW#x+a5^a|vwA?~|xslm}2H ze}I9c(~#umQDic#{3D>;zzVGGT`;;$gqIq-tX*VY+ugLXD(cZ2pWnEIx+xzDd; zFYU~b318@Zp|PQ$uc4r=p`fdwIv0#b=RV+O^DggYjBhnru|N6p1Nb4fak`!6`S36k zIafovop2rUrww3&0su&lME0*WUg|^pF$8N!`N)6-5`zX`4rzk?UmrbPzXlwYKa~PW z6iB>0=`S7|C4djn%#Hgt8!+E)vlp-bydRwQH;;(23+~{_$dmi@qwi4nUFwVlUz$+9 zhb6QFMR!_gwe4t667Nb4fDK!e7R-6m-9R(Y` z!H4Ob4;STPNfiJe>SHm~=|L{W@)uZWyk){~?*Cn9oT_3GxL@nt!v##Ap#3gYT|vt+5W zTx~3oZOqn}v1Lvs?lvXZSf}c9WjteCVyZ546B+E76EM5naE>`dN=a4tc75ypl3=C2 eg)XupC$b~dZ|k>zNf9Ts+rfbKZ-w88g8u?-`qstats''') + + view_stats.short_description = 'Stats' + view_stats.allow_tags = True + + list_display = ("job_name", "start_date", "end_date", "no_of_blogs", "start_no", "created_date", "view_stats") + list_filter = ("job_name", "start_date") + readonly_fields = ("created_date",) + + def run(self, obj): + return format_html('RUN', reverse('scraping', args=(str(obj.pk)))) + + run.short_description = 'Run' + run.allow_tags = True + list_display = ("job_name", "start_date", "end_date", "no_of_blogs", "start_no", "created_date", "run", "view_stats") + +class DjJobStats(admin.ModelAdmin): + def view_logs(self, obj): + path = "../joblogs/?q={}".format(obj.pk) + return format_html(f'''Logs''') + + view_logs.short_description = 'Stats' + view_logs.allow_tags = True + list_display = ("job", "status", "view_logs", "total_blogs", "no_of_blogs_extracted", "start_date", "end_date") + search_fields = ('job__pk',) + +class DjJobLogs(admin.ModelAdmin): + list_display = ("date", "log", "function_name") + search_fields = ('job_stats__pk',) + + + + +# Register your models here. +admin.site.register(Students, DjStudentAdmin) +admin.site.register(Blog, DjBlogAdmin) + +admin.site.register(Job, DjJob) +admin.site.register(JobStats, DjJobStats) +admin.site.register(JobLogs, DjJobLogs) diff --git a/myworld1/members/apps.py b/myworld1/members/apps.py new file mode 100644 index 0000000..0ec23dd --- /dev/null +++ b/myworld1/members/apps.py @@ -0,0 +1,108 @@ +from django.apps import AppConfig + + +from django.apps import AppConfig + +import psycopg2 +import requests +import re +from bs4 import BeautifulSoup, element +from django.apps import AppConfig +import psycopg2 +import requests +import re +from bs4 import BeautifulSoup, element +import datetime +from dateutil.parser import parse + + +db_name = 'member_db' +db_user = 'postgres' +db_pass = '123456' +db_host = 'psql-db' +db_port = '5432' + +conn = psycopg2.connect(dbname=db_name, user=db_user, password=db_pass, host=db_host, port=db_port) + +def add_row_to_blog(title, author, date, time): + sql = """INSERT INTO members_blog (title, release_date, blog_time, author, created_date) VALUES (%s, %s::DATE, %s::TIME, %s, NOW())""" + + with conn: + with conn.cursor() as curs: + time=time.replace('\u202f',"") + curs.execute(sql, (title, date, time, author)) + +def truncate_table(): + print("Truncating contents all the tables") + with conn: + with conn.cursor() as curs: + curs.execute("TRUNCATE members_blog CASCADE;") + + +def start_extraction(start_date=None, end_date=None, no_of_articles=None, start_id = None): + print("Extraction started") + url = "https://blog.python.org/" + + data = requests.get(url) + page_soup = BeautifulSoup(data.text, 'html.parser') + + if start_date: + start_date = parse(start_date) + if end_date: + end_date = parse(end_date) + + blogs = page_soup.select('div.date-outer') + truncate_table() + article_count = 0 + counter = 1 + for blog in blogs: + article_count += 1 + if start_id and article_count < int(start_id): + continue + if no_of_articles and counter > int(no_of_articles): + continue + date = blog.select('.date-header span')[0].get_text() + + converted_date = parse(date) + + if start_date and converted_date < start_date: + continue + if end_date and converted_date > end_date: + continue + + post = blog.select('.post')[0] + + title = "" + title_bar = post.select('.post-title') + if len(title_bar) > 0: + title = title_bar[0].text + else: + title = post.select('.post-body')[0].contents[0].text + + # getting the author and blog time + post_footer = post.select('.post-footer')[0] + + author = post_footer.select('.post-author span')[0].text + + time = post_footer.select('abbr')[0].text + + add_row_to_blog(title, author, date, time) + + print("\nTitle:", title.strip('\n')) + print("Date:", date, ) + print("Time:", time) + print("Author:", author) + + # print("Number of blogs read:", count) + print( + "\n---------------------------------------------------------------------------------------------------------------\n") + counter += 1 + + +if __name__ == "__main__": + start_extraction() + + +class MembersConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'members' diff --git a/myworld1/members/migrations/0001_initial.py b/myworld1/members/migrations/0001_initial.py new file mode 100644 index 0000000..c02674d --- /dev/null +++ b/myworld1/members/migrations/0001_initial.py @@ -0,0 +1,22 @@ +# Generated by Django 4.2 on 2023-04-25 05:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Members', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('firstname', models.CharField(max_length=255)), + ('lastname', models.CharField(max_length=255)), + ], + ), + ] diff --git a/myworld1/members/migrations/0002_students_delete_members.py b/myworld1/members/migrations/0002_students_delete_members.py new file mode 100644 index 0000000..35b345f --- /dev/null +++ b/myworld1/members/migrations/0002_students_delete_members.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2 on 2023-04-25 08:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Students', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('first_name', models.CharField(max_length=200)), + ('last_name', models.CharField(max_length=200)), + ('address', models.CharField(max_length=200)), + ('roll_number', models.IntegerField()), + ('mobile', models.CharField(max_length=10)), + ('branch', models.CharField(choices=[('BA', 'BA'), ('B.COM', 'B.COM'), ('MBA', 'MBA'), ('CA', 'CA')], max_length=10)), + ], + ), + migrations.DeleteModel( + name='Members', + ), + ] diff --git a/myworld1/members/migrations/0003_blog.py b/myworld1/members/migrations/0003_blog.py new file mode 100644 index 0000000..4c4cd65 --- /dev/null +++ b/myworld1/members/migrations/0003_blog.py @@ -0,0 +1,24 @@ +# Generated by Django 4.2.1 on 2023-05-23 04:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0002_students_delete_members'), + ] + + operations = [ + migrations.CreateModel( + name='Blog', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('title', models.CharField(max_length=500)), + ('release_date', models.DateTimeField(verbose_name='Realse Date')), + ('blog_time', models.CharField(max_length=50)), + ('author', models.CharField(max_length=200)), + ('created_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Created Date')), + ], + ), + ] diff --git a/myworld1/members/migrations/0004_blog_content.py b/myworld1/members/migrations/0004_blog_content.py new file mode 100644 index 0000000..8214234 --- /dev/null +++ b/myworld1/members/migrations/0004_blog_content.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.1 on 2023-05-23 09:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0003_blog'), + ] + + operations = [ + migrations.AddField( + model_name='blog', + name='content', + field=models.CharField(default=None, max_length=20000), + preserve_default=False, + ), + ] diff --git a/myworld1/members/migrations/0005_blog_path.py b/myworld1/members/migrations/0005_blog_path.py new file mode 100644 index 0000000..08327be --- /dev/null +++ b/myworld1/members/migrations/0005_blog_path.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.1 on 2023-05-28 12:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0004_blog_content'), + ] + + operations = [ + migrations.AddField( + model_name='blog', + name='path', + field=models.CharField(default=None, max_length=500), + preserve_default=False, + ), + ] diff --git a/myworld1/members/migrations/0006_alter_blog_path.py b/myworld1/members/migrations/0006_alter_blog_path.py new file mode 100644 index 0000000..46547f1 --- /dev/null +++ b/myworld1/members/migrations/0006_alter_blog_path.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.1 on 2023-05-28 12:39 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0005_blog_path'), + ] + + operations = [ + migrations.AlterField( + model_name='blog', + name='path', + field=models.CharField(max_length=500, null=True), + ), + ] diff --git a/myworld1/members/migrations/0007_alter_blog_author_alter_blog_blog_time_and_more.py b/myworld1/members/migrations/0007_alter_blog_author_alter_blog_blog_time_and_more.py new file mode 100644 index 0000000..b7cd122 --- /dev/null +++ b/myworld1/members/migrations/0007_alter_blog_author_alter_blog_blog_time_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.1 on 2023-05-28 13:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0006_alter_blog_path'), + ] + + operations = [ + migrations.AlterField( + model_name='blog', + name='author', + field=models.CharField(max_length=200, null=True), + ), + migrations.AlterField( + model_name='blog', + name='blog_time', + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name='blog', + name='content', + field=models.CharField(max_length=20000, null=True), + ), + migrations.AlterField( + model_name='blog', + name='release_date', + field=models.DateTimeField(null=True, verbose_name='Realse Date'), + ), + migrations.AlterField( + model_name='blog', + name='title', + field=models.CharField(max_length=500, null=True), + ), + ] diff --git a/myworld1/members/migrations/0008_alter_blog_author_alter_blog_blog_time_and_more.py b/myworld1/members/migrations/0008_alter_blog_author_alter_blog_blog_time_and_more.py new file mode 100644 index 0000000..36ee48e --- /dev/null +++ b/myworld1/members/migrations/0008_alter_blog_author_alter_blog_blog_time_and_more.py @@ -0,0 +1,48 @@ +# Generated by Django 4.2.1 on 2023-05-28 14:02 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0007_alter_blog_author_alter_blog_blog_time_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='blog', + name='author', + field=models.CharField(max_length=200), + ), + migrations.AlterField( + model_name='blog', + name='blog_time', + field=models.CharField(default=None, max_length=50), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='content', + field=models.CharField(default=None, max_length=20000), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='path', + field=models.CharField(default=None, max_length=500), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='release_date', + field=models.DateTimeField(default=None, verbose_name='Realse Date'), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='title', + field=models.CharField(default=None, max_length=500), + preserve_default=False, + ), + ] diff --git a/myworld1/members/migrations/0009_alter_blog_path.py b/myworld1/members/migrations/0009_alter_blog_path.py new file mode 100644 index 0000000..eb26a5f --- /dev/null +++ b/myworld1/members/migrations/0009_alter_blog_path.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.1 on 2023-05-28 14:11 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0008_alter_blog_author_alter_blog_blog_time_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='blog', + name='path', + field=models.CharField(max_length=500, null=True), + ), + ] diff --git a/myworld1/members/migrations/0010_alter_blog_author_alter_blog_blog_time_and_more.py b/myworld1/members/migrations/0010_alter_blog_author_alter_blog_blog_time_and_more.py new file mode 100644 index 0000000..11bc263 --- /dev/null +++ b/myworld1/members/migrations/0010_alter_blog_author_alter_blog_blog_time_and_more.py @@ -0,0 +1,38 @@ +# Generated by Django 4.2.1 on 2023-05-28 14:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0009_alter_blog_path'), + ] + + operations = [ + migrations.AlterField( + model_name='blog', + name='author', + field=models.CharField(max_length=200, null=True), + ), + migrations.AlterField( + model_name='blog', + name='blog_time', + field=models.CharField(max_length=50, null=True), + ), + migrations.AlterField( + model_name='blog', + name='content', + field=models.CharField(max_length=20000, null=True), + ), + migrations.AlterField( + model_name='blog', + name='release_date', + field=models.DateTimeField(null=True, verbose_name='Realse Date'), + ), + migrations.AlterField( + model_name='blog', + name='title', + field=models.CharField(max_length=500, null=True), + ), + ] diff --git a/myworld1/members/migrations/0011_alter_blog_author_alter_blog_blog_time_and_more.py b/myworld1/members/migrations/0011_alter_blog_author_alter_blog_blog_time_and_more.py new file mode 100644 index 0000000..b979ff7 --- /dev/null +++ b/myworld1/members/migrations/0011_alter_blog_author_alter_blog_blog_time_and_more.py @@ -0,0 +1,49 @@ +# Generated by Django 4.2.1 on 2023-05-29 13:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0010_alter_blog_author_alter_blog_blog_time_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='blog', + name='author', + field=models.CharField(default=None, max_length=200), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='blog_time', + field=models.CharField(default=None, max_length=50), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='content', + field=models.CharField(default=None, max_length=20000), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='path', + field=models.CharField(default=None, max_length=500), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='release_date', + field=models.DateTimeField(default=None, verbose_name='Realse Date'), + preserve_default=False, + ), + migrations.AlterField( + model_name='blog', + name='title', + field=models.CharField(default=None, max_length=500), + preserve_default=False, + ), + ] diff --git a/myworld1/members/migrations/0012_blog_recommended.py b/myworld1/members/migrations/0012_blog_recommended.py new file mode 100644 index 0000000..8b9c303 --- /dev/null +++ b/myworld1/members/migrations/0012_blog_recommended.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.1 on 2023-05-29 13:53 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0011_alter_blog_author_alter_blog_blog_time_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='blog', + name='recommended', + field=models.CharField(default=None, max_length=500), + preserve_default=False, + ), + ] diff --git a/myworld1/members/migrations/0013_job_jobstats_joblogs.py b/myworld1/members/migrations/0013_job_jobstats_joblogs.py new file mode 100644 index 0000000..9d42066 --- /dev/null +++ b/myworld1/members/migrations/0013_job_jobstats_joblogs.py @@ -0,0 +1,48 @@ +# Generated by Django 4.2.1 on 2023-09-26 09:55 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0012_blog_recommended'), + ] + + operations = [ + migrations.CreateModel( + name='Job', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('job_name', models.CharField(max_length=500)), + ('start_date', models.DateTimeField(null=True, verbose_name='Blog start date')), + ('end_date', models.DateTimeField(null=True, verbose_name='Blog end date')), + ('start_no', models.IntegerField(null=True, verbose_name='No of blogs to skip')), + ('no_of_blogs', models.IntegerField(null=True, verbose_name='No of blogs to extract')), + ('created_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Job created date')), + ], + ), + migrations.CreateModel( + name='JobStats', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('status', models.CharField(max_length=50)), + ('total_blogs', models.IntegerField(null=True, verbose_name='Total blogs found')), + ('no_of_blogs_extracted', models.IntegerField(null=True, verbose_name='No of blogs extracted')), + ('start_date', models.DateTimeField(null=True, verbose_name='Extraction start date')), + ('end_date', models.DateTimeField(null=True, verbose_name='Extraction start date')), + ('job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='members.job')), + ], + ), + migrations.CreateModel( + name='JobLogs', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('log', models.TextField(verbose_name='job logs')), + ('function_name', models.TextField(verbose_name='Function name')), + ('date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Log date')), + ('job_stats', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='members.jobstats')), + ], + ), + ] diff --git a/myworld1/members/migrations/__init__.py b/myworld1/members/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myworld1/members/migrations/__pycache__/0001_initial.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0001_initial.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3dcb15a372158d92cd98c616a6f6aa9987d1df7e GIT binary patch literal 709 zcmYjP&2AGh5VpOWWOuWvIG|qOjFdyWDQAQVEv-1D7b+nw2g}J!Hcs{@*c%{RLE@AL z;gx*l)K@@)GP?@5P|bbRPQck9mt^yn>w0KzRNFFN!x5Bgh?@ic;%+8MZT{J)P| z*BfIoL1UfJaOK3?%c1XLwXQ2UHK=L?*IyZ3D6^4E+_=!9(JI$p(Z!$8Oe%}AR0X1nZ_LK&9LtR*`|=vMN$$i>m13BrTn;7 zc>^Y|r%PQ^VHX3gummip8m*xfG1tFnI~_NK8n1bYU+e4@&2ZJ!aN)&VnVlI`E8K73 zCo@k7+p6Cml6h5NqROSNRr0wqOFOSt$)&2*n=;cS($r;k@_rw9o)jAb<#3i1JNlA` z)h46S=uGbWJzZ@~^Jltr?J=o<`GSrw_+xRzkGS~b{|)Y2%neZ=!a|iO2z1-aqqu#Wnx{ literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0002_students_delete_members.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0002_students_delete_members.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e8a6d2d26747b78b9d6a7cd7845903e2c9b7db9c GIT binary patch literal 940 zcmY*YJ9E=O5WYLfFIgr5LIMmW!=NS()C`lvnPgHp6roacBYW4f96ijP3?wbI?igyI z<9E>T8?L2FO+^aAo@^$`-R#bO+S~W4=yW_r=j_t~{BRiiWx@6164)lvPly2np0bRG zJQJbd?0|s)=OY6L`Na9)gbujJEU2B+TNVi8Wl5}}GAVLx>RADp>Y-I#KOF+wMEWx^ zWFZF@3g9sh9dsu(8vyH*2Do%

06c9SLaCw97wzi(h zBu~mDN=@^)oB`&gHX?!7`$in@1+MXUsqYRBVN|Y`u`TQrfCE(P`=)FuakIS)XrO{Gr8WDg(txa1MX)5!XP0aRskfeBS zM{DW>73ITm;8NydcSmhlhvK5PyV-lySNAJqoIYtDEM(E!Q4belV>m97A!@a{z;2aZ zeUxAdI&Ab8eF(UEMEaw^H%(b(Ev50Kq>0Z`B-xkp%`8f-u|x4W zuZdgSUXN&vi$CsP*B9KqZ2qn+j%W*7Fp2WG*n~kZ(|G{sEdu1#|!a literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0003_blog.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0003_blog.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e25eb97888b1967f08ad765c99b579007ccd130c GIT binary patch literal 868 zcmYjPy-pl45VpO0%kAAB5JE^pktk85YdAzwMiBu-lmd!`>Ke;;Gb}##=VI@LKou!z z??Lh=c?P$s=u=UIjCT)#Esy+p#xvjil)YYu(R#j{!EwOY9}ga`kiae-`-%iG;2F#L zkmq73IQzgrK=6}6fP51C2!;WKpIN_kO}4BrTqoD5QI#$W>!Q2_%CLjY54#cO_m^}-GT^~=HaP;HodhWhR&Q5>Jo|}fnJT4Yu(C1w_2k| z3$#{SdS8!WeIA^OG=e8{LD4#}LDZkR_~?!^C9srZK0;&N=JxjXjwNJVkau?ykC@P-PaM9T4bq;|u8G5iyXJ&hdK3I#ri>`<>GVC7=+L`%@&1`I&9&#VI-xuo z*eR#Dn4n+Gyr8eOxOp=$Wmyd_N^@?fYK#M`D;zBPF}NSQ0rl;*^u^hlUK&5l5gofp zqRm7Q-i9r~*Tph#_8J#A;lK86ZFx+6jA5dRwA_MGok>l;R^D@IZZFBBYrk!3@{U@_ KtdF)THiG|8&-EJs literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0004_blog_content.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0004_blog_content.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e3b3a071b9a09f7df075edc034648d5bfdf67ce GIT binary patch literal 596 zcmYjPJ#W-N5Zy1I?NcJ9fYek-*Tg458KFc-q2d}kH!I)G#PPBAnq6Oj8ru6CNc^R3 zQ{^upq%gZVl#Db_`!q8<@2zc*kLQd+?G60mjNL`@JQR@&O8=Sw7;wXSUh`hmg0l|{ z1jwHZB=W8NA!`Zh6I*2W)XNqk%=@l&&D7b!hgoj{jo-x5^T>%@Q2HeSvYG>{1@M;F z64f?~3vjm0K|uzyy?`8wy`pN7R+XXr_!0IpB4(GZTqyzG;I)JKbC@6w#cixs424dOT8FvT zG`KTJ=OeAZ>>3k?BLf_9030?QdMC(bC@ul6;-yYTTE4$&T;gGFM?AdPZ{}W7HNl_r z%2_*AU#$7X|-?dH2FJapZtv!QvQkwLo;**xg(<$8)it|U$@958AK5jV?r}7^LDxEO^ literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0005_blog_path.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0005_blog_path.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..959d33f29501d3aba90f5b4fac047fa7694fb881 GIT binary patch literal 598 zcmYjOJ#W-N5Zy1|*{4Jb667Z!T@zms3M7Prkb)>k5jr=ky_<>87u#!geGqDB^F#8N zwxyz{LXpDk-l1fqdD^G>e6#lOaK^|kKYF<1jQxt_sVO37rqrss^_Z9WNa$ba7 zr$nv8OlvCKwn*;-t-owd8;c_Y9IyxMR}Fe6NNUL60i4ICPDWb1UmKTrnArjMXS-wW z1w|A5eO)?h$MTzX*M2Z7EPXR#xtCl%-nu05uXf_2+LPhCi{CgQ_s(Y|@J?(nc+WVdeveC3S&OvPhURGg8AXB2<|x2)wAZ$%|I zd&5A0{LVlk-^g#WlAzwRdG?ED*<8e;ZEA0Z#&sc1S_f#uIzf+ZO2rv@SWrM#abT4I zUh_(#+GL3UCz~lK$Y63OV0tGB%ONMCS)9H5&p>q37TXp25RaD2<#WBVuGYO7uA)k5 zR4H$Mlgba{Zsv=&*wqQ|w${!i?VEPgooSJJDP|uVv~WpGxgPsk$3knG+ghajK^ z^k$7^Xa+1F#FhJ-mL&6qHi=*_cG2JX^l20F@PtCngjDw`6I}2EuKwotBYrtoaRwXH i)ouZ+aoKUf5!^4c%TDN>n7`b~enl5#A#ogw6ZsDT!k7sF literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0007_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0007_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..24bf11f7c251bf91db11d0ba860996c82cc44343 GIT binary patch literal 859 zcma)4J&zMH5ViLs+1(^?AVdec3Td0$69q^Jp+g|3NC8)^ge)f>k`3N2zTGIB-R%rv)+>Wu5PN`N#|zqPvtm>i** zPw0TuRFFCnbVlnKl11t-fZn1n5|N7DNhJCwXLuIy{2X2cd~gmQBAy{S5*PmIb1~HK zZR=-gor_CIp=6(920V!M_y4j|M^-VnQ`cu_e=xnn;#%!oT0yn?} zxA1=O6u307{}7Wh(q2b5d<}Y{uwv8HHdaFOgAy_>3zx9~Gv*4$aDtTtjE9W9T5;(Q z_)NeOIsx6Z0iz?|v$*ehKJxSF*aP{4Ip5@NW$Jb&5}={GL41mRLyGpNh;L!_O7lQEq9T>0x{kRWTD|;`A+*^+CwvU&}Afwlix`i z(TEOd^6P)2?0e`lE)$oF1@C6+Kum%Lfp^&4lG;Sq@jjQ`-KcpFg(k+&vKL*A{{TJ2 B=;8nX literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0008_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0008_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b2e89f11b7392a1decfca06630985b0acbd594ed GIT binary patch literal 994 zcma)5J8u&~5Z=98pYJX<3C|KzAzc$k2nh;=f;C}_ zzk}bfC5R$D6#~KR*&!BbveJ%b=KFST=bOEz-%lB>?=K8|>@fDt2kT2vut(e6Bm)e1 z$ts@nD#!!Q9y16a{J5X%sv5x3j6AfydKB!@ zHlN8L%Q>(-06ykjOC?lVN1+X7t1ywrAs9>3G))k?@9S;u1Zh! z5#S7KzW* zc-`oiW~N3sFongzYCwb2cR8R@-CZBdc}BG}d#nAA5z#iJ6^mIAia(+g@PMaW{9bxJ zio`Dw`Rw`q_-kAHAQLWusjA1#4h&meS}j`ew9%z^gYMwsdrO7y)5w_d_uC4tg?|An CI1IJ` literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0009_alter_blog_path.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0009_alter_blog_path.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c3571dacf50b5b6973c479deca9815bccb73f178 GIT binary patch literal 616 zcmYjOJ&zMH5cSt4o7J6!khq_KlqQ>k0we^7kf7oksJ5}3%&^(uT`%@VKn?Bv4#Zz_ zOGQtGgpRRu=rYobJoD_CdC%_rykcbB1o+Mw``S>QByp2&36z9l8nYTM6nr1$`?;XZ|u38`*W yA-LdYT>UL?GJdsGaRGZXv~C5Pylh@DgLlguPZ!iDmQRngpOJzrq#5_bh5QHnA*X== literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0010_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0010_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88d91bf7e038f3b5687680229dcb30216e2c2166 GIT binary patch literal 859 zcma)4J&)8d5OthnH_7h8fe;l)S0Saj1Ue8xhd@xV1zeTJ@~($m7AM}by?nLM9{+=X z!VjQwOSPhP&sLL3MsR(d8&O1reDlffLW~@unOo=t5rHIa zPz5cmXh5D4K}GnHh!BEB_#v!95xpRJyvACRQYxfCTEzYW7%YY&60sPZQZYO|$CDmUFW^~^k1pV2#1lj(;?h5TF8B3& z+xj}LQgH=|Ty^QQa|TFf*g`u4ZK`W!S>9z_TF`7JRn3;%9=phgZopUeSn2f_q;u43 zz~3?nUF|Enm6Umix_B*E$6G+m`}k!8QnW~#LzfU5M!%z& z1~eE4(XaoFlJ9+=X&$*$EO=L|12OA0=y~^>TT+|oCf?_g`x`YMpwPtlS@!7l@DIT= B=!O6Q literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0011_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0011_alter_blog_author_alter_blog_blog_time_and_more.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf43569848ec093ca9b20871c7d8070e00016f68 GIT binary patch literal 976 zcmbVLJCD;q5MDoH`y2=$(UJ-&&G`;dB6J9JR741<+^ja)Imf~F`gS+RqlNZ<$hG_m ze#3T%hV)bj31%H17m)_B(u`*2o5#+~t{4szg6G?X<|ZKIt5vp-kIE5}x{Hhx?o_09 za;J84*C9^{ce(eGa1WfN_rc3O?!P3HV2!?H;+mu`3(0gT8fC&-aH!PWLfa=r=IiZ)KJerDs%tf;P8O5N`s`cN)(i>^g_sOOJ~+)4U{#U z)CN9TZau6)=U7OvC%V7r-h7<0q(0Txs8mNzXve z1bS>L>QYxQNlZjL1EnUR*bHA)fO1Od>55g>)oL7CXgD{Hxk%$KYw#6F*Olj)|>fmf|= Xq%q#YL=QGIeuzsW%HD9~-u8Y2C;JQL literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0012_blog_recommended.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0012_blog_recommended.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db337c208fb26e483ecd9aa9a55e6c49a7fec04a GIT binary patch literal 647 zcmYjOJ&zMH5VgJDWH;PNSD@n`Af>s@Axc0&NJqsrblX@?W^S`#yM&S z#=7H|^gOl(mF__q>oCap79KcUTVt>XgzNSacId&DC5IjBBZ0 z8!AseziF|9a5gyf=suu2RL;v#sFbKwm?>g}U59i(QR@9pcdlVEeBuXe= z0$jzWE{m{ywb3r&FtY<51pda{IYpEEv#gx8WA)LxcYe@otb99Sbr7ez|CNelda@Vj zXz7dLvy1;eA@dK(6-;FEPUeCOenKYy7I!J*niMEutNX@Yz$#6fW=z5RdG@zQ)F~9t Q_Itk|S(%S4XW~5j2N&S8rT_o{ literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/0013_job_jobstats_joblogs.cpython-310.pyc b/myworld1/members/migrations/__pycache__/0013_job_jobstats_joblogs.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b5f43ad1f3b3ed28bf859fdd4b27dcb774debddb GIT binary patch literal 1629 zcmZux&6C?S6t`t3u@fhoO}k6Wr}-v)B&7_T7^d6Zrczbml&xd?7y zD@WLY1$JiT_SDYFl-LTB%$d4(EpWj7$pV+zY2!y{+5mE94I7Vfk2NH+ohPw~s)QG% zY~~zTR-Vq)-0s&ya0^@cgKb&Nn%ZDZ3D~hcb(ou?M{rm(ZGnq_1Dep5cxPCKzBnB_YB@MxR1Dv^ZN!548CIURkPlO1326`3lg{Br47;Xue~t-b=={> z%O?IS#^0;P-)jcHZt#}h8wPLrZE=X}J1j`MX*2Dmd+;W_rSjNe@%9T@yo3At@GiWk zRd`>c4-6V2QkuB%VVx6v^jwsWO_YYpHFl=^@CjCLB>nr_vSAHdIm>5El+%M7H$J^d zXBm&FU~``5tN;em`I^rpN#J}U$-R@ID}!iNaXJ?)s&Klq6iFV5OM1aBrC&0UL|L+C zGWec}87~{=`EtwDIIbk7&k+ZzBP*>8`W%6);t-O!hsmiik_jjUd z%js|+J*wSMD%+Ie6S2w|jsr@+TSb}js1UHFI&8%}VWl8it7Cn>?yEONILZ>-PU0gx zc@w3v^p77zq8@fn@SL0{IjdoCuc%ne%w&7Q1xw=M8$6v&NTVX=lE6%QS`SQ8wvLa^ zj*m|6NdFwgt$noll9_kyS4(`Z3az@oPsW1tYW$Fki*gyw*|Dhhr4{Y+6`@{{NwO)J1+Or2XTK;jL zcz07>6h|=AGS(LiDA-yl_f_;T|)+(FW^@(R6 GI{yJo3F3?Z literal 0 HcmV?d00001 diff --git a/myworld1/members/migrations/__pycache__/__init__.cpython-310.pyc b/myworld1/members/migrations/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..96bf2fba02f2cb10c087daf352c9e80bd2d4a287 GIT binary patch literal 180 zcmd1j<>g`kf>ZJCDIoeWh(HF6K#l_t7qb9~6oz01O-8?!3`HPe1o10eKO;XkRX;H= zJF_HFzdXMvyEr4iK;I>?B+)f5Ju@#gwJ0+$T{j#mVWgj12~?DmVyK^+nwykbRIHzy rnO>Awl9``ZtREkrnU`4-AFo$Xd5gm)H$SB`C)EyQVKEbsU||3NZ#XVS literal 0 HcmV?d00001 diff --git a/myworld1/members/models.py b/myworld1/members/models.py new file mode 100644 index 0000000..4162cc1 --- /dev/null +++ b/myworld1/members/models.py @@ -0,0 +1,64 @@ +from django.db import models + +BRANCH_CHOICES = ( + ("BA", "BA"), + ("B.COM", "B.COM"), + ("MBA", "MBA"), + ("CA", "CA"), +) + +# Create your models here. +class Students(models.Model): + first_name = models.CharField(max_length=200) + last_name = models.CharField(max_length=200) + address = models.CharField(max_length=200) + roll_number = models.IntegerField() + mobile = models.CharField(max_length=10) + branch = models.CharField(max_length=10, choices=BRANCH_CHOICES) + + def __str__(self): + return self.first_name + " " + self.last_name + +class Blog(models.Model): + title = models.CharField(max_length=500) + release_date = models.DateTimeField('Realse Date') + blog_time = models.CharField(max_length=50) + created_date = models.DateTimeField('Created Date', auto_now_add=True, null=True) + content = models.CharField(max_length=20000) + author = models.CharField(max_length=200) + recommended =models.CharField(max_length=500) + path= models.CharField(max_length=500) + + def __str__(self): + return self.title + +class Job(models.Model): + job_name = models.CharField(max_length=500) + start_date = models.DateTimeField('Blog start date', null=True) + end_date = models.DateTimeField('Blog end date', null=True) + start_no = models.IntegerField(verbose_name="No of blogs to skip", null=True) + no_of_blogs = models.IntegerField(verbose_name="No of blogs to extract", null=True) + created_date = models.DateTimeField('Job created date', auto_now_add=True, null=True) + + def __str__(self): + return self.job_name + + +class JobStats(models.Model): + job = models.ForeignKey(Job, on_delete=models.CASCADE) + status = models.CharField(max_length=50) + total_blogs = models.IntegerField(verbose_name="Total blogs found", null=True) + no_of_blogs_extracted = models.IntegerField(verbose_name='No of blogs extracted', null=True) + start_date = models.DateTimeField('Extraction start date', null=True) + end_date = models.DateTimeField('Extraction start date', null=True) + + def __str__(self): + return self.job + + +class JobLogs(models.Model): + job_stats = models.ForeignKey(JobStats, on_delete=models.CASCADE) + log = models.TextField(verbose_name="job logs") + function_name = models.TextField(verbose_name="Function name") + date = models.DateTimeField('Log date', null=True, auto_now_add=True) + diff --git a/myworld1/members/tasks.py b/myworld1/members/tasks.py new file mode 100644 index 0000000..2cc39b1 --- /dev/null +++ b/myworld1/members/tasks.py @@ -0,0 +1,83 @@ +import datetime +from myworld1.celery import app +from .models import Job, Blog, JobStats, JobLogs +import requests +from bs4 import BeautifulSoup +from dateutil.parser import parse +import pytz + +utc=pytz.UTC + +@app.task(bind=True, name="extract") +def extract(self, job_id): + job_obj = Job.objects.get(pk=job_id) + job_stats_obj = JobStats(job=job_obj, status="IN PROGRESS", start_date=datetime.datetime.now(), no_of_blogs_extracted=0) + job_stats_obj.save() + JobLogs(job_stats=job_stats_obj, log="Extraction stated", function_name="extract", date=datetime.datetime.now()).save() + start_date = job_obj.start_date + end_date = job_obj.end_date + start_id = job_obj.start_no + no_of_articles = job_obj.no_of_blogs + url = "https://blog.python.org/" + try: + data = requests.get(url) + page_soup = BeautifulSoup(data.text, 'html.parser') + + blogs = page_soup.select('div.date-outer') + article_count = 0 + counter = 1 + for blog in blogs: + article_count += 1 + if start_id and article_count < int(start_id): + continue + if no_of_articles and counter > int(no_of_articles): + continue + date = blog.select('.date-header span')[0].get_text() + + converted_date = parse(date) + JobLogs(job_stats=job_stats_obj, log=f"Extracting {article_count}", function_name="extract", date=datetime.datetime.now()).save() + if start_date and utc.localize(converted_date) < start_date: + continue + if end_date and utc.localize(converted_date) > end_date: + continue + + post = blog.select('.post')[0] + + title = "" + title_bar = post.select('.post-title') + if len(title_bar) > 0: + title = title_bar[0].text + else: + title = post.select('.post-body')[0].contents[0].text + + # getting the author and blog time + post_footer = post.select('.post-footer')[0] + + author = post_footer.select('.post-author span')[0].text + + time = post_footer.select('abbr')[0].text + + blog_obj = Blog(title=title, author=author, release_date=date, blog_time=time) + blog_obj.save() + job_stats_obj.no_of_blogs_extracted += job_stats_obj.no_of_blogs_extracted + job_stats_obj.save() + + print("\nTitle:", title.strip('\n')) + print("Date:", date, ) + print("Time:", time) + print("Author:", author) + counter += 1 + JobLogs(job_stats=job_stats_obj, log=f"Total {counter} articles extracted: ", function_name="extract", date=datetime.datetime.now()).save() + job_stats_obj.end_date = datetime.datetime.now() + job_stats_obj.total_blogs = article_count + job_stats_obj.status = "COMPLETED" + job_stats_obj.save() + JobLogs(job_stats=job_stats_obj, log="Extraction Done", function_name="extract", date=datetime.datetime.now()).save() + except Exception as ex: + JobLogs(job_stats=job_stats_obj, log=str(ex), function_name="extract", date=datetime.datetime.now()).save() + job_stats_obj.end_date = datetime.datetime.now() + job_stats_obj.total_blogs = article_count + job_stats_obj.status = "FAILED" + job_stats_obj.save() + JobLogs(job_stats=job_stats_obj, log="Extraction Done", function_name="extract", date=datetime.datetime.now()).save() + diff --git a/myworld1/members/templates/.myfirst.html.swp b/myworld1/members/templates/.myfirst.html.swp new file mode 100644 index 0000000000000000000000000000000000000000..46e64193e825cf731f8a9f09056ba1c39374a2fc GIT binary patch literal 12288 zcmeI&Jx;?g6bEpZjRAq+0xrzdZCMziEVL2~s0@UP(P>PZ){djZ73u;ZA^;P*(D)!p+{`?E3bq H&&&1?X)9OZ literal 0 HcmV?d00001 diff --git a/myworld1/members/templates/myfirst.html b/myworld1/members/templates/myfirst.html new file mode 100644 index 0000000..ef1398d --- /dev/null +++ b/myworld1/members/templates/myfirst.html @@ -0,0 +1,9 @@ + + + + +

Hello World!

+

Welcome to my first Django project!

+ + + diff --git a/myworld1/members/tests.py b/myworld1/members/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/myworld1/members/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/myworld1/members/urls.py b/myworld1/members/urls.py new file mode 100644 index 0000000..3c63964 --- /dev/null +++ b/myworld1/members/urls.py @@ -0,0 +1,20 @@ +from django.urls import path +from . import views + +urlpatterns = [ + path('', views.index, name='index'), + path('rest/student/', views.StudentView.as_view()), + path('rest/student/', views.StudentView.as_view()), + path('rest/student/', views.StudentView.as_view()), + path('rest/student/', views.StudentView.as_view()), + path('rest/student/', views.StudentView.as_view()), + path('rest/student/', views.StudentView.as_view()), + path('start_python_blog_scraping', views.python_blog_scrap, name='triger'), + path('rest/student/', views.StudentView.as_view()), + path('rest/student/', views.StudentView.as_view()), + path('rest/student/', views.StudentView.as_view()), + path('start_python_blog_scraping', views.python_blog_scrap, name='triger'), + path('rest/blog/', views.BlogView.as_view()), + path('python_blog_scraping/', views.python_blog_scraping, name="scraping") +] + diff --git a/myworld1/members/views.py b/myworld1/members/views.py new file mode 100644 index 0000000..7620be0 --- /dev/null +++ b/myworld1/members/views.py @@ -0,0 +1,104 @@ +from django.shortcuts import render + +# Create your views here. +from django.http import HttpResponse + +from django.template import loader +from django.views import View +from .models import Students +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator +from django.views import View +from .models import Students, Blog +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt +from django.utils.decorators import method_decorator +from . import apps +from members.tasks import extract +from django.shortcuts import redirect + +def index(request): + template = loader.get_template('myfirst.html') + return HttpResponse(template.render()) + + + + +@method_decorator(csrf_exempt, name='dispatch') +class BlogView(View): + def post(self, request): + start_date = request.POST.get('start_date', None) + end_date = request.POST.get('end_date', None) + no_of_articles = request.POST.get('no_of_articles', None) + start_id = request.POST.get('start_id', None) + + apps.start_extraction(start_date=start_date, end_date=end_date, no_of_articles=no_of_articles, start_id=start_id) + + blog_model_list = Blog.objects.filter() + + blogs = [] + for blog in blog_model_list: + data = { + "Title": blog.title, + "Release Date": blog.release_date, + "Author": blog.author, + "Blog time": blog.blog_time + } + blogs.append(data) + + return JsonResponse({'status': 'success', "Blogs": blogs}, status=200) + + + + +class StudentView(View): + + def get(self, request, rolno=None, branch=None): + student_model_list = [] + try: + if rolno: + student_model_list = Students.objects.filter(roll_number=rolno) + elif branch: + student_model_list = Students.objects.filter(branch=branch) + except Students.DoesNotExist: + return JsonResponse({'status': 'failed', "students": None}, status=400) + students = [] + for student in student_model_list: + data = { + "first_name" : student.first_name, + "last_name": student.last_name, + "address": student.address, + "roll_number": student.roll_number, + "mobile": student.mobile, + "branch": student.branch + } + students.append(data) + return JsonResponse({'status': 'success', "students": students}, status=200) + + def post(self, request): + if not request.POST.get('first_name') or not request.POST.get('last_name') or not request.POST.get('address') or not request.POST.get('roll_number') or not request.POST.get('mobile'): + return JsonResponse({'status': 'failed', "message" : "all fields required"}, status=500) + + Students.objects.create( + first_name= request.POST.get('first_name'), + last_name= request.POST.get('last_name'), + address= request.POST.get('address'), + roll_number= request.POST.get('roll_number'), + mobile= request.POST.get('mobile'), + branch= request.POST.get('branch')) + return JsonResponse({'status': 'sucess'}, status=200) + + +def python_blog_scrap(request): + apps.start_extraction() + return JsonResponse({'status': 'sucess', "message" : "Extracted and populated the table."}, status=200) + + + + +def python_blog_scraping(request, job_id): + extract.delay(job_id) + return redirect('/admin/members/job/') + + diff --git a/myworld1/myworld1/__init__.py b/myworld1/myworld1/__init__.py new file mode 100644 index 0000000..15d7c50 --- /dev/null +++ b/myworld1/myworld1/__init__.py @@ -0,0 +1,5 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ('celery_app',) diff --git a/myworld1/myworld1/__pycache__/__init__.cpython-310.pyc b/myworld1/myworld1/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7dea43410a774a65731218401bbf958254c3e60 GIT binary patch literal 238 zcmd1j<>g`kf@g(7sq29BV-N=!FabFZKwK;WBvKes7;_kM8KamO8B&;{m{XX888lg5 zG6JPE8E-Ks78C#}uH@95)S}9GAkR;eDj69BJ9J;?w7 literal 0 HcmV?d00001 diff --git a/myworld1/myworld1/__pycache__/celery.cpython-310.pyc b/myworld1/myworld1/__pycache__/celery.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62dba02b5ffcb024488d3d18ede58de956278b1a GIT binary patch literal 733 zcmYjNO>YuG7@nD3V4($DLOhx1l>-nHPfcTLTGFVMCN0LuWi#wMY?m%GI5Sis-s-j9 zjheJa|A0ScuAcfAdN962Mf)c6%=`WEsA{!BP`>=Ei_ZZezr1l*3TW)$GdThfpg9@% zpXGGI280fblHVo*4zgMuiGi%^6TL>Jy$xGY@}N}voCSc7$_ehV^+IZ<3j ztUwL!T~OG7%})fY*MJ_AX8jr`l3tU#;;zVrp1b>dFP`=G598x*zyG|qe;mI$+9amZylDZ&=SgD=z)(C8r=*}K>axx4V8b+hgkA36_hsNLWElwRr6Y2z-L{n%Q&7jL@7R{k6?-GPl zC(%4w0KR#2HSDj2{dK~%cy$WhP^ZyNbq2IvhG!Oiqt2m>dIf!}&cm~SzEiKFTko&o zq5bRV_ChQi??mI>XuKDV_ZMOxhSeKrNRDN*`a>=jOBeTbt7T^>_B>*>XlBoLGv@J; z?HXt)K;a!#&PBz3WXMRWD+YwJ>B&p&U={-1(-~PnirNqc0GW! zU7R5npqsi!Y%2qQuT4N7G4S`G9;e%$=g=3q9I~5K=xXTcZ`x)KTRDKsdA38EG}p5? z*M;}mIJ4I=jEq63m$CQG&ZAoh6k+0K4t1l0>HpE?;=Mjae?XqZvK-^Lbgy0Y*CHnn zi{fU-#fykM@g8Wmm)+Yd5Az*uf8|d0QU1_rzR5qon_qtBuehBz$TFJE)dzPvw{%x) zD1P#h5H>bq*iGGN+tkY@{mWfwp=n#5OBzDxXMbjJx8t?_YXh5s2(~=$*um7lGC-u5 zQtWB|L~$Gl&t;Ol&r=O9kC=s8H!H3bwUJT43{HBxw?a41A=!SP1<> zykkKQ90Zff7n+|RR1}#e^vTQJy*{bzc_okh1{z=ynJ`>SKL6V2cip z?YBdOHTaY?!_4gy96UtNjs3!vu50_GR=#owgESBJ_2vQ8j_~uktDCsy1Z6_|hs zDRf+e`nd?k!=__EWeN?D#}?|E!0F<*P-C9|Fyals!VC^v`W58B)&AoF<}Ysiia?K+?RRIAGV!WWMn(H05v7W^CrktGq{(>sRe--wi>#+d^hD8rVE>05)1bdz2~ z4VLO{v&7Q! zaC_A1iaQQhiT`Tg++p51I;C*?A`Px9fKX#BvFUHK?qyvRK{N7B?iV zP%BCRSgJ^xT7yJSf#bB&(z7Z{1T#ei%N0N_1lrkj&%J=b*L@XxnyjMcvaSt3@mf32C(!cOybR*Sj^&CcUo>!8vBBA0QPw27U=& zPcga&9UmJVO>vV$e3Bo6_BoYErTKK6k8^P@6_0=B|K>mQlksG*?lt}s>`eiU?|c5@ YXLt~Umf(kZp68&&C%H*3#ijoF7g(65w*UYD literal 0 HcmV?d00001 diff --git a/myworld1/myworld1/__pycache__/urls.cpython-310.pyc b/myworld1/myworld1/__pycache__/urls.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b2cbf27e15935f778880c26164bcc77d4fd6bc42 GIT binary patch literal 1018 zcmb7DU2oGc6m`;cEm?>56a3ai+G1_mQw7ryY%mG&z=x3f(z)?%>n^q<+huGoJnYx- zm-5OJe}MW;e+fv!j(5#5LAA;x+K;^oe=_YlwHabNV_k*v?Z#Dt_D-oB=V z(K9(;S?;CLbY?77celot;+R&}ETQmun!aBEy%fMkQP^pnlt%<7=xC80{Mk&R$@V;^*#)K(cE|y%+&8A&p6a&pp^AqMxB@4#_ zm|=Uz(u>brA(tDamn&T~nwz!_>2Xe<2|>B0QdP$K5v4P06gryls%l@ZWj6Se79Si0 z16vO9Q#89c7!D6xUkjt)kZP{r=`hs8EPY;b=Z>amOl-0KO@;9but7-PDBj3cmM8brS5=yIUjwO6+a` zh3&@`3o~Y1-$>6HUWgmFFco{jJ-^U%sR1meeDtyX;K$g8`$|3n$FD&Z_b TVmH7oA%p$>XwVr%{hz-8u@^mS literal 0 HcmV?d00001 diff --git a/myworld1/myworld1/__pycache__/wsgi.cpython-310.pyc b/myworld1/myworld1/__pycache__/wsgi.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3166f0fee77eb5889250d743a34c40d0d1de0ae6 GIT binary patch literal 575 zcmYjP!EO^V5cN7~Qx=M-ggE;eiLRkSLP7{pDIp4i^pK)TIj}M7$;PSIv$EZU^va39 zfFpl|eC5P1aKf7qk&!(1_-W?785@s}Qxfg@$ASEs5b~=D_N^U**KxTg(Md8PGLcj# zlO*mfO{>Ve-gW=DzGd{zH+7F>F+oXGU_JbVF^BO?(#hu;Zg7H`Dk#FkNcOG!_i=zzrW~xc-Ox@ z*|ncJ@LrkHbvvPx=TSqR2Z}B{*ec12*g)3QwKOt?4w$8~XhIqjCCtQ)_Tlax;Ivl&gykIdbu6`tN1;L PYbe^}=*~TQpQiLLp_Q}D literal 0 HcmV?d00001 diff --git a/myworld1/myworld1/asgi.py b/myworld1/myworld1/asgi.py new file mode 100644 index 0000000..5f6a330 --- /dev/null +++ b/myworld1/myworld1/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for myworld1 project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myworld1.settings') + +application = get_asgi_application() diff --git a/myworld1/myworld1/celery.py b/myworld1/myworld1/celery.py new file mode 100644 index 0000000..03ccf50 --- /dev/null +++ b/myworld1/myworld1/celery.py @@ -0,0 +1,35 @@ +import os +from celery import Celery + +# Set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myworld1.settings') + +app = Celery('myworld1') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django apps. +app.autodiscover_tasks() + +app.conf.beat_schedule = { + #Scheduler Name + 'run-task-ten-seconds': { + # Task Name (Name Specified in Decorator) + 'task': 'extract', + # Schedule + 'schedule': 60.0, + # Function Arguments + 'args': (1,) + } +} + + +@app.task(bind=True) +def debug_task(self): + print(f'Request: {self.request!r}') + + diff --git a/myworld1/myworld1/settings.py b/myworld1/myworld1/settings.py new file mode 100644 index 0000000..28bc5f4 --- /dev/null +++ b/myworld1/myworld1/settings.py @@ -0,0 +1,141 @@ +""" +Django settings for myworld1 project. + +Generated by 'django-admin startproject' using Django 4.2. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/4.2/ref/settings/ +""" + +from pathlib import Path +import os + +# Build paths inside the project like this: BASE_DIR / 'subdir'. +BASE_DIR = Path(__file__).resolve().parent.parent + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/4.2/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'django-insecure-di5t*_$2r$6+4u_j6%)04vpc^49&428z6ru^dnlcc@*%u#ar_b' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [ +'0.0.0.0','localhost'] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'members.apps.MembersConfig', + ] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'myworld1.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'myworld1.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/4.2/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': 'member_db', + 'USER': 'postgres', + 'PASSWORD': '123456', + 'HOST': 'psql-db', + 'PORT': 5432, + } +} + + +# Password validation +# https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/4.2/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/4.2/howto/static-files/ + +STATIC_URL = 'static/' + +# Default primary key field type +# https://docs.djangoproject.com/en/4.2/ref/settings/#default-auto-field + +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + + + +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = 'America/Los_Angeles' +# This configures rabbitmq as the datastore between Django + Celery +CELERY_BROKER_URL = 'amqp://{0}:{1}@{2}:{3}/{4}'.format( + os.environ["RABBITMQ_DEFAULT_USER"], os.environ["RABBITMQ_DEFAULT_PASS"], + os.environ["BROKER_HOST"], os.environ["BROKER_PORT"], + os.environ["RABBITMQ_DEFAULT_VHOST"]) diff --git a/myworld1/myworld1/urls.py b/myworld1/myworld1/urls.py new file mode 100644 index 0000000..c7c68c3 --- /dev/null +++ b/myworld1/myworld1/urls.py @@ -0,0 +1,23 @@ +""" +URL configuration for myworld1 project. + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/4.2/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import include, path + +urlpatterns = [ + path('members/', include('members.urls')), + path('admin/', admin.site.urls), +] diff --git a/myworld1/myworld1/wsgi.py b/myworld1/myworld1/wsgi.py new file mode 100644 index 0000000..2aed5e7 --- /dev/null +++ b/myworld1/myworld1/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for myworld1 project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/4.2/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myworld1.settings') + +application = get_wsgi_application() diff --git a/myworld1/scrapped_data.html b/myworld1/scrapped_data.html new file mode 100644 index 0000000..835c686 --- /dev/null +++ b/myworld1/scrapped_data.html @@ -0,0 +1,37 @@ +
+ +

+Python 3.12.0 beta 1 released +

+
+
+
+
+

I'm pleased to announce the release of Python 3.12 beta 1 (and feature freeze for Python 3.12).

https://www.python.org/downloads/release/python-3120b1/

This is a beta preview of Python 3.12


Python 3.12 is still in development. This release, 3.12.0b1, is the first of four planned beta release previews of 3.12.

Beta release previews are intended to give the wider community the opportunity to test new features and bug fixes and to prepare their projects to support the new feature release.

We strongly encourage maintainers of third-party Python projects to test with 3.12 during the beta phase and report issues found to [the Python bug tracker (Issues · python/cpython · GitHub) as soon as possible. While the release is planned to be feature complete entering the beta phase, it is possible that features may be modified or, in rare cases, deleted up until the start of the release candidate phase (Monday, 2023-07-31). Our goal is to have no ABI changes after beta 4 and as few code changes as possible after 3.12.0rc1, the first release candidate. To achieve that, it will be extremely important to get as much exposure for 3.12 as possible during the beta phase.

Please keep in mind that this is a preview release and its use is not recommended for production environments.


Major new features of the 3.12 series, compared to 3.11

Some of the new major new features and changes in Python 3.12 are:

  • New type annotation syntax for generic classes (PEP 695).
  • More flexible f-string parsing, allowing many things previously disallowed (PEP 701).
  • Even more improved error messages. More exceptions potentially caused by typos now make suggestions to the user.
  • Many large and small performance improvements (like PEP 709).
  • Support for the Linux perf profiler to report Python function names in traces.
  • The deprecated wstr and wstr_length members of the C implementation of unicode objects were removed, per PEP 623.
  • In the unittest module, a number of long deprecated methods and classes were removed. (They had been deprecated since Python 3.1 or 3.2).
  • The deprecated smtpd and distutils modules have been removed (see PEP 594 and PEP 632. The setuptools package (installed by default in virtualenvs and many other places) continues to provide the distutils module.
  • A number of other old, broken and deprecated functions, classes and methods have been removed.
  • Invalid backslash escape sequences in strings now warn with SyntaxWarning instead of DeprecationWarning, making them more visible. (They will become syntax errors in the future.)
  • The internal representation of integers has changed in preparation for performance enhancements. (This should not affect most users as it is an internal detail, but it may cause problems for Cython-generated code.)
  • (Hey, fellow core developer, if a feature you find important is missing from this list, let Thomas know.)

For more details on the changes to Python 3.12, see What’s new in Python 3.12. The next pre-release of Python 3.12 will be 3.12.0b2, currently scheduled for 2023-05-29.


More resources


PEP 693, the Python 3.12 Release Schedule.
Report bugs via GitHub Issues.


And now for something completely different

As the first beta release marks the point at which we fork off the release branch from the main development branch, here’s a poem about forks in the road.

Two roads diverged in a yellow wood,
And sorry I could not travel both
And be one traveler, long I stood
And looked down one as far as I could
To where it bent in the undergrowth;

Then took the other, as just as fair,
And having perhaps the better claim,
Because it was grassy and wanted wear;
Though as for that the passing there
Had worn them really about the same,

And both that morning equally lay
In leaves, no step had trodden black.
Oh, I kept the first for another day!
Yet knowing how way leads on to way,
I doubted if I should ever come back.

I shall be telling this with a sigh
Somewhere ages and ages hence:
Two roads diverged in a wood, and I —
I took the one less traveled by,
And that has made all the difference.

The Road Not Taken, by Robert Frost.

Enjoy the new release


Thanks to all of the many volunteers who help make Python Development and these releases possible! Please consider supporting our efforts by volunteering yourself or through organization contributions to the Python Software Foundation.

Your release team,
Thomas Wouters
Ned Deily
Steve Dower
+
+
+
+
+Posted by +Thomas Wouters + + +at +4:55 PM + + + + + +
+ +
+
+
+ +
+
+ +
+
+
\ No newline at end of file diff --git a/myworld1/web_scrapper.py b/myworld1/web_scrapper.py new file mode 100644 index 0000000..7eac4fd --- /dev/null +++ b/myworld1/web_scrapper.py @@ -0,0 +1,95 @@ +import psycopg2 +import requests +import re +from bs4 import BeautifulSoup, element +import os + +# For the credentials mentioned below, you may refer the docker-compose.yml present in myworld . +db_name = 'member_db' +db_user = 'postgres' +db_pass = '123456' +db_host = 'psql-db' +db_port = '5432' + +# This will create the connection the to postgres database. +conn = psycopg2.connect(dbname=db_name, user=db_user, password=db_pass, host=db_host, port=db_port) + + +def add_row_to_blog(title, author, date, time,content,recommended,path): + # This function will add the entry to database + sql = """INSERT INTO members_blog (title, release_date, blog_time, created_date,content, author,recommended,path) VALUES (%s, %s::DATE, %s::TIME, NOW(),%s,%s,%s,%s)""" + + with conn: + with conn.cursor() as curs: + time=time.replace('\u202f',"") + curs.execute(sql, (title, date, time, content,author,recommended,path)) + + +def truncate_table(): + # This function will delete the existing entries from the database. + with conn: + with conn.cursor() as curs: + curs.execute("TRUNCATE members_blog CASCADE;") + + +def start_extraction(): + print("Extraction started") + url = "https://blog.python.org/" + + # Each time when we add new entry we delete the existing entries. + truncate_table() + data = requests.get(url) + page_soup = BeautifulSoup(data.text, 'html.parser') + + # Getting all the articles + blogs = page_soup.select('div.date-outer') + + fp=open("scrapped_data.html","w") + path=os.path.abspath("scrapped_data.html") + + for blog in blogs: + # loop through each article + date = blog.select('.date-header span')[0].get_text() + + post = blog.select('.post')[0] + #print(post) + fp.write(str(post)) + + content=post.select('.post-body')[0].get_text() + recommended =post.select('.post-share-buttons')[0].get_text() + + + title = "" + title_bar = post.select('.post-title') + if len(title_bar) > 0: + title = title_bar[0].text + else: + title = post.select('.post-body')[0].contents[0].text + + # getting the author and blog time + post_footer = post.select('.post-footer')[0] + + author = post_footer.select('.post-author span')[0].text + + time = post_footer.select('abbr')[0].text + + + # Inserting data into database + add_row_to_blog(title, author, date, time,content,recommended,path) + + print("\nTitle:", title.strip('\n')) + print("Date:", date, ) + print("Time:", time) + print("Author:", author) + print("Content:",content) + print("recommended: ",recommended) + + # print("Number of blogs read:", count) + print( + "\n---------------------------------------------------------------------------------------------------------------\n") + + fp.close() + + +if __name__ == "__main__": + start_extraction() diff --git a/test_celery/__init__.py b/test_celery/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test_celery/__pycache__/__init__.cpython-310.pyc b/test_celery/__pycache__/__init__.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fb161c731365389bce4d234ba4738aa4046e11f5 GIT binary patch literal 164 zcmd1j<>g`kg5(6DR1p0bL?8o3AjbiSi&=m~3PUi1CZpdqPR&Uzs??8<&&p(v`XtYr@jQ>89*z1a);m9fpV&WJv0GUb*YUV<1c3yWNZ}dAEl*^k$c$hFL=cgR zj>r~Rbl=ihDth-Q-}{PhP>#a^pF1rZ53E#1TOY5o8<1dEVc@}eR&2!jn5$Dl?U)CHGI?flkYFD+kZ))aAeeYle__S{u2Cn1|#U4aeVIWH=xMd+4wAsYdA zc_;K4du)7TfiauJ4MVp5-;!1kZp;U=gUC1VJ*7*nfbzO3J*ThQR&J>cz2%-y>qS`u r*s@-nJp3CMbY~&DO&*(7=v^u;8VOf8CcFIzLo~!`lHdfVxc~bH2gY-v literal 0 HcmV?d00001 diff --git a/test_celery/__pycache__/run_tasks.cpython-310.pyc b/test_celery/__pycache__/run_tasks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e5349de78af60dafb61c0c94b7ed2378b436e7b GIT binary patch literal 428 zcmZ`!Jx{|h5VhU7O{;zH7&0kV_@Az%QBi3KF2E|ygvZHVJUwkwpb{E>F#FL`Ck zFF-;JTml0NC%x0XyL(S}#b`7j81v;Jd{IK)8}c_IBv%M$Fe##_COKVDN)%Ha%@zzg zD$1BW1-grH&i8D=p{u&UcfCD%Wt1T60e|Lg&33Vh_o)AWM|JK=gIT}Ei^z_w+=5h9;7$XSZ^TYe+NRIjWL@NtNRy>rCNG7}+`1@} zYw6{US)~R54OYjGzk_+=!FjO+4Yo?`#)$Told=l&d0wcEhKuH3cZ6w3Vj8m{9U}&8 J%noSu@eSeea;5+P literal 0 HcmV?d00001 diff --git a/test_celery/__pycache__/tasks.cpython-310.pyc b/test_celery/__pycache__/tasks.cpython-310.pyc new file mode 100644 index 0000000000000000000000000000000000000000..82231f43a4da0d8e788544fa95913d5b4537723e GIT binary patch literal 502 zcmZ8dJxc>Y5Z#Zvc*H~@AZdf$73M%JMFh1?VcLJgd-i@{jU^3rZYAXxSPVbi?K?Oe&2 z`>-~7>3o2VKD1h?I%b}}db<212KVxPpUkU5C8Av9P9!gtS=f11C-=gMhZ01T0*cwi z(;m1=oU$&PD6Pyg!CCCEUiuEsfGU}g5~s2egow3SL?dcCP{c>Ron_O;HAZEb!A1Dq r0om}6)8KYY3()}ScfGq&55h6R(RESDMyneNmBr6=NC(scc0zvv!BTq! literal 0 HcmV?d00001 diff --git a/test_celery/celery.py b/test_celery/celery.py new file mode 100644 index 0000000..a5041f9 --- /dev/null +++ b/test_celery/celery.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import +from celery import Celery + +app = Celery('test_celery', + broker='amqp://jimmy:jimmy123@localhost/jimmy_vhost', + backend='rpc://', + include=['test_celery.tasks']) + + diff --git a/test_celery/run_tasks.py b/test_celery/run_tasks.py new file mode 100644 index 0000000..24a1442 --- /dev/null +++ b/test_celery/run_tasks.py @@ -0,0 +1,13 @@ +from .tasks import longtime_add +import time + +if __name__ == '__main__': + result = longtime_add.delay(1,2) + # at this time, our task is not finished, so it will return False + print ('Task finished? ', result.ready()) + print ('Task result: ', result.result) + # sleep 10 seconds to ensure the task has been finished + time.sleep(10) + # now the task should be finished and ready method will return True + print ('Task finished? ', result.ready()) + print ('Task result: ', result.result) diff --git a/test_celery/tasks.py b/test_celery/tasks.py new file mode 100644 index 0000000..88f2f1e --- /dev/null +++ b/test_celery/tasks.py @@ -0,0 +1,11 @@ +from __future__ import absolute_import +from test_celery.celery import app +import time + +@app.task +def longtime_add(x, y): + print ('long time task begins') + # sleep 5 seconds + time.sleep(5) + print ('long time task finished') + return x + y