Vivekananda_Session4_4VP20CS005_Ankita Hegde

myworld1/breakpoint_demo.py

@@ -0,0 +1,16 @@
+breakpoint()
+
+print("This is line number 1")
+
+a = 1
+b = 2
+
+sum1 = a + b
+
+print(f"The sum is {sum}")
+
+for i in range(5):
+	sum1 += 1
+
+print(f"The sum at the end is {sum1}")
+

myworld1/docker-compose.yml

@@ -0,0 +1,63 @@
+ version: "3"
+ services:
+  web_service:
+    build:
+      context: ./
+      dockerfile: ./dockerfiles/Dockerfile
+    image: workshop1_web
+    container_name: workshop_web_container
+    stdin_open: true #  docker attach container_id
+    tty: true
+    environment:
+      - RABBITMQ_DEFAULT_USER=myuser
+      - RABBITMQ_DEFAULT_PASS=mypassword
+      - BROKER_HOST=service-rabbitmq
+      - RABBITMQ_DEFAULT_VHOST=extractor
+      - BROKER_PORT=5672
+    ports:
+     - "8000:8000"
+    volumes:
+     - .:/root/workspace/site
+  psql-db:
+    image: 'postgres:14'
+    container_name: psql-db
+    environment:
+      - PGPASSWORD=123456
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=123456
+    ports:
+      - '5432:5432'
+  service-rabbitmq:
+    container_name: "service_rabbitmq"
+    image: rabbitmq:3.8-management-alpine
+    environment:
+      - RABBITMQ_DEFAULT_USER=myuser
+      - RABBITMQ_DEFAULT_PASS=mypassword
+      - BROKER_HOST=service-rabbitmq
+      - RABBITMQ_DEFAULT_VHOST=extractor
+      - BROKER_PORT=5672
+    ports:
+      - '5672:5672'
+      - '15676:15672'
+  worker:
+    build:
+      context: ./
+      dockerfile: ./dockerfiles/Dockerfile
+    image: workshop1_web
+    container_name: worker
+    stdin_open: true #  docker attach container_id
+    tty: true
+    environment:
+     - RABBITMQ_DEFAULT_USER=myuser
+     - RABBITMQ_DEFAULT_PASS=mypassword
+     - BROKER_HOST=service-rabbitmq
+     - RABBITMQ_DEFAULT_VHOST=extractor
+     - BROKER_PORT=5672
+    ports:
+     - "4356:8000"
+    volumes:
+     - .:/root/workspace/site
+ volumes:
+    db:
+      driver: local
+

myworld1/dockerfiles/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.10.2-alpine3.15
+# Install required packages
+# For psycopg2
+RUN apk update && \
+apk --no-cache add --virtual build-deps-alpine build-base && \
+apk --no-cache add --virtual postgresql-deps libpq-dev
+# Install requirements
+RUN pip install --upgrade pip
+RUN pip install Django psycopg2==2.9.3 bs4 html5lib requests python-dateutil celery curl
+# Create directories
+RUN mkdir -p /root/workspace/src
+COPY ./  /root/workspace/site
+# Switch to project directory
+WORKDIR /root/workspace/site

myworld1/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myworld1.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

myworld1/members/admin.py

@@ -0,0 +1,60 @@
+from django.contrib import admin
+from .models import Students, Blog, Job, JobLogs, JobStats
+from django.urls import reverse
+from django.utils.html import format_html
+
+class DjStudentAdmin(admin.ModelAdmin):
+    list_display = ("first_name", "last_name", "address", "roll_number", "mobile", "branch")
+    list_filter = ("branch",)
+
+
+class DjBlogAdmin(admin.ModelAdmin):
+  list_display = ("title", "release_date", "blog_time", "created_date","content","author", "recommended","path")
+  list_filter = ("author",)
+
+
+
+class DjJob(admin.ModelAdmin):
+
+  def view_stats(self, obj):
+    path = "../jobstats/?q={}".format(obj.pk)
+    return format_html(f'''<a class="button" href="{path}">stats</a>''')
+
+  view_stats.short_description = 'Stats'
+  view_stats.allow_tags = True
+
+  list_display = ("job_name", "start_date", "end_date", "no_of_blogs", "start_no", "created_date", "view_stats")
+  list_filter = ("job_name", "start_date")
+  readonly_fields = ("created_date",)
+
+  def run(self, obj):
+    return format_html('<a class="button" href="{}">RUN</a>', reverse('scraping', args=(str(obj.pk))))
+
+  run.short_description = 'Run'
+  run.allow_tags = True
+  list_display = ("job_name", "start_date", "end_date", "no_of_blogs", "start_no", "created_date", "run", "view_stats")
+
+class DjJobStats(admin.ModelAdmin):
+  def view_logs(self, obj):
+    path = "../joblogs/?q={}".format(obj.pk)
+    return format_html(f'''<a class="button" href="{path}">Logs</a>''')
+
+  view_logs.short_description = 'Stats'
+  view_logs.allow_tags = True
+  list_display = ("job", "status", "view_logs", "total_blogs", "no_of_blogs_extracted", "start_date", "end_date")
+  search_fields = ('job__pk',)
+
+class DjJobLogs(admin.ModelAdmin):
+  list_display = ("date", "log", "function_name")
+  search_fields = ('job_stats__pk',)
+
+
+
+
+# Register your models here.
+admin.site.register(Students, DjStudentAdmin)   
+admin.site.register(Blog, DjBlogAdmin)
+
+admin.site.register(Job, DjJob)
+admin.site.register(JobStats, DjJobStats)
+admin.site.register(JobLogs, DjJobLogs)

myworld1/members/apps.py

@@ -0,0 +1,108 @@
+from django.apps import AppConfig
+
+
+from django.apps import AppConfig
+
+import psycopg2
+import requests
+import re
+from bs4 import BeautifulSoup, element
+from django.apps import AppConfig
+import psycopg2
+import requests
+import re
+from bs4 import BeautifulSoup, element
+import datetime
+from dateutil.parser import parse
+
+
+db_name = 'member_db'
+db_user = 'postgres'
+db_pass = '123456'
+db_host = 'psql-db'
+db_port = '5432'
+
+conn = psycopg2.connect(dbname=db_name, user=db_user, password=db_pass, host=db_host, port=db_port)
+
+def add_row_to_blog(title, author, date, time):
+    sql = """INSERT INTO members_blog (title, release_date, blog_time, author, created_date) VALUES (%s, %s::DATE, %s::TIME, %s, NOW())"""
+
+    with conn:
+        with conn.cursor() as curs:
+            time=time.replace('\u202f',"")
+            curs.execute(sql, (title, date, time, author))
+
+def truncate_table():
+    print("Truncating contents all the tables")
+    with conn:
+        with conn.cursor() as curs:
+            curs.execute("TRUNCATE members_blog CASCADE;")
+
+
+def start_extraction(start_date=None, end_date=None, no_of_articles=None, start_id = None):
+  print("Extraction started")
+  url = "https://blog.python.org/"
+
+  data = requests.get(url)
+  page_soup = BeautifulSoup(data.text, 'html.parser')
+
+  if start_date:
+      start_date = parse(start_date)
+  if end_date:
+      end_date = parse(end_date)
+
+  blogs = page_soup.select('div.date-outer')
+  truncate_table()
+  article_count = 0
+  counter = 1
+  for blog in blogs:
+      article_count += 1
+      if start_id and article_count < int(start_id):
+          continue
+      if no_of_articles and counter > int(no_of_articles):
+          continue
+      date = blog.select('.date-header span')[0].get_text()
+
+      converted_date = parse(date)
+
+      if start_date and converted_date < start_date:
+          continue
+      if end_date and converted_date > end_date:
+          continue
+
+      post = blog.select('.post')[0]
+
+      title = ""
+      title_bar = post.select('.post-title')
+      if len(title_bar) > 0:
+          title = title_bar[0].text
+      else:
+          title = post.select('.post-body')[0].contents[0].text
+
+      # getting the author and blog time
+      post_footer = post.select('.post-footer')[0]
+
+      author = post_footer.select('.post-author span')[0].text
+
+      time = post_footer.select('abbr')[0].text
+
+      add_row_to_blog(title, author, date, time)
+
+      print("\nTitle:", title.strip('\n'))
+      print("Date:", date, )
+      print("Time:", time)
+      print("Author:", author)
+
+      # print("Number of blogs read:", count)
+      print(
+          "\n---------------------------------------------------------------------------------------------------------------\n")
+      counter += 1
+
+
+if __name__ == "__main__":
+    start_extraction()
+
+
+class MembersConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'members'

myworld1/members/migrations/0001_initial.py

@@ -0,0 +1,22 @@
+# Generated by Django 4.2 on 2023-04-25 05:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Members',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('firstname', models.CharField(max_length=255)),
+                ('lastname', models.CharField(max_length=255)),
+            ],
+        ),
+    ]

myworld1/members/migrations/0002_students_delete_members.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2 on 2023-04-25 08:51
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Students',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('first_name', models.CharField(max_length=200)),
+                ('last_name', models.CharField(max_length=200)),
+                ('address', models.CharField(max_length=200)),
+                ('roll_number', models.IntegerField()),
+                ('mobile', models.CharField(max_length=10)),
+                ('branch', models.CharField(choices=[('BA', 'BA'), ('B.COM', 'B.COM'), ('MBA', 'MBA'), ('CA', 'CA')], max_length=10)),
+            ],
+        ),
+        migrations.DeleteModel(
+            name='Members',
+        ),
+    ]

myworld1/members/migrations/0003_blog.py

@@ -0,0 +1,24 @@
+# Generated by Django 4.2.1 on 2023-05-23 04:25
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0002_students_delete_members'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Blog',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=500)),
+                ('release_date', models.DateTimeField(verbose_name='Realse Date')),
+                ('blog_time', models.CharField(max_length=50)),
+                ('author', models.CharField(max_length=200)),
+                ('created_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Created Date')),
+            ],
+        ),
+    ]

myworld1/members/migrations/0004_blog_content.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.1 on 2023-05-23 09:53
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0003_blog'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='blog',
+            name='content',
+            field=models.CharField(default=None, max_length=20000),
+            preserve_default=False,
+        ),
+    ]