Vivekananda_session3_4VP20CS066_Preetham

Vivekananda_session3_4VP20CS066_preetham/myworld1/docker-compose.yml

@@ -0,0 +1,30 @@
+  version: "3"
+  services:
+   web_service:
+     build:
+       context: ./
+       dockerfile: ./dockerfiles/Dockerfile
+     image: workshop1_web
+     container_name: workshop_web_container
+     stdin_open: true #  docker attach container_id
+     tty: true
+     ports:
+      - "8000:8000"
+     volumes:
+      - .:/root/workspace/site
+
+   psql-db:
+    image: 'postgres:14'
+    container_name: psql-db2.0
+    environment:
+      - PGPASSWORD=123456
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=123456
+    ports:
+      - '5446:5432'
+    volumes:
+      - db:/var/lib/postgresql/data
+
+  volumes:
+    db:
+      driver: local

Vivekananda_session3_4VP20CS066_preetham/myworld1/dockerfiles/Dockerfile

@@ -0,0 +1,20 @@
+  FROM python:3.10.2-alpine3.15
+  # Install required packages
+  # For psycopg2
+  RUN apk update && \
+      apk --no-cache add --virtual build-deps-alpine build-base && \
+      apk --no-cache add --virtual postgresql-deps libpq-dev
+  # Install requirements
+  RUN pip install --upgrade pip
+  #RUN pip install Django psycopg2==2.9.3
+  RUN pip install Django psycopg2==2.9.3 bs4 html5lib requests python-dateutil
+  # Create directories  
+  RUN mkdir -p /root/workspace/src
+  COPY ./  /root/workspace/site
+  # Switch to project directory
+  WORKDIR /root/workspace/site
+
+
+
+
+

Vivekananda_session3_4VP20CS066_preetham/myworld1/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'Batman.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

Vivekananda_session3_4VP20CS066_preetham/myworld1/members/admin.py

@@ -0,0 +1,16 @@
+from django.contrib import admin
+from .models import Students, Blog
+
+class DjStudentAdmin(admin.ModelAdmin):
+    list_display = ("first_name", "last_name", "address", "roll_number", "mobile", "branch")
+    list_filter = ("branch",)
+
+class DjBlogAdmin(admin.ModelAdmin):
+  list_display = ("title", "release_date", "blog_time", "author", "created_date")
+  list_filter = ("author",)
+
+
+# Register your models here.
+admin.site.register(Blog, DjBlogAdmin)
+admin.site.register(Students, DjStudentAdmin) 
+

Vivekananda_session3_4VP20CS066_preetham/myworld1/members/apps.py

@@ -0,0 +1,124 @@
+from django.apps import AppConfig
+import psycopg2
+import requests
+import re
+from bs4 import BeautifulSoup, element
+import datetime
+from dateutil.parser import parse
+
+
+db_name = 'member_db2'
+db_user = 'postgres'
+db_pass = '123456'
+db_host = 'psql-db2.0'
+db_port = '5432'
+
+conn = psycopg2.connect(dbname=db_name, user=db_user, password=db_pass, host=db_host, port=db_port)
+
+
+def add_row_to_blog(title, author, date, time):
+    sql = """INSERT INTO members_blog (title, release_date, blog_time, author, created_date) VALUES (%s, %s::DATE, %s::TIME, %s, NOW())"""
+
+    with conn:
+        with conn.cursor() as curs:
+            curs.execute(sql, (title, date, time, author, content, recommended, html))
+
+
+def truncate_table():
+    print("Truncating contents all the tables")
+    with conn:
+        with conn.cursor() as curs:
+            curs.execute("TRUNCATE members_blog CASCADE;")
+
+
+def start_extraction(start_date=None, end_date=None, no_of_articles=None, start_id=None):
+    print("Extraction started")
+    url = "https://blog.python.org/"
+
+    data = requests.get(url)
+    page_soup = BeautifulSoup(data.text, 'html.parser')
+
+    if start_date:
+        start_date = parse(start_date)
+    if end_date:
+        end_date = parse(end_date)
+
+    blogs = page_soup.select('div.date-outer')
+    truncate_table()
+    article_count = 0
+    counter = 1
+    for blog in blogs:
+        article_count += 1
+        if start_id and article_count < int(start_id):
+            continue
+        if no_of_articles and counter > int(no_of_articles):
+            continue
+        date = blog.select('.date-header span')[0].get_text()
+
+        converted_date = parse(date)
+
+        if start_date and converted_date < start_date:
+            continue
+        if end_date and converted_date > end_date:
+            continue
+
+        post = blog.select('.post')[0]
+
+        title = ""
+        title_bar = post.select('.post-title')
+        if len(title_bar) > 0:
+            title = title_bar[0].text
+        else:
+            title = post.select('.post-body')[0].contents[0].text
+
+        # getting the author and blog time
+        post_footer = post.select('.post-footer')[0]
+
+        author = post_footer.select('.post-author span')[0].text
+
+        time = post_footer.select('abbr')[0].text
+
+        post_header = post.select('.post-header')[0]
+
+        '''contents = page_soup.find_all('div',class_='date-posts')
+        #print(contents)
+        for con in contents:'''
+        content = post.select('.post-body')[0].text
+        html = 'S:\web_scapping\python_blogs.html'
+        with open(html, 'w', encoding='utf-8') as f:
+            f.write(data.text)
+
+
+        #con = content.find('p', class_='post-body entry-content').p.text
+
+        #contents =post_header.select('div', class_='post-body entry-content')
+        google = page_soup.find_all('div', class_='widget LinkList', id='LinkList1')
+        # print(google)
+        for gg in google:
+           recommended = gg.select('.widget-content')[0].li.a['href']
+            #rem2 = gg.find('div', class_='widget-content')
+            #print(rem2)
+
+        #print(rem2)
+        # Inserting data into database
+        add_row_to_blog(title, date, time, author, content, recommended, html)
+        print("\nTitle:", title.strip('\n'))
+        print(f'''Posted date: {date}
+        Posted time: {time}
+        Author: {author}
+        Content: {content}
+        Recommended this on google: {recommended}
+        HTML saved to: {html}''')
+
+        # print("Number of blogs read:", count)
+        print(
+            "\n---------------------------------------------------------------------------------------------------------------\n")
+        counter += 1
+
+
+if __name__ == "__main__":
+    start_extraction()
+
+
+class MembersConfig(AppConfig):
+    name = 'members'

Vivekananda_session3_4VP20CS066_preetham/myworld1/members/migrations/0001_initial.py

@@ -0,0 +1,40 @@
+# Generated by Django 4.2.1 on 2023-06-15 01:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Blog',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=500)),
+                ('release_date', models.DateTimeField(verbose_name='Realse Date')),
+                ('blog_time', models.CharField(max_length=50)),
+                ('author', models.CharField(max_length=200)),
+                ('created_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Created Date')),
+                ('content', models.CharField(max_length=20000)),
+                ('recommended', models.CharField(max_length=800)),
+                ('html', models.CharField(max_length=500)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Students',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('first_name', models.CharField(max_length=200)),
+                ('last_name', models.CharField(max_length=200)),
+                ('address', models.CharField(max_length=200)),
+                ('roll_number', models.IntegerField()),
+                ('mobile', models.CharField(max_length=10)),
+                ('branch', models.CharField(choices=[('BA', 'BA'), ('B.COM', 'B.COM'), ('MBA', 'MBA'), ('CA', 'CA')], max_length=10, null=True)),
+            ],
+        ),
+    ]

Vivekananda_session3_4VP20CS066_preetham/myworld1/members/models.py

@@ -0,0 +1,34 @@
+from django.db import models
+
+BRANCH_CHOICES = (
+    ("BA", "BA"),
+    ("B.COM", "B.COM"),
+    ("MBA", "MBA"),
+    ("CA", "CA"),
+)
+
+
+# Create your models here.
+class Students(models.Model):
+    first_name = models.CharField(max_length=200)
+    last_name = models.CharField(max_length=200)
+    address = models.CharField(max_length=200)
+    roll_number = models.IntegerField()
+    mobile = models.CharField(max_length=10)
+    branch = models.CharField(max_length=10, choices=BRANCH_CHOICES, null=True)
+
+    def __str__(self):
+        return self.first_name + " " + self.last_name
+
+
+class Blog(models.Model):
+    title = models.CharField(max_length=500)
+    release_date = models.DateTimeField('Realse Date')
+    blog_time = models.CharField(max_length=50)
+    author = models.CharField(max_length=200)
+    created_date = models.DateTimeField('Created Date', auto_now_add=True, null=True)
+    content= models.CharField(max_length=20000)
+    recommended= models.CharField(max_length=800)
+    html= models.CharField(max_length=500)
+    def __str__(self):
+        return self.title

Vivekananda_session3_4VP20CS066_preetham/myworld1/members/myfirst.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<body>
+
+<h1>Hello World!</h1>
+<p>Welcome to my first Django project!</p>
+
+</body>
+</html>

Vivekananda_session3_4VP20CS066_preetham/myworld1/members/templates/myfirst.html

@@ -0,0 +1,9 @@
+<!DOCTYPE html>
+<html>
+<body>
+
+<h1>Hello World!</h1>
+<p>Welcome to my first Django project!</p>
+
+</body>
+</html>

Vivekananda_session3_4VP20CS066_preetham/myworld1/members/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

Vivekananda_session3_4VP20CS066_preetham/myworld1/members/urls.py

@@ -0,0 +1,10 @@
+from django.urls import path
+from . import views
+
+urlpatterns = [
+    path('students/<int:rolno>', views.StudentView.as_view()),
+    path('students/', views.StudentView.as_view()),
+    path('students/<str:branch>', views.StudentView.as_view()),
+    path('start_python_blog_scraping', views.python_blog_scrap, name='triger'),
+    path('rest/blog/', views.BlogView.as_view()),
+    ]