Vivekananda_session3_4VP20CS068_Raghavendra

hw3/breakpoint_demo.py

@@ -0,0 +1,14 @@
+print("This is line number 1")
+
+a = 1
+b = 2
+
+sum = a + b
+
+print(f"The sum is {sum}")
+
+for i in range(5):
+	sum += 1
+
+print(f"The sum at the end is {sum}")
+

hw3/docker-compose.yml

@@ -0,0 +1,26 @@
+ version: "3"
+ services:
+  web_service:
+    build:
+      context: ./
+      dockerfile: ./dockerfiles/Dockerfile
+    image: workshop1_web
+    container_name: workshop_web_container1
+    stdin_open: true #  docker attach container_id
+    tty: true
+    ports:
+     - "8000:8000"
+    volumes:
+     - .:/root/workspace/site
+  psql-db:
+    image: 'postgres:14'
+    container_name: psql-db1
+    environment:
+      - PGPASSWORD=123456
+      - POSTGRES_USER=postgres
+      - POSTGRES_PASSWORD=123456
+    ports:
+      - '5432:5432'
+ volumes:
+    db:
+      driver: local

hw3/dockerfiles/Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.10.2-alpine3.15
+# Install required packages
+# For psycopg2
+RUN apk update && \
+apk --no-cache add --virtual build-deps-alpine build-base && \
+apk --no-cache add --virtual postgresql-deps libpq-dev
+# Install requirements
+RUN pip install --upgrade pip
+RUN pip install Django psycopg2==2.9.3 bs4 html5lib requests python-dateutil
+# Create directories
+RUN mkdir -p /root/workspace/src
+COPY ./  /root/workspace/site
+# Switch to project directory
+WORKDIR /root/workspace/site

hw3/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+"""Django's command-line utility for administrative tasks."""
+import os
+import sys
+
+
+def main():
+    """Run administrative tasks."""
+    os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'myworld.settings')
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError as exc:
+        raise ImportError(
+            "Couldn't import Django. Are you sure it's installed and "
+            "available on your PYTHONPATH environment variable? Did you "
+            "forget to activate a virtual environment?"
+        ) from exc
+    execute_from_command_line(sys.argv)
+
+
+if __name__ == '__main__':
+    main()

hw3/members/admin.py

@@ -0,0 +1,16 @@
+from django.contrib import admin
+from .models import Students,Blog
+
+class DjStudentAdmin(admin.ModelAdmin):
+    list_display = ("first_name", "last_name", "address", "roll_number", "mobile", "branch")
+    list_filter = ("branch",)
+
+
+class DjBlogAdmin(admin.ModelAdmin):
+  list_display = ("title", "release_date", "blog_time", "created_date","content","author", "recommended","path")
+  list_filter = ("author",)
+
+
+# Register your models here.
+admin.site.register(Students, DjStudentAdmin)   
+admin.site.register(Blog, DjBlogAdmin)

hw3/members/apps.py

@@ -0,0 +1,108 @@
+from django.apps import AppConfig
+
+
+from django.apps import AppConfig
+
+import psycopg2
+import requests
+import re
+from bs4 import BeautifulSoup, element
+from django.apps import AppConfig
+import psycopg2
+import requests
+import re
+from bs4 import BeautifulSoup, element
+import datetime
+from dateutil.parser import parse
+
+
+db_name = 'member_db'
+db_user = 'postgres'
+db_pass = '123456'
+db_host = 'psql-db1'
+db_port = '5432'
+
+conn = psycopg2.connect(dbname=db_name, user=db_user, password=db_pass, host=db_host, port=db_port)
+
+def add_row_to_blog(title, author, date, time):
+    sql = """INSERT INTO members_blog (title, release_date, blog_time, author, created_date) VALUES (%s, %s::DATE, %s::TIME, %s, NOW())"""
+
+    with conn:
+        with conn.cursor() as curs:
+            time=time.replace('\u202f',"")
+            curs.execute(sql, (title, date, time, author))
+
+def truncate_table():
+    print("Truncating contents all the tables")
+    with conn:
+        with conn.cursor() as curs:
+            curs.execute("TRUNCATE members_blog CASCADE;")
+
+
+def start_extraction(start_date=None, end_date=None, no_of_articles=None, start_id = None):
+  print("Extraction started")
+  url = "https://blog.python.org/"
+
+  data = requests.get(url)
+  page_soup = BeautifulSoup(data.text, 'html.parser')
+
+  if start_date:
+      start_date = parse(start_date)
+  if end_date:
+      end_date = parse(end_date)
+
+  blogs = page_soup.select('div.date-outer')
+  truncate_table()
+  article_count = 0
+  counter = 1
+  for blog in blogs:
+      article_count += 1
+      if start_id and article_count < int(start_id):
+          continue
+      if no_of_articles and counter > int(no_of_articles):
+          continue
+      date = blog.select('.date-header span')[0].get_text()
+
+      converted_date = parse(date)
+
+      if start_date and converted_date < start_date:
+          continue
+      if end_date and converted_date > end_date:
+          continue
+
+      post = blog.select('.post')[0]
+
+      title = ""
+      title_bar = post.select('.post-title')
+      if len(title_bar) > 0:
+          title = title_bar[0].text
+      else:
+          title = post.select('.post-body')[0].contents[0].text
+
+      # getting the author and blog time
+      post_footer = post.select('.post-footer')[0]
+
+      author = post_footer.select('.post-author span')[0].text
+
+      time = post_footer.select('abbr')[0].text
+
+      add_row_to_blog(title, author, date, time)
+
+      print("\nTitle:", title.strip('\n'))
+      print("Date:", date, )
+      print("Time:", time)
+      print("Author:", author)
+
+      # print("Number of blogs read:", count)
+      print(
+          "\n---------------------------------------------------------------------------------------------------------------\n")
+      counter += 1
+
+
+if __name__ == "__main__":
+    start_extraction()
+
+
+class MembersConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'members'

hw3/members/migrations/0001_initial.py

@@ -0,0 +1,22 @@
+# Generated by Django 3.1 on 2022-09-15 16:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Members',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('firstname', models.CharField(max_length=255)),
+                ('lastname', models.CharField(max_length=255)),
+            ],
+        ),
+    ]

hw3/members/migrations/0002_students_delete_members.py

@@ -0,0 +1,27 @@
+# Generated by Django 4.1.1 on 2022-09-15 17:56
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Students',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('first_name', models.CharField(max_length=200)),
+                ('last_name', models.CharField(max_length=200)),
+                ('address', models.CharField(max_length=200)),
+                ('roll_number', models.IntegerField()),
+                ('mobile', models.CharField(max_length=10)),
+            ],
+        ),
+        migrations.DeleteModel(
+            name='Members',
+        ),
+    ]

hw3/members/migrations/0003_blog_students_branch.py

@@ -0,0 +1,29 @@
+# Generated by Django 4.2.1 on 2023-05-23 04:30
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0002_students_delete_members'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Blog',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('title', models.CharField(max_length=500)),
+                ('release_date', models.DateTimeField(verbose_name='Realse Date')),
+                ('blog_time', models.CharField(max_length=50)),
+                ('author', models.CharField(max_length=200)),
+                ('created_date', models.DateTimeField(auto_now_add=True, null=True, verbose_name='Created Date')),
+            ],
+        ),
+        migrations.AddField(
+            model_name='students',
+            name='branch',
+            field=models.CharField(choices=[('BA', 'BA'), ('B.COM', 'B.COM'), ('MBA', 'MBA'), ('CA', 'CA')], max_length=10, null=True),
+        ),
+    ]

hw3/members/migrations/0004_blog_content_alter_students_branch.py

@@ -0,0 +1,25 @@
+# Generated by Django 4.2.1 on 2023-05-23 10:21
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0003_blog_students_branch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='blog',
+            name='content',
+            field=models.CharField(default=None, max_length=2000000),
+            preserve_default=False,
+        ),
+        migrations.AlterField(
+            model_name='students',
+            name='branch',
+            field=models.CharField(choices=[('BA', 'BA'), ('B.COM', 'B.COM'), ('MBA', 'MBA'), ('CA', 'CA')], default=None, max_length=10),
+            preserve_default=False,
+        ),
+    ]

hw3/members/migrations/0005_blog_rcg.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.1 on 2023-05-24 02:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0004_blog_content_alter_students_branch'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='blog',
+            name='rcg',
+            field=models.CharField(max_length=2000000, null=True),
+        ),
+    ]

hw3/members/migrations/0006_remove_blog_rcg_blog_path_blog_recommended_and_more.py

@@ -0,0 +1,42 @@
+# Generated by Django 4.2.1 on 2023-06-09 15:58
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('members', '0005_blog_rcg'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='blog',
+            name='rcg',
+        ),
+        migrations.AddField(
+            model_name='blog',
+            name='path',
+            field=models.CharField(max_length=500, null=True),
+        ),
+        migrations.AddField(
+            model_name='blog',
+            name='recommended',
+            field=models.CharField(max_length=500, null=True),
+        ),
+        migrations.AlterField(
+            model_name='blog',
+            name='content',
+            field=models.CharField(max_length=20000),
+        ),
+        migrations.AlterField(
+            model_name='blog',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+        migrations.AlterField(
+            model_name='students',
+            name='id',
+            field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'),
+        ),
+    ]