Merge pull request #29 from yankeexe/data-transfer-pg

Generate and transfer 10000 data postgres

examples/data-transfer-pg/Dockerfile

@@ -5,4 +5,6 @@ WORKDIR /app
 # Copy code to container
 COPY . .
 
+RUN pip install -r requirements.txt 
+
 CMD ["python", "main.py"]

examples/data-transfer-pg/main.py

@@ -3,8 +3,13 @@
 import os
 import time
 import pyodbc
+from faker import Faker
+from typing import List, Tuple
+
 
 CONNECTION_STRING = 'DRIVER={{PostgreSQL Unicode}};SERVER={server};DATABASE={database};UID={username};PWD={password};'
+RECORD_COUNT = 10000
+SQL_INSERT_DATA = 'INSERT INTO users (id, name, city) VALUES (?, ?, ?);'
 
 
 def main():
@@ -21,18 +26,20 @@ def main():
 
         with source_db_cur, dest_db_cur:
 
-            print(f'Create fruits table and populate data in source database.')
+            print(f'Create users table and populate data in source database.')
+
             source_db_cur.execute(extract_sql('sql/source_db_setup.sql'))
+            populate_data(RECORD_COUNT, source_db_cur)
             source_db_conn.commit()
 
-            print(f'Create fruits table in destination database.')
+            print(f'Create users table in destination database.')
             dest_db_cur.execute(extract_sql('sql/dest_db_setup.sql'))
             dest_db_conn.commit()
 
             transfer_data(source_db_cur, dest_db_cur, dest_db_conn)
 
-            print(f'Display fruits data in destination database.')
-            display_fruits(dest_db_cur)
+            print(f'Display users data in destination database.')
+            display_users(dest_db_cur)
 
 
 def get_connection(host: str, db_name: str, db_user: str, db_password: str): 
@@ -62,41 +69,52 @@ def connect_to_databases():
             os.environ['DB2_USERNAME'], 
             os.environ['DB2_PASSWORD']
         )
+
     return source_db_conn, dest_db_conn
 
 
+def populate_data(count: int, db_cursor):
+    ''' Generate user data. '''
+    fake = Faker()
+    row = lambda n: (n + 1, repr(fake.name()), repr(fake.city()))
+
+    for i in range(count):
+        db_cursor.execute(SQL_INSERT_DATA, row(i))
+
+
 def extract_sql(file: str):
     ''' Reads an SQL file and returns it's contents.'''
     with open (file, 'rt') as file:
         contents = file.read()  
+
     return contents
 
 
 def transfer_data(source_db_cursor, dest_db_cursor, dest_db_conn):
-    ''' Extracts fruits data from source database and stores them in destination database.'''
-    print(f'Extracting fruits data from source database.')
-    source_db_cursor.execute('SELECT * FROM fruits')
+    ''' Extracts users data from source database and stores them in destination database.'''
+    print(f'Extracting users data from source database.')
+    source_db_cursor.execute('SELECT * FROM users')
     rows = source_db_cursor.fetchall()
 
-    print(f'Transferring fruits data to destination database.')
+    print(f'Transferring users data to destination database.')
     for row in rows:
-        dest_db_cursor.execute('INSERT INTO fruits VALUES (?, ?, ?)', (row.id, row.name, row.quantity))
+        dest_db_cursor.execute(SQL_INSERT_DATA, (row.id, row.name, row.city))
     dest_db_conn.commit()
 
-    print(f"Transferred {len(rows)} rows of fruits data from source database to destination database.")
+    print(f"Transferred {len(rows)} rows of users data from source database to destination database.")
 
 
-def display_fruits(db_cursor):
-    ''' Displays fruits data. '''
-    db_cursor.execute('SELECT * FROM fruits')
+def display_users(db_cursor):
+    ''' Displays users data. '''
+    db_cursor.execute('SELECT * FROM users')
     transfered_data = db_cursor.fetchall()
-    template = '{:<5} {:<15} {:<10}'
+    template = '{:<5} {:<20} {:<10}'
 
-    print(template.format('ID', 'NAME', 'QUANTITY'))
+    print(template.format('ID', 'NAME', 'CITY'))
     print('-' * 32)
 
     for row in transfered_data:
-        print(template.format(row.id, row.name, row.quantity))
+        print(template.format(row.id, row.name, row.city))
 
 
 if __name__ == '__main__':

examples/data-transfer-pg/requirements.txt

@@ -0,0 +1 @@
+faker==2.0.4

examples/data-transfer-pg/sql/dest_db_setup.sql

@@ -1,2 +1,2 @@
--- Create table for fruits data (destination database).
-CREATE TABLE fruits (id INT, name VARCHAR(50), quantity INT);
+-- Create table for users data (destination database).
+CREATE TABLE users (id INT, name VARCHAR(50), city VARCHAR(50));

examples/data-transfer-pg/sql/source_db_setup.sql

@@ -1,7 +1,2 @@
--- Create table for fruits data (source database).
-CREATE TABLE fruits (id INT, name VARCHAR(50), quantity INT);
-
--- Insert data into fruits table
-INSERT INTO fruits VALUES (1, 'Banana', 150);
-INSERT INTO fruits VALUES (2, 'Orange', 64);
-INSERT INTO fruits VALUES (3, 'Apple', 35);
+-- Create table for users data (source database).
+CREATE TABLE users (id INT, name VARCHAR(50), city VARCHAR(50));