Merge pull request #18 from razzkumar/data-transfer-mssql

Add an example data pipeline to transfer data based on MSSQL

examples/app-mssql/Dockerfile

@@ -1,6 +1,4 @@
 FROM laudio/pyodbc:1.0.8
-LABEL author="Vishwa"
-LABEL maintainer="Kabir Baidhya <[email protected]>"
 
 WORKDIR /app
 

examples/data-transfer-mssql/.env.example

@@ -0,0 +1,7 @@
+SOURCE_DB_NAME=tempdb
+SOURCE_DB_USERNAME=SA
+SOURCE_DB_PASSWORD=TestPassword@1
+
+DESTINATION_DB_NAME=tempdb
+DESTINATION_DB_USERNAME=SA
+DESTINATION_DB_PASSWORD=TestPassword@2

examples/data-transfer-mssql/Dockerfile

@@ -0,0 +1,8 @@
+FROM laudio/pyodbc:1.0.8
+
+WORKDIR /app
+
+# Copy code to container
+COPY . .
+
+CMD ["python", "main.py"]

examples/data-transfer-mssql/README.md

@@ -0,0 +1,37 @@
+# MSSQL Data Transfer Example
+
+Example project to demonstrate a data pipeline container built using laudio/pyodbc as a base image - that transfers data from one mssql database to another mssql database.
+
+### Running
+
+Create a `.env` file using the example file.
+
+```bash
+$ cp .env.example .env
+```
+
+Run the example.
+
+```bash
+$ docker-compose up
+```
+
+**Output**
+
+The output of the application should look like this.
+
+```
+app_1                | Create fruits table and populate data in source database.
+app_1                | Create fruits table in destination database.
+app_1                | Extracting fruits data from source database.
+app_1                | Transferring fruits data to destination database.
+app_1                |
+app_1                | 3 rows transferred
+app_1                | 
+app_1                | Display fruits data of destination database.
+app_1                | ID    NAME            QUANTITY  
+app_1                | --------------------------------
+app_1                | 1     Banana          200       
+app_1                | 2     Orange          20        
+app_1                | 3     Apple           350  
+```

examples/data-transfer-mssql/docker-compose.yml

@@ -0,0 +1,41 @@
+version: "3.3"
+
+services:
+  mssql_source:
+    image: "mcr.microsoft.com/mssql/server:latest"
+    container_name: "mssql_source_db"
+    ports:
+      - "1433:1433"
+    environment:
+      ACCEPT_EULA: Y
+      SA_PASSWORD: ${SOURCE_DB_PASSWORD}
+    env_file:
+      - .env
+
+  mssql_destination:
+    image: "mcr.microsoft.com/mssql/server:latest"
+    container_name: "mssql_destination_db"
+    ports:
+      - "1434:1433"
+    environment:
+      ACCEPT_EULA: Y
+      SA_PASSWORD: ${DESTINATION_DB_PASSWORD}
+    env_file:
+      - .env
+
+  app:
+    build: .
+    depends_on:
+      - mssql_source
+      - mssql_destination
+    environment:
+      SOURCE_DB_HOST: mssql_source
+      SOURCE_DB_NAME: ${SOURCE_DB_NAME}
+      SOURCE_DB_USERNAME: ${SOURCE_DB_USERNAME}
+      SOURCE_DB_PASSWORD: ${SOURCE_DB_PASSWORD}
+      DESTINATION_DB_HOST: mssql_destination
+      DESTINATION_DB_NAME: ${DESTINATION_DB_NAME}
+      DESTINATION_DB_USERNAME: ${DESTINATION_DB_USERNAME}
+      DESTINATION_DB_PASSWORD: ${DESTINATION_DB_PASSWORD}
+    env_file:
+      - .env

examples/data-transfer-mssql/main.py

@@ -0,0 +1,103 @@
+import sys
+import os
+import time
+import pyodbc
+
+CONNECTION_STRING = 'DRIVER={{ODBC Driver 17 for SQL Server}};SERVER={server};DATABASE={database};UID={username};PWD={password};'
+
+
+def main():
+    ''' App entrypoint. '''
+    # Wait for mssql database server to fully spawn.
+    time.sleep(5)
+
+    source_db_conn, dest_db_conn = connect_to_databases()
+
+    with source_db_conn, dest_db_conn:
+        source_db_cur = source_db_conn.cursor()
+        dest_db_cur = dest_db_conn.cursor()
+
+        with source_db_cur, dest_db_cur:
+            print('Create fruits table and populate data in source database.')
+            source_db_cur.execute(extract_sql('sql/source_db_setup.sql'))
+            source_db_conn.commit()
+
+            print('Create fruits table in destination database.')
+            dest_db_cur.execute(extract_sql('sql/dest_db_setup.sql'))
+            dest_db_conn.commit()
+
+            transfer_data(source_db_cur, dest_db_cur, dest_db_conn)
+
+            print('Display fruits data of destination database.')
+            display_fruits(dest_db_cur)
+
+
+def connect_to_databases():
+    ''' Extracts databases credentials from the environment and returns their connections. '''
+    source_db_conn = get_connection(
+        os.environ['SOURCE_DB_HOST'],
+        os.environ['SOURCE_DB_NAME'],
+        os.environ['SOURCE_DB_USERNAME'],
+        os.environ['SOURCE_DB_PASSWORD']
+    )
+
+    dest_db_conn = get_connection(
+        os.environ['DESTINATION_DB_HOST'],
+        os.environ['DESTINATION_DB_NAME'],
+        os.environ['DESTINATION_DB_USERNAME'],
+        os.environ['DESTINATION_DB_PASSWORD']
+    )
+
+    return source_db_conn, dest_db_conn
+
+
+def get_connection(db_host, db_name, db_username, db_password):
+    ''' Create database connection and returns connection. '''
+    connection_str = CONNECTION_STRING.format(
+        server=db_host,
+        database=db_name,
+        username=db_username,
+        password=db_password
+    )
+
+    return pyodbc.connect(connection_str, timeout=300)
+
+
+def extract_sql(file: str):
+    ''' Reads an SQL file and returns it's contents. '''
+    with open(file, 'rt') as file:
+        contents = file.read()
+
+    return contents
+
+
+def transfer_data(source_db_cursor, dest_db_cursor, dest_db_conn):
+    ''' Extracts fruits data from source database and stores them in destination database. '''
+    print('Extracting fruits data from source database.')
+    source_db_cursor.execute('SELECT * FROM fruits')
+    rows = source_db_cursor.fetchall()
+
+    print('Transferring fruits data to destination database.')
+    for row in rows:
+        dest_db_cursor.execute('INSERT INTO fruits VALUES (?, ?, ?)', (row.id, row.name, row.quantity))
+
+    print(f'{len(rows)} rows transferred\n')
+    dest_db_conn.commit()
+
+
+def display_fruits(db_cursor):
+    ''' Displays fruits data. '''
+    db_cursor.execute('SELECT * FROM fruits')
+    transferred_data = db_cursor.fetchall()
+    template = '{:<5} {:<15} {:<10}'
+
+    print(template.format('ID', 'NAME', 'QUANTITY'))
+    print('-' * 32)
+
+    for row in transferred_data:
+        print(template.format(row.id, row.name, row.quantity))
+
+
+if __name__ == '__main__':
+    main()
+    sys.exit(0)

examples/data-transfer-mssql/sql/dest_db_setup.sql

@@ -0,0 +1,2 @@
+-- Create table for fruits data (destination database).
+CREATE TABLE fruits (id INT, name VARCHAR(50), quantity INT);

examples/data-transfer-mssql/sql/source_db_setup.sql

@@ -0,0 +1,7 @@
+-- Create table for fruits data (source database).
+CREATE TABLE fruits (id INT, name VARCHAR(50), quantity INT);
+
+-- Insert data into fruits table
+INSERT INTO fruits VALUES (1, 'Banana', 200);
+INSERT INTO fruits VALUES (2, 'Orange', 20);
+INSERT INTO fruits VALUES (3, 'Apple', 350);