retail_market_customer_analysis.py

# -*- coding: utf-8 -*-
"""Retail Market Customer Analysis.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/14I5G3-Fily-Or8KhQ-wxydtsOlEdg1ji

![](https://miro.medium.com/max/700/1*h0zZjCSgcjZh0Bh2T4kMDg.png)

<h1>Introduction</h1>

<h3>Dataset</h3>

*  <span><a href="https://archive.ics.uci.edu/ml/datasets/online+retail">Link to the dataset</a></span>
*   This is a transnational data set which contains all the transactions occurring between 01/12/2010 and 09/12/2011 for a UK-based and registered non-store online retail.

<h3>Aim</h3>

The objective is to segment the customers based on recency, frequency and monetary so that the company is able to filter out the target audience.

<h1>Reading and Understanding Data</h1>
"""

#importing important libraries

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import datetime as dt

import sklearn
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.cluster import AgglomerativeClustering
from scipy.cluster.hierarchy import linkage
from scipy.cluster.hierarchy import dendrogram
from scipy.cluster.hierarchy import cut_tree

#Reading data 

# df = pd.read_csv('OnlineRetail.csv', sep=',', encoding="ISO-8859-1", header=0)
data = pd.read_csv('OnlineRetail.csv', sep=',', encoding="ISO-8859-1", header=0)
data.head()

#shape of df
data.shape

data.info()

data.describe()

"""<h1>Data Cleansing</h1>"""

#percent contribution of missing values

data_null = round(100*(data.isnull().sum())/len(data), 2)
data_null

data['CustomerID'].nunique()

data['CustomerID'].mean()

data['CustomerID'].median()

data['CustomerID'].mode()

#Dropping null values

data = data.dropna()
data.shape

data['CustomerID'] = data['CustomerID'].astype(str)



"""<h1>Data Preparation</h1>

Analysis is going to be based on three factors:


*   R(Recency): last purchased date
*   F(Frequency): Number of transactions
*   Tr(Total Revenue): Total revenue contributed
"""

#Introducing new attribute Total revenue

data['TotalRevenue'] = data['Quantity']*data['UnitPrice']
data_m = data.groupby('CustomerID')['TotalRevenue'].sum()
data_m = data_m.reset_index()
data_m.head()

data['Country'].unique()

#Introducing new attribute frequency

data_f = data.groupby('CustomerID')['InvoiceNo'].count()
data_f = data_f.reset_index();
data_f.columns = ['CustomerID', 'Frequency']
data_f.head()

#Merging the two dataframes
RFTr = pd.merge(data_f, data_m, on='CustomerID', how='inner')
RFTr.head()

#Introducing new attribute recency
#Convert Invoice Date to proper datetime format
data['InvoiceDate'] = pd.to_datetime(data['InvoiceDate'], format='%d-%m-%Y %H:%M')

#maximum date
max_date = max(data['InvoiceDate'])
#Computing difference 
data['Diff'] = max_date-data['InvoiceDate']
data.head()

#Compute last transaction date to get recency

RFTr_r = data.groupby('CustomerID')['Diff'].min()
RFTr_r = RFTr_r.reset_index()
RFTr_r.head()

#Extracting the number of days from difference

RFTr_r['Diff'] = RFTr_r['Diff'].dt.days
RFTr_r.head()

#Merging the dataframes to get final RFTr dataframe
RFTr = pd.merge(RFTr, RFTr_r, on='CustomerID', how='inner')
RFTr.columns = ['CustomerID', 'Frequency', 'TotalRevenue', 'Recency']
RFTr.head()

#Outlier Analysis for TotalRevenue, Frequency and Recency
attributes = ['Frequency','TotalRevenue','Recency']
plt.rcParams['figure.figsize'] = [10,8]
sns.boxplot(data = RFTr[attributes], orient="v", palette="Set2" ,whis=1.5,saturation=1, width=0.7)
plt.title("Outliers Variable Distribution", fontsize = 14, fontweight = 'bold')
plt.ylabel("Range", fontweight = 'bold')
plt.xlabel("Attributes", fontweight = 'bold')

# Removing (statistical) outliers for Total Revenue
Q1 = RFTr.TotalRevenue.quantile(0.05)
Q3 = RFTr.TotalRevenue.quantile(0.95)
IQR = Q3 - Q1
RFTr = RFTr[(RFTr.TotalRevenue >= Q1 - 1.5*IQR) & (RFTr.TotalRevenue <= Q3 + 1.5*IQR)]

# Removing (statistical) outliers for Recency
Q1 = RFTr.Recency.quantile(0.05)
Q3 = RFTr.Recency.quantile(0.95)
IQR = Q3 - Q1
RFTr = RFTr[(RFTr.Recency >= Q1 - 1.5*IQR) & (RFTr.Recency <= Q3 + 1.5*IQR)]

# Removing (statistical) outliers for Frequency
Q1 = RFTr.Frequency.quantile(0.05)
Q3 = RFTr.Frequency.quantile(0.95)
IQR = Q3 - Q1
RFTr = RFTr[(RFTr.Frequency >= Q1 - 1.5*IQR) & (RFTr.Frequency <= Q3 + 1.5*IQR)]

# Outlier Analysis of TotalRevenue Frequency and Recency

attributes = ['Frequency','TotalRevenue','Recency']
plt.rcParams['figure.figsize'] = [10,8]
sns.boxplot(data = RFTr[attributes], orient="v", palette="Set2" ,whis=1.5,saturation=1, width=0.7)
plt.title("Outliers Variable Distribution", fontsize = 14, fontweight = 'bold')
plt.ylabel("Range", fontweight = 'bold')
plt.xlabel("Attributes", fontweight = 'bold')

"""<h1>Data Scaling</h1>
Using Standardisation Scaling

"""

#Rescaling the attributes

RFTr_df = RFTr[['Frequency', 'TotalRevenue', 'Recency']]
sc = StandardScaler()
RFTr_df_scaled = sc.fit_transform(RFTr_df)
RFTr_df_scaled.shape

RFTr_df_scaled = pd.DataFrame(RFTr_df_scaled)
RFTr_df_scaled.columns = ['Frequency', 'TotalRevenue', 'Recency']
RFTr_df_scaled.head()

"""<h1>ML Model</h1>

<h2>K-Means Clustering</h2>

<h3>Elbow Curve</h3>
"""

# Elbow-curve/SSD

ssd = []
range_n_clusters = [2, 3, 4, 5, 6, 7, 8]
for num_clusters in range_n_clusters:
    kmeans = KMeans(n_clusters=num_clusters, max_iter=50)
    kmeans.fit(RFTr_df_scaled)
    
    ssd.append(kmeans.inertia_)
    
# plot the SSDs for each n_clusters
plt.plot(ssd)
plt.title('Elbow Graph')
plt.xlabel('Number of clusters')
plt.ylabel('WCSS')
plt.show()

# Final Model with k=5

kmeans = KMeans(n_clusters=5, max_iter=50)
kmeans.fit(RFTr_df_scaled)

kmeans.labels_

kmeans.cluster_centers_

cluster_data= RFTr_df.copy()
cluster_data['cluster_pred'] = kmeans.fit_predict(RFTr_df_scaled)

cluster_data.head()

cluster_data.loc[cluster_data['cluster_pred']==0]

"""<h1>Agglomerative Clustering</h1>"""

from scipy.cluster.hierarchy import dendrogram, linkage

link = linkage(RFTr_df_scaled, method = 'complete')

fig, ax = plt.subplots(figsize = (10,8))
ax = dendrogram(link)
plt.tight_layout()
plt.show()

from sklearn.cluster import AgglomerativeClustering

ag_cluster = AgglomerativeClustering(n_clusters = 2, affinity = 'euclidean')

ag_cluster.fit(RFTr_df_scaled)

cluster_data= RFTr_df.copy()
cluster_data['cluster_pred'] = ag_cluster.fit_predict(RFTr_df_scaled)

cluster_data.head()

"""**Getting Optimal Number of Clusters**

<h3>Silhouette Analysis</h3>
"""

# Silhouette analysis
range_n_clusters = [2, 3, 4, 5, 6, 7, 8]
sa = []
for num_clusters in range_n_clusters:
    
    # intialise kmeans
    kmeans = KMeans(n_clusters=num_clusters, max_iter=50)
    kmeans.fit(RFTr_df_scaled)

    cluster_labels = kmeans.labels_
    
    # silhouette score
    silhouette_avg = silhouette_score(RFTr_df_scaled, cluster_labels)
    sa.append(silhouette_avg)
    print("For n_clusters={0}, the silhouette score is {1}".format(num_clusters, silhouette_avg))

# plot the Sas for each n_clusters
plt.plot(sa)
plt.title('Silhouette Graph')
plt.xlabel('Number of clusters')
plt.ylabel('Silhouette Average')
plt.show()

# Final model with k=2
kmeans = KMeans(n_clusters=2, max_iter=50)
kmeans.fit(RFTr_df_scaled)

kmeans.labels_

# assign the label
RFTr['Cluster_Id'] = kmeans.labels_
RFTr.head()

# Box plot to visualize Cluster Id vs Total Revenue

sns.boxplot(x='Cluster_Id', y='TotalRevenue', data=RFTr)

# Box plot to visualize Cluster Id vs Frequency

sns.boxplot(x='Cluster_Id', y='Frequency', data=RFTr)

# Box plot to visualize Cluster Id vs Recency

sns.boxplot(x='Cluster_Id', y='Recency', data=RFTr)

RFTr['Cluster_Id'].unique()

"""<h1>Hierarchical Clustering</h1>

**Single Linkage**
"""

# Single linkage: 

mergings = linkage(RFTr_df_scaled, method="single", metric='euclidean')
dendrogram(mergings)
plt.show()

"""**Complete Linkage**"""

# Complete linkage

mergings = linkage(RFTr_df_scaled, method="complete", metric='euclidean')
dendrogram(mergings)
plt.show()

"""**Average Linkage**"""

# Average linkage

mergings = linkage(RFTr_df_scaled, method="average", metric='euclidean')
dendrogram(mergings)
plt.show()

"""Cutting Dendrogram by k"""

# 2 clusters
cluster_labels = cut_tree(mergings, n_clusters=2).reshape(-1, )
cluster_labels

ag_cluster = AgglomerativeClustering(n_clusters = 2, affinity = 'euclidean')
ag_cluster.fit(RFTr_df_scaled)

# Assign cluster labels

RFTr['Cluster_Labels'] = cluster_labels
RFTr.head()

cluster_data= RFTr.copy()
cluster_data['cluster_pred'] = ag_cluster.fit_predict(RFTr_df_scaled)

ag_cluster.labels_

cluster_data

# Plot Cluster Id vs TotalRevenue

sns.boxplot(x='Cluster_Labels', y='TotalRevenue', data=RFTr)

plt.scatter(cluster_data['TotalRevenue'], cluster_data['cluster_pred'], cmap='rainbow')

# Plot Cluster Id vs Frequency

sns.boxplot(x='Cluster_Labels', y='Frequency', data=RFTr)

# Plot Cluster Id vs Recency

sns.boxplot(x='Cluster_Labels', y='Recency', data=RFTr)

"""<h1>Inference</h1>

K-Means Clustering with 2 Cluster Ids
- Customers with Cluster Id 1 are the customers with high amount of transactions as compared to other customers.
- Customers with Cluster Id 1 are frequent buyers.
- Customers with Cluster Id 1 are recent buyers and hence of most importance from business point of view.

Hierarchical Clustering with 2 Cluster Ids
- Customers with Cluster Id 1 are the customers with high amount of transactions as compared to other customers.
- Customers with Cluster Id 1 are frequent buyers.
- Customers with Cluster Id 1 are recent buyers and hence of most importance from business point of view.

**Customers with cluster id 1 are therefore customers of most importance.**
"""