Improve: Analytics Data Masking

BLasan · Nov 29, 2023 · 1202f23 · 1202f23
1 parent bb82dad
commit 1202f23
Show file tree

Hide file tree

Showing 8 changed files with 133 additions and 0 deletions.
diff --git a/...mgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/Constants.java b/...mgt.common.analytics/src/main/java/org/wso2/carbon/apimgt/common/analytics/Constants.java
@@ -28,4 +28,13 @@ public class Constants {
     public static final String ANONYMOUS_VALUE = "anonymous";
     public static final String UNKNOWN_VALUE = "UNKNOWN";
     public static final int UNKNOWN_INT_VALUE = -1;
+    public static final String IPV4_PROP_TYPE = "IPV4";
+    public static final String IPV6_PROP_TYPE = "IPV6";
+    public static final String EMAIL_PROP_TYPE = "EMAIL";
+    public static final String USERNAME_PROP_TYPE = "USERNAME";
+
+    public static final String IPV4_MASK_VALUE = "***";
+    public static final String IPV6_MASK_VALUE = "**";
+    public static final String EMAIL_MASK_VALUE = "*****";
+    public static final String USERNAME_MASK_VALUE = "*****";
 }
diff --git a/...c/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/AnalyticsDataProvider.java b/...c/main/java/org/wso2/carbon/apimgt/common/analytics/collectors/AnalyticsDataProvider.java
@@ -73,4 +73,6 @@ public interface AnalyticsDataProvider {
     default Map<String, Object> getProperties() {
         return Collections.EMPTY_MAP;
     }
+
+    Map<String, String> getMaskProperties();
 }
diff --git a/.../org/wso2/carbon/apimgt/common/analytics/collectors/impl/SuccessRequestDataCollector.java b/.../org/wso2/carbon/apimgt/common/analytics/collectors/impl/SuccessRequestDataCollector.java
@@ -32,6 +32,19 @@
 import org.wso2.carbon.apimgt.common.analytics.publishers.dto.Target;
 import org.wso2.carbon.apimgt.common.analytics.publishers.impl.SuccessRequestDataPublisher;
 
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.wso2.carbon.apimgt.common.analytics.Constants.EMAIL_MASK_VALUE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.EMAIL_PROP_TYPE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV4_MASK_VALUE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV4_PROP_TYPE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV6_MASK_VALUE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.IPV6_PROP_TYPE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.USERNAME_MASK_VALUE;
+import static org.wso2.carbon.apimgt.common.analytics.Constants.USERNAME_PROP_TYPE;
+
+
 /**
  * Success request data collector.
  */
@@ -58,6 +71,22 @@ public void collectData() throws AnalyticsException {
 
         Event event = new Event();
         event.setProperties(provider.getProperties());
+
+        // Masking the configured data
+        Map<String, String> maskData = provider.getMaskProperties();
+        Iterator<Map.Entry<String, String>> iterator = maskData.entrySet().iterator();
+        while (iterator.hasNext()) {
+            Map.Entry<String, String> entry = iterator.next();
+            Map<String, Object> props = event.getProperties();
+            if (props != null) {
+                Object value = props.get(entry.getKey());
+                if (value != null) {
+                    String maskStr = maskAnalyticsData(entry.getValue(), value);
+                    props.replace(entry.getKey(), maskStr);
+                }
+            }
+        }
+
         API api = provider.getApi();
         Operation operation = provider.getOperation();
         Target target = provider.getTarget();
@@ -72,12 +101,32 @@ public void collectData() throws AnalyticsException {
         MetaInfo metaInfo = provider.getMetaInfo();
         String userAgent = provider.getUserAgentHeader();
         String userName = provider.getUserName();
+
+        // Mask UserName if configured
+        if (userName != null) {
+            if (maskData.containsKey("api.ut.userName")) {
+                userName = maskAnalyticsData(maskData.get("api.ut.userName"), userName);
+            } else if (maskData.containsKey("api.ut.userId")) {
+                userName = maskAnalyticsData(maskData.get("api.ut.userId"), userName);
+            }
+        }
+
         String userIp = provider.getEndUserIP();
         if (userIp == null) {
             userIp = Constants.UNKNOWN_VALUE;
+        } else {
+            // Mask User IP if configured
+            if (maskData.containsKey("api.analytics.user.ip")) {
+                userIp = maskAnalyticsData(maskData.get("api.analytics.user.ip"), userIp);
+            }
         }
+
         if (userAgent == null) {
             userAgent = Constants.UNKNOWN_VALUE;
+        } else {
+            if (maskData.containsKey("api.analytics.user.agent")) {
+                userAgent = maskAnalyticsData(maskData.get("api.analytics.user.agent"), userAgent);
+            }
         }
 
         event.setApi(api);
@@ -95,4 +144,32 @@ public void collectData() throws AnalyticsException {
         this.processor.publish(event);
     }
 
+    private String maskAnalyticsData(String type, Object value) {
+        if (value instanceof String) {
+            switch (type) {
+                case IPV4_PROP_TYPE:
+                    String[] octets = value.toString().split("\\.");
+
+                    // Sample output: 192.168.***.98
+                    return octets[0] + "." + octets[1] + "." + IPV4_MASK_VALUE + "." + octets[3];
+                case IPV6_PROP_TYPE:
+                    octets = value.toString().split(":");
+
+                    // Sample output: 2001:0db8:85a3:****:****:****:****:7334
+                    return octets[0] + ":" + octets[1] + ":" + octets[2] + ":" + IPV6_MASK_VALUE + ":" + IPV6_MASK_VALUE
+                            + ":" + IPV6_MASK_VALUE + ":" + IPV6_MASK_VALUE + ":" + octets[7];
+                case EMAIL_PROP_TYPE:
+                    String[] email = value.toString().split("@");
+
+                    // Sample output: *****@gmail.com
+                    return EMAIL_MASK_VALUE + "@" + email[1];
+                case USERNAME_PROP_TYPE:
+                    return USERNAME_MASK_VALUE;
+                default:
+                    // Sample output: ********
+                    return USERNAME_MASK_VALUE;
+            }
+        }
+        return null;
+    }
 }
diff --git a/.../java/org/wso2/carbon/apimgt/gateway/handlers/analytics/SynapseAnalyticsDataProvider.java b/.../java/org/wso2/carbon/apimgt/gateway/handlers/analytics/SynapseAnalyticsDataProvider.java
@@ -251,6 +251,13 @@ public MetaInfo getMetaInfo() {
         return metaInfo;
     }
 
+    @Override
+    public Map<String, String> getMaskProperties() {
+        Map<String, String> maskProperties = ServiceReferenceHolder.getInstance().getApiManagerConfigurationService()
+                .getAPIAnalyticsConfiguration().getMaskDataProperties();
+        return maskProperties;
+    }
+
     @Override
     public int getProxyResponseCode() {
 

diff --git a/...o2/carbon/apimgt/gateway/handlers/streaming/websocket/WebSocketAnalyticsDataProvider.java b/...o2/carbon/apimgt/gateway/handlers/streaming/websocket/WebSocketAnalyticsDataProvider.java
@@ -254,6 +254,13 @@ public MetaInfo getMetaInfo() {
         return metaInfo;
     }
 
+    @Override
+    public Map<String, String> getMaskProperties() {
+        Map<String, String> maskProperties = ServiceReferenceHolder.getInstance().getApiManagerConfigurationService()
+                .getAPIAnalyticsConfiguration().getMaskDataProperties();
+        return maskProperties;
+    }
+
     @Override
     public int getProxyResponseCode() {
         if (isSuccessRequest()) {

diff --git a/...imgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerAnalyticsConfiguration.java b/...imgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerAnalyticsConfiguration.java
@@ -57,6 +57,7 @@ public class APIManagerAnalyticsConfiguration {
     private String responseSchemaName;
     private String faultSchemaName;
     private Map<String, String> reporterProperties;
+    private Map<String, String> maskDataProperties;
 
     private APIManagerAnalyticsConfiguration() {
     }
@@ -77,6 +78,7 @@ public void setAPIManagerConfiguration(APIManagerConfiguration config){
             this.responseSchemaName = config.getFirstProperty(APIConstants.API_ANALYTICS_RESPONSE_SCHEMA_NAME);
             this.faultSchemaName = config.getFirstProperty(APIConstants.API_ANALYTICS_FAULT_SCHEMA_NAME);
             this.reporterProperties = config.getAnalyticsProperties();
+            this.maskDataProperties = config.getAnalyticsMaskProperties();
         }
     }
 
@@ -247,4 +249,8 @@ public String getResponseSchemaName() {
     public String getFaultSchemaName() {
         return faultSchemaName;
     }
+
+    public Map<String, String> getMaskDataProperties() {
+        return maskDataProperties;
+    }
 }
diff --git a/...carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java b/...carbon.apimgt.impl/src/main/java/org/wso2/carbon/apimgt/impl/APIManagerConfiguration.java
@@ -127,6 +127,7 @@ public class APIManagerConfiguration {
     private RedisConfig redisConfig = new RedisConfig();
     private Map<String, List<String>> restApiJWTAuthAudiences = new HashMap<>();
     private JSONObject subscriberAttributes = new JSONObject();
+    private static Map<String, String> analyticsMaskProps;
 
     public Map<String, List<String>> getRestApiJWTAuthAudiences() {
         return restApiJWTAuthAudiences;
@@ -357,6 +358,19 @@ private void readChildElements(OMElement serverConfig,
                         analyticsProps.put(name, value);
                     }
                 }
+
+                // Load all the mask properties
+                OMElement maskProperties = element.getFirstChildWithName(new QName("MaskProperties"));
+                Iterator maskPropertiesIterator = maskProperties.getChildrenWithLocalName("Property");
+                Map<String, String> maskProps = new HashMap<>();
+                while (maskPropertiesIterator.hasNext()) {
+                    OMElement propertyElem = (OMElement) maskPropertiesIterator.next();
+                    String name = propertyElem.getAttributeValue(new QName("name"));
+                    String value = propertyElem.getText();
+                    maskProps.put(name, value.toUpperCase());
+                }
+                analyticsMaskProps = maskProps;
+
                 OMElement authTokenElement = element.getFirstChildWithName(new QName("AuthToken"));
                 String resolvedAuthToken = MiscellaneousUtil.resolve(authTokenElement, secretResolver);
                 analyticsProps.put("auth.api.token", resolvedAuthToken);
@@ -2211,6 +2225,8 @@ public GatewayCleanupSkipList getGatewayCleanupSkipList() {
     public static Map<String, String> getAnalyticsProperties() {
         return analyticsProperties;
     }
+
+
 
     public static Map<String, String> getPersistenceProperties() {
         return persistenceProperties;
@@ -2277,4 +2293,8 @@ public HttpClientConfigurationDTO getHttpClientConfiguration() {
     public void setHttpClientConfiguration(HttpClientConfigurationDTO httpClientConfiguration) {
         this.httpClientConfiguration = httpClientConfiguration;
     }
+
+    public static Map<String, String> getAnalyticsMaskProperties() {
+        return analyticsMaskProps;
+    }
 }
diff --git a/...re.feature/src/main/resources/conf_templates/templates/repository/conf/api-manager.xml.j2 b/...re.feature/src/main/resources/conf_templates/templates/repository/conf/api-manager.xml.j2
@@ -304,6 +304,11 @@
             {% endfor %}
         </Properties>
 
+        <MaskProperties>
+            {% for key,value in apim.analytics.mask.items() %}
+            <Property name="{{key}">{{value}}</Property>
+            {% endfor %}
+        </MaskProperties>
     </Analytics>
 
     <!--