Merge pull request #9 from Pactortester/Argo

feat:support Argo 易观方舟

README.md

@@ -12,7 +12,7 @@
 - [x] 诸葛 IO
 - [x] 神策数据
 - [x] GrowingIO 埋点数据
-- [ ] Argo 易观方舟
+- [x] Argo 易观方舟
 - [ ] 友盟
 - [ ] C4J
 - [ ] Mixpanel 
@@ -150,7 +150,6 @@ addons = [
 from garbevents.growingio_events import GetData
 from garbevents.settings import Settings as ST
 
-'mitmdump -p 8889 -s test_growingio_events.py'
 # 埋点上传url 
 ST.url = 'https://wxapi.growingio.com'
 # 报告生成路径 
@@ -163,6 +162,24 @@ addons = [
 
 ```
 
+## Argo 易观方舟 Demo
+
+```python
+from garbevents.argo_events import GetData
+from garbevents.settings import Settings as ST
+
+# 埋点上传url 
+ST.url = 'https://uat.analysys.cn:4089/'
+# 报告生成路径 
+ST.report_path = 'report'
+# 所有事件名称 
+ST.all_events = ['event_name_1', 'event_name_2']
+addons = [
+    GetData()
+]
+
+```
+
 ## 运行截图
 
 ![](images/demo.png)

build/lib/garbevents/argo_events.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+import base64
+import gzip
+import os
+import urllib
+import urllib.parse
+import urllib.error
+from pprint import pprint
+
+import mitmproxy
+from mitmproxy import http
+from mitmproxy import ctx
+import json
+from garbevents.settings import Settings as ST
+
+
+class GetData:
+    """
+    Personal customized version of Argo(易观方舟)
+    A garbevents HTTP request class.
+    """
+    events_list = []
+
+    @staticmethod
+    def chunks(arr, n):
+        """
+
+        :param arr:
+        :param n:
+        :return:
+        """
+        return [arr[i:i + n] for i in range(0, len(arr), n)]
+
+    @staticmethod
+    def gzip_decompress(data):
+        """
+        解密上报数据
+        :param data:
+        :return:
+        """
+        try:
+            return gzip.decompress(data)
+        except AttributeError:
+            from io import StringIO
+
+            buf = StringIO()
+            buf.write(data)
+            fd = gzip.GzipFile(fileobj=buf, mode="r")
+            fd.rewind()
+            value = fd.read()
+            fd.close()
+            return value
+
+    def request(self, flow: mitmproxy.http.HTTPFlow):
+        """
+        代理服务数据分析
+        :param flow:
+        :return:
+        """
+
+        request_data = flow.request
+        self.request_url = request_data.url
+
+        if ST.url in self.request_url:
+
+            ctx.log.info("Get the complete URL after splitting ====>{}".format(self.request_url))
+            api = self.request_url.split('/')[3].split('?')[0]
+            ctx.log.error("Get API address after splitting ====>{}".format(api))
+
+            request_content = flow.request.content.decode('utf8', errors='ignore')
+            ctx.log.info("Get encrypted data after splitting ====>{}".format(request_content))
+
+            gzip_data = urllib.parse.unquote(request_content)
+            data_list = json.loads(gzip_data)
+
+            for result_list in data_list:
+                ctx.log.error("Get JSON string after decrypting data ====>")
+                pprint(result_list)
+
+                try:
+                    event = result_list["xwhat"]
+                    ctx.log.error("Get the event name after decrypting the data ====> {}".format(event))
+                    self.events_list.append(event)
+                except KeyError:
+                    ctx.log.warn("No events！")
+                event_list = list(set(self.events_list))
+
+                if not os.path.exists(ST.report_path):
+                    os.mkdir(ST.report_path)
+                    ctx.log.info(ST.report_path + 'Successfully created！')
+
+                file = open('{}/now_event.txt'.format(ST.report_path), 'w')
+                for line in event_list:
+                    file.write(line + '\n')
+                ctx.log.warn("Current event name collection ====>{}".format(event_list))
+                lost_list = list(set(ST.all_events).difference(set(event_list)))
+                ctx.log.warn("Missing event name collection ====>{}".format(lost_list))
+                file = open('{}/lost_event.txt'.format(ST.report_path), 'w')
+                for line in lost_list:
+                    file.write(line + '\n')

build/lib/garbevents/version.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "2.0.0"
+__version__ = "2.0.1"
 
 import os
 import sys

garbevents.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: garbevents
-Version: 2.0.0
+Version: 2.0.1
 Summary: grabbing buried points
 Home-page: https://github.com/Pactortester/garbevents
 Author: lijiawei
@@ -19,7 +19,8 @@ Description: # garbevents
 
         - [x] 诸葛 IO
         - [x] 神策数据
-        - [X] GrowingIO 埋点数据
+        - [x] GrowingIO 埋点数据
+        - [x] Argo 易观方舟
         - [ ] 友盟
         - [ ] C4J
         - [ ] Mixpanel 
@@ -46,7 +47,7 @@ Description: # garbevents
 
         ## 适用场景
 
-        1. 使用 诸葛IO/神策数据 作为埋点收集工具的 。
+        1. 使用 以上厂商 作为埋点收集工具的 。
         2. 需要回归验证大批量埋点是否丢失的。
         3. 使用本工具 自动抓取解析埋点信息数据校验埋点是否 丢失。
 
@@ -116,7 +117,7 @@ Description: # garbevents
         ]
         ```
 
-        ## 神策数据 Demo
+        ## 神策数据 Demo 1
 
         ```python
         from garbevents.sensors_events import GetData
@@ -129,6 +130,23 @@ Description: # garbevents
         # 所有事件名称 
         ST.all_events = ['event_name_1', 'event_name_2']
 
+        addons = [
+            GetData()
+        ]
+        ```
+        ## 神策数据 Demo 2
+
+        ```python
+        from garbevents.custom_sensors_events import GetData
+        from garbevents.settings import Settings as ST
+
+        # 埋点上传url 
+        ST.url = 'http://sensor.wodidashi.com'
+        # 报告生成路径 
+        ST.report_path = 'report'
+        # 所有事件名称 
+        ST.all_events = ['event_name_1', 'event_name_2']
+
         addons = [
             GetData()
         ]
@@ -140,7 +158,6 @@ Description: # garbevents
         from garbevents.growingio_events import GetData
         from garbevents.settings import Settings as ST
 
-        'mitmdump -p 8889 -s test_growingio_events.py'
         # 埋点上传url 
         ST.url = 'https://wxapi.growingio.com'
         # 报告生成路径 
@@ -153,6 +170,24 @@ Description: # garbevents
 
         ```
 
+        ## Argo 易观方舟 Demo
+
+        ```python
+        from garbevents.argo_events import GetData
+        from garbevents.settings import Settings as ST
+
+        # 埋点上传url 
+        ST.url = 'https://uat.analysys.cn:4089/'
+        # 报告生成路径 
+        ST.report_path = 'report'
+        # 所有事件名称 
+        ST.all_events = ['event_name_1', 'event_name_2']
+        addons = [
+            GetData()
+        ]
+
+        ```
+
         ## 运行截图
 
         ![](images/demo.png)

garbevents.egg-info/SOURCES.txt

@@ -1,6 +1,7 @@
 README.md
 setup.py
 garbevents/__init__.py
+garbevents/argo_events.py
 garbevents/custom_sensors_events.py
 garbevents/events.py
 garbevents/growingio_events.py

garbevents/argo_events.py

@@ -0,0 +1,100 @@
+# -*- coding: utf-8 -*-
+import base64
+import gzip
+import os
+import urllib
+import urllib.parse
+import urllib.error
+from pprint import pprint
+
+import mitmproxy
+from mitmproxy import http
+from mitmproxy import ctx
+import json
+from garbevents.settings import Settings as ST
+
+
+class GetData:
+    """
+    Personal customized version of Argo(易观方舟)
+    A garbevents HTTP request class.
+    """
+    events_list = []
+
+    @staticmethod
+    def chunks(arr, n):
+        """
+
+        :param arr:
+        :param n:
+        :return:
+        """
+        return [arr[i:i + n] for i in range(0, len(arr), n)]
+
+    @staticmethod
+    def gzip_decompress(data):
+        """
+        解密上报数据
+        :param data:
+        :return:
+        """
+        try:
+            return gzip.decompress(data)
+        except AttributeError:
+            from io import StringIO
+
+            buf = StringIO()
+            buf.write(data)
+            fd = gzip.GzipFile(fileobj=buf, mode="r")
+            fd.rewind()
+            value = fd.read()
+            fd.close()
+            return value
+
+    def request(self, flow: mitmproxy.http.HTTPFlow):
+        """
+        代理服务数据分析
+        :param flow:
+        :return:
+        """
+
+        request_data = flow.request
+        self.request_url = request_data.url
+
+        if ST.url in self.request_url:
+
+            ctx.log.info("Get the complete URL after splitting ====>{}".format(self.request_url))
+            api = self.request_url.split('/')[3].split('?')[0]
+            ctx.log.error("Get API address after splitting ====>{}".format(api))
+
+            request_content = flow.request.content.decode('utf8', errors='ignore')
+            ctx.log.info("Get encrypted data after splitting ====>{}".format(request_content))
+
+            gzip_data = urllib.parse.unquote(request_content)
+            data_list = json.loads(gzip_data)
+
+            for result_list in data_list:
+                ctx.log.error("Get JSON string after decrypting data ====>")
+                pprint(result_list)
+
+                try:
+                    event = result_list["xwhat"]
+                    ctx.log.error("Get the event name after decrypting the data ====> {}".format(event))
+                    self.events_list.append(event)
+                except KeyError:
+                    ctx.log.warn("No events！")
+                event_list = list(set(self.events_list))
+
+                if not os.path.exists(ST.report_path):
+                    os.mkdir(ST.report_path)
+                    ctx.log.info(ST.report_path + 'Successfully created！')
+
+                file = open('{}/now_event.txt'.format(ST.report_path), 'w')
+                for line in event_list:
+                    file.write(line + '\n')
+                ctx.log.warn("Current event name collection ====>{}".format(event_list))
+                lost_list = list(set(ST.all_events).difference(set(event_list)))
+                ctx.log.warn("Missing event name collection ====>{}".format(lost_list))
+                file = open('{}/lost_event.txt'.format(ST.report_path), 'w')
+                for line in lost_list:
+                    file.write(line + '\n')

garbevents/version.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-__version__ = "2.0.0"
+__version__ = "2.0.1"
 
 import os
 import sys

setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name="garbevents",
-    version="2.0.0",
+    version="2.0.1",
     keywords=["pip", "garbevents", "get_events", "buried points"],
     description="grabbing buried points",
     long_description=long_description,

tests/test_argo_events.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+
+from garbevents.argo_events import GetData
+
+
+from garbevents.settings import Settings as ST
+
+'mitmdump -p 8889 -s test_argo_events.py'
+ST.url = 'https://uat.analysys.cn:4089/'
+ST.report_path = 'report'
+ST.all_events = ['LX_CONVERSATION_MSG_ITEM_FORWAR_CLICK', 'LX_CONVERSATION_SETTING_TOPS_CLICK']
+
+addons = [
+    GetData()
+]