Merge pull request #31 from PiwikPRO/PPPSYS-44518-removing-json-int-f…

…ields

PPPSYS-44518 Converted int values like `gt_ms` to strings

.github/workflows/package.yaml

@@ -3,15 +3,15 @@ on:
     pull_request:
 jobs:
     build:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-20.04
         strategy:
             matrix:
                 python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']
         name: Tests (Python ${{ matrix.python-version }})
         steps:
           - uses: actions/checkout@v2
           - name: Set up Python
-            uses: actions/setup-python@v2
+            uses: actions/setup-python@v4
             with:
                 python-version: ${{ matrix.python-version }}
                 architecture: x64

.github/workflows/test.yaml

@@ -3,7 +3,7 @@ on:
     pull_request:
 jobs:
     build:
-        runs-on: ubuntu-latest
+        runs-on: ubuntu-20.04
         strategy:
             matrix:
                 python-version: ['3.6', '3.7', '3.8', '3.9', '3.10']

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## [Unreleased]
 
+## [5.2.1] - 2023-11-09
+
+- PPPSYS-44518 Converted int values like `gt_ms` to strings
+- DEVOPS-5661 Missing timeouts for gha workflow
 
 ## [5.2.0] - 2022-07-11
 

piwik_pro_log_analytics/import_logs.py

@@ -239,7 +239,6 @@ def remove_ignored_groups(self, groups):
 
 
 class W3cExtendedFormat(RegexFormat):
-
     FIELDS_LINE_PREFIX = "#Fields: "
     REGEX_UNKNOWN_FIELD = r'(?:".*?"|\S+)'
 
@@ -336,7 +335,6 @@ def create_regex(self, file):
         self.regex = re.compile(full_regex)
 
     def _configure_expected_fields(self):
-
         expected_fields = type(
             self
         ).fields.copy()  # turn custom field mapping into field => regex mapping
@@ -384,7 +382,6 @@ def _is_time_taken_milli(self):
 
 
 class IisFormat(W3cExtendedFormat):
-
     fields = W3cExtendedFormat.fields.copy()
     fields.update(
         {
@@ -405,7 +402,6 @@ def __init__(self):
 
 
 class IncapsulaW3CFormat(W3cExtendedFormat):
-
     # use custom unknown field regex to make resulting regex much simpler
     REGEX_UNKNOWN_FIELD = r'".*?"'
 
@@ -438,7 +434,6 @@ def get(self, key):
 
 
 class ShoutcastFormat(W3cExtendedFormat):
-
     fields = W3cExtendedFormat.fields.copy()
     fields.update(
         {
@@ -461,7 +456,6 @@ def get(self, key):
 
 
 class AmazonCloudFrontFormat(W3cExtendedFormat):
-
     fields = W3cExtendedFormat.fields.copy()
     fields.update(
         {
@@ -2154,7 +2148,7 @@ def execute(self, args_config, initial_args=None):
         args = initial_args or {}
 
         if args_config.hit.generation_time_milli > 0:
-            args["gt_ms"] = int(args_config.hit.generation_time_milli)
+            args["gt_ms"] = str(int(args_config.hit.generation_time_milli))
 
         if args_config.hit.event_category and args_config.hit.event_action:
             args["e_c"] = args_config.hit.event_category
@@ -2164,7 +2158,7 @@ def execute(self, args_config, initial_args=None):
                 args["e_n"] = args_config.hit.event_name
 
         if args_config.hit.length:
-            args["bw_bytes"] = args_config.hit.length
+            args["bw_bytes"] = str(args_config.hit.length)
 
         # convert custom variable args to JSON
         if "cvar" in args and not isinstance(args["cvar"], str):
@@ -2372,7 +2366,6 @@ def _record_hits(self, hits, single=False):
 
             if hit_count > 0:
                 try:
-
                     response = piwik.call(
                         config.options.piwik_tracker_endpoint_path,
                         args=args,
@@ -2713,7 +2706,6 @@ def filtered_line(line, reason):
             logging.debug("Filtered line out (%s): %s" % (reason, line))
 
     def _get_file_and_filename(self, filename):
-
         if filename == "-":
             return "(stdin)", sys.stdin
         else:
@@ -2741,7 +2733,6 @@ def _get_file_and_filename(self, filename):
 
     # Returns True if format was configured
     def _configure_format(self, file):
-
         if config.format:
             # The format was explicitly specified.
             format = config.format
@@ -2996,7 +2987,7 @@ def parse(self, filename):  # noqa C901
                 if config.options.seconds_to_add_to_date:
                     for param in ["_idts", "_viewts", "_ects", "_refts"]:
                         if param in hit.args:
-                            hit.args[param] = (
+                            hit.args[param] = str(
                                 int(hit.args[param]) + config.options.seconds_to_add_to_date
                             )
 

tests/test_main.py

@@ -119,7 +119,6 @@ def _test_ipv6(self, format_name, log_file=None):
         assert groups["ip"] == "0:0:0:0:0:ffff:7b2d:4359"
 
     def test_format_detection(self):
-
         for format_name in import_logs.FORMATS.keys():
             # w3c extended tested by iis and netscaler log files; amazon cloudfront tested later
             if (
@@ -259,20 +258,20 @@ def test_replay_tracking_seconds_to_add_to_date():
 
     hits = [hit.args for hit in import_logs.Recorder.recorders]
 
-    assert hits[0]["_idts"] == 1360047661 + 3600
-    assert hits[0]["_viewts"] == 1360047661 + 3600
-    assert hits[0]["_refts"] == 1360047661 + 3600
-    assert hits[0]["_ects"] == 1360047634 + 3600
+    assert hits[0]["_idts"] == str(1360047661 + 3600)
+    assert hits[0]["_viewts"] == str(1360047661 + 3600)
+    assert hits[0]["_refts"] == str(1360047661 + 3600)
+    assert hits[0]["_ects"] == str(1360047634 + 3600)
 
-    assert hits[1]["_idts"] == 1360047661 + 3600
-    assert hits[1]["_viewts"] == 1360047661 + 3600
-    assert hits[1]["_refts"] == 1360047661 + 3600
-    assert hits[1]["_ects"] == 1360047534 + 3600
+    assert hits[1]["_idts"] == str(1360047661 + 3600)
+    assert hits[1]["_viewts"] == str(1360047661 + 3600)
+    assert hits[1]["_refts"] == str(1360047661 + 3600)
+    assert hits[1]["_ects"] == str(1360047534 + 3600)
 
-    assert hits[2]["_idts"] == 1360047661 + 3600
-    assert hits[2]["_viewts"] == 1360047661 + 3600
-    assert hits[2]["_refts"] == 1360047661 + 3600
-    assert hits[2]["_ects"] == 1360047614 + 3600
+    assert hits[2]["_idts"] == str(1360047661 + 3600)
+    assert hits[2]["_viewts"] == str(1360047661 + 3600)
+    assert hits[2]["_refts"] == str(1360047661 + 3600)
+    assert hits[2]["_ects"] == str(1360047614 + 3600)
 
 
 def test_replay_tracking_arguments():
@@ -714,7 +713,7 @@ def test_shoutcast_parsing():
     assert hits[0]["is_download"] is False
     assert hits[0]["referrer"] == ""
     assert hits[0]["args"] == {}
-    assert hits[0]["generation_time_milli"] == 1000.0
+    assert hits[0]["generation_time_milli"] == 1000
     assert hits[0]["host"] == "foo"
     assert hits[0]["filename"] == "logs/shoutcast.log"
     assert hits[0]["is_redirect"] is False
@@ -754,7 +753,7 @@ def test_splitted_date_and_time_parsing():
     assert hits[0]["is_download"] is False
     assert hits[0]["referrer"] == ""
     assert hits[0]["args"] == {}
-    assert hits[0]["generation_time_milli"] == 1000.0
+    assert hits[0]["generation_time_milli"] == 1000
     assert hits[0]["host"] == "foo"
     assert hits[0]["filename"] == "logs/splitted_date_and_time.log"
     assert hits[0]["is_redirect"] is False
@@ -1458,7 +1457,6 @@ def test_bz2_parsing():
 
 
 def test_static_resolver_with_idsite():
-
     import_logs.piwik = Piwik()
     import_logs.stats = import_logs.Statistics()
     import_logs.resolver = import_logs.StaticResolver("194edb22-394a-48e5-aed8-0797ab29d2ae")