CQI-21: standarization of output for requests and isolation

performance/config.yml

@@ -1,4 +1,6 @@
 modules:
+  local:
+    sequential: true
   jmeter:
     properties:
       base-uri: http://localhost
@@ -15,6 +17,9 @@ modules:
   local:
     sequential: true
 reporting:
+  - module: console
+    summary: true
+    details: true
   - module: final-stats
   - module: passfail
     criteria:

performance/test.sh

@@ -15,7 +15,7 @@ export BASE_URL="${URL}"
 docker run --rm -e BASE_URL -v $(pwd):/bzt-configs \
   -v $(pwd)/../build/performance-artifacts:/tmp/artifacts \
   -v $(pwd)/resources:/tmp/resources \
-  blazemeter/taurus:1.10.3 \
+  blazemeter/taurus:1.16.35 \
   -o modules.jmeter.properties.base-uri="${BASE_URL}" \
   -o reporting.2.dump-xml=/tmp/artifacts/stats.xml \
   config.yml \

performance/tests/approvedProducts.yml

@@ -1,6 +1,8 @@
 execution:
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,1m)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: approved-products
 
 scenarios:
@@ -17,6 +19,8 @@ scenarios:
           Authorization: Bearer ${access_token}
 
 reporting:
-    - module: passfail
-      criteria:
-        Get 1000 Approved Full Supply Products For Health Clinic too slow: p90 of Get1000ApprovedFullSupplyProductsForHealthClinic>3000ms
+  - module: passfail
+    criteria:
+      - 'Get1000ApprovedFullSupplyProductsForHealthClinic too high error rate: failures>1%, continue as failed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic error rate too high"'
+      - 'Get1000ApprovedFullSupplyProductsForHealthClinic is too slow: p90>5000ms, continue as failed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic response time too high"'
+      - 'Get1000ApprovedFullSupplyProductsForHealthClinic needs improvement: p90>3000ms and p90<=5000ms, continue as passed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic needs improvement"'

performance/tests/approvedProductsForEssentialMedsAndDistrictHospital.yml

@@ -1,10 +1,9 @@
 execution:
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
-    scenario: full-supply-approved-products
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(long-scenario-duration,60)}
-    scenario: non-full-supply-approved-products
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,1m)}
+    hold-for: ${__P(scenario-duration,10m)}
+    scenario: approved-products
 
 scenarios:
   full-supply-approved-products:
@@ -25,7 +24,8 @@ scenarios:
           Authorization: Bearer ${access_token}
 
 reporting:
-    - module: passfail
-      criteria:
-        Get 1000 Approved Full Supply Products For District Hospital too slow: p90 of Get1000ApprovedFullSupplyProductsForDistrictHospital>2500ms
-        Get 9000 Approved Non Full Supply Products For District Hospital too slow: p90 of Get9000ApprovedNonFullSupplyProductsForDistrictHospital>25000ms
+  - module: passfail
+    criteria:
+      - 'p90>3000ms, continue as failed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic response time too high"'
+      - 'p90>1500ms and p90<=3000ms, continue as passed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic needs improvement"'
+      - 'failures>1%, continue as failed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic error rate too high"'

performance/tests/facilities.yml

@@ -1,40 +1,64 @@
 # /api/facilities/${id}/approvedProducts endpoint is tested in another config files
 execution:
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facility-create
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-get-minimal-all
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-get-minimal-one-page
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-get-audit-log
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facility-update
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-get-one
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-get-by-boundary-all
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-get-by-boundary-one-page
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-delete-one
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-get-all
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-search-all
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,10s)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facilities-search-one-page
 
 scenarios:
@@ -294,17 +318,52 @@ scenarios:
           Content-Type: application/json
 
 reporting:
-    - module: passfail
-      criteria:
-        Create Facility too slow: p90 of CreateFacility>500ms
-        Get All Minimal Facilities too slow: p90 of GetAllMinimalFacilities>1500ms
-        Get First Ten Minimal Facilities too slow: p90 of GetFirstTenMinimalFacilities>500ms
-        Get Facility Audit Log too slow: p90 of GetFacilityAuditLog>500ms
-        Update Facility too slow: p90 of UpdateFacility>500ms
-        Get Facility too slow: p90 of GetFacility>500ms
-        Get All Facilities By Boundary too slow: p90 of GetAllFacilitiesByBoundary>500ms
-        Get First ten Facilities By Boundary too slow: p90 of GetFirstTenFacilitiesByBoundary>500ms
-        Delete Facility too slow: p90 of DeleteFacility>500ms
-        Get All Facilities too slow: p90 of GetAllFacilities>2000ms
-        Find All Health Center Facilities too slow: p90 of FindAllHealthCenterFacilities>2000ms
-        Find First Ten Health Center Facilities too slow: p90 of FindFirstTenHealthCenterFacilities>500ms
+  - module: passfail
+    criteria:
+      - 'p90<=2000ms, continue as passed, label=CreateFacility, title="Create Facility is successful"'
+      - 'p90>2000ms and p90<=3000ms, continue as passed, label=CreateFacility, title="Create Facility needs improvement"'
+      - 'p90>3000ms, continue as failed, label=CreateFacility, title="Create Facility is too slow"'
+
+      - 'p90<=1500ms, continue as passed, label=GetAllMinimalFacilities, title="Get All Minimal Facilities is successful"'
+      - 'p90>1500ms and p90<=2500ms, continue as passed, label=GetAllMinimalFacilities, title="Get All Minimal Facilities needs improvement"'
+      - 'p90>2500ms, continue as failed, label=GetAllMinimalFacilities, title="Get All Minimal Facilities is too slow"'
+
+      - 'p90<=1500ms, continue as passed, label=GetFirstTenMinimalFacilities, title="Get First Ten Minimal Facilities is successful"'
+      - 'p90>1500ms and p90<=2500ms, continue as passed, label=GetFirstTenMinimalFacilities, title="Get First Ten Minimal Facilities needs improvement"'
+      - 'p90>2500ms, continue as failed, label=GetFirstTenMinimalFacilities, title="Get First Ten Minimal Facilities is too slow"'
+
+      - 'p90<=3000ms, continue as passed, label=GetFacilityAuditLog, title="Get Facility Audit Log is successful"'
+      - 'p90>3000ms and p90<=5000ms, continue as passed, label=GetFacilityAuditLog, title="Get Facility Audit Log needs improvement"'
+      - 'p90>5000ms, continue as failed, label=GetFacilityAuditLog, title="Get Facility Audit Log is too slow"'
+
+      - 'p90<=2500ms, continue as passed, label=UpdateFacility, title="Update Facility is successful"'
+      - 'p90>2500ms and p90<=4000ms, continue as passed, label=UpdateFacility, title="Update Facility needs improvement"'
+      - 'p90>4000ms, continue as failed, label=UpdateFacility, title="Update Facility is too slow"'
+
+      - 'p90<=1500ms, continue as passed, label=GetFacility, title="Get Facility is successful"'
+      - 'p90>1500ms and p90<=2500ms, continue as passed, label=GetFacility, title="Get Facility needs improvement"'
+      - 'p90>2500ms, continue as failed, label=GetFacility, title="Get Facility is too slow"'
+
+      - 'p90<=3000ms, continue as passed, label=GetAllFacilitiesByBoundary, title="Get All Facilities By Boundary is successful"'
+      - 'p90>3000ms and p90<=5000ms, continue as passed, label=GetAllFacilitiesByBoundary, title="Get All Facilities By Boundary needs improvement"'
+      - 'p90>5000ms, continue as failed, label=GetAllFacilitiesByBoundary, title="Get All Facilities By Boundary is too slow"'
+
+      - 'p90<=3000ms, continue as passed, label=GetFirstTenFacilitiesByBoundary, title="Get First Ten Facilities By Boundary is successful"'
+      - 'p90>3000ms and p90<=5000ms, continue as passed, label=GetFirstTenFacilitiesByBoundary, title="Get First Ten Facilities By Boundary needs improvement"'
+      - 'p90>5000ms, continue as failed, label=GetFirstTenFacilitiesByBoundary, title="Get First Ten Facilities By Boundary is too slow"'
+
+      - 'p90<=1500ms, continue as passed, label=DeleteFacility, title="Delete Facility is successful"'
+      - 'p90>1500ms and p90<=2500ms, continue as passed, label=DeleteFacility, title="Delete Facility needs improvement"'
+      - 'p90>2500ms, continue as failed, label=DeleteFacility, title="Delete Facility is too slow"'
+
+      - 'p90<=1500ms, continue as passed, label=GetAllFacilities, title="Get All Facilities is successful"'
+      - 'p90>1500ms and p90<=2500ms, continue as passed, label=GetAllFacilities, title="Get All Facilities needs improvement"'
+      - 'p90>2500ms, continue as failed, label=GetAllFacilities, title="Get All Facilities is too slow"'
+
+      - 'p90<=1500ms, continue as passed, label=FindAllHealthCenterFacilities, title="Find All Health Center Facilities is successful"'
+      - 'p90>1500ms and p90<=2500ms, continue as passed, label=FindAllHealthCenterFacilities, title="Find All Health Center Facilities needs improvement"'
+      - 'p90>2500ms, continue as failed, label=FindAllHealthCenterFacilities, title="Find All Health Center Facilities is too slow"'
+
+      - 'p90<=1500ms, continue as passed, label=FindFirstTenHealthCenterFacilities, title="Find First Ten Health Center Facilities is successful"'
+      - 'p90>1500ms and p90<=2500ms, continue as passed, label=FindFirstTenHealthCenterFacilities, title="Find First Ten Health Center Facilities needs improvement"'
+      - 'p90>2500ms, continue as failed, label=FindFirstTenHealthCenterFacilities, title="Find First Ten Health Center Facilities is too slow"'

performance/tests/facilityTypeApprovedProducts.yml

@@ -1,6 +1,8 @@
 execution:
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,1m)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: ftaps-get-first-10
 
 scenarios:
@@ -15,6 +17,8 @@ scenarios:
           Content-Type: application/json
 
 reporting:
-    - module: passfail
-      criteria:
-        90% of get facility type approved products page of ten fail to get in 100ms milliseconds: p90 of GetAFacilityTypeApprovedProductsPageOfTen>500ms, continue as failed
+  - module: passfail
+    criteria:
+      - 'GetAFacilityTypeApprovedProductsPageOfTen too high error rate: failures>1%, continue as failed, label=GetAFacilityTypeApprovedProductsPageOfTen, title="GetAFacilityTypeApprovedProductsPageOfTen error rate too high"'
+      - 'GetAFacilityTypeApprovedProductsPageOfTen is too slow: p90>2500ms, stop as failed, label=GetAFacilityTypeApprovedProductsPageOfTen, title="GetAFacilityTypeApprovedProductsPageOfTen response time too high"'
+      - 'GetAFacilityTypeApprovedProductsPageOfTen needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAFacilityTypeApprovedProductsPageOfTen, title="GetAFacilityTypeApprovedProductsPageOfTen needs improvement"'

performance/tests/facilityUpdateAsync.yml

@@ -1,6 +1,8 @@
 execution:
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,1m)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: facility-update-async
 
 scenarios:
@@ -38,6 +40,11 @@ scenarios:
         body: ${facility}
 
 reporting:
-    - module: passfail
-      criteria:
-        Update Facility too slow: p90 of UpdateFacilityAsync>500ms
+  - module: passfail
+    criteria:
+      - 'UpdateFacilityAsync too high error rate: failures>1%, continue as failed, label=UpdateFacilityAsync, title="UpdateFacilityAsync error rate too high"'
+      - 'UpdateFacilityAsync is too slow: p90>4000ms, stop as failed, label=UpdateFacilityAsync, title="UpdateFacilityAsync response time too high"'
+      - 'UpdateFacilityAsync needs improvement: p90>2500ms and p90<=4000ms, continue as passed, label=UpdateFacilityAsync, title="UpdateFacilityAsync needs improvement"'
+      - 'GetGivenFacility too high error rate: failures>1%, continue as failed, label=GetGivenFacility, title="GetGivenFacility error rate too high"'
+      - 'GetGivenFacility is too slow: p90>2500ms, stop as failed, label=GetGivenFacility, title="GetGivenFacility response time too high"'
+      - 'GetGivenFacility needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetGivenFacility, title="GetGivenFacility needs improvement"'

performance/tests/idealStockAmounts.yml

@@ -1,12 +1,18 @@
 execution:
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,1m)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: download-ideal-stock-amounts-csv
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,1m)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: get-ideal-stock-amounts
-  - concurrency: ${__P(scenario-concurrency,1)}
-    hold-for: ${__P(scenario-duration,60)}
+  - concurrency: ${__P(scenario-concurrency,10)}
+    iterations: ${__P(scenario-iterations,10)}
+    ramp-up: ${__P(scenario-ramp-up,1m)}
+    hold-for: ${__P(scenario-duration,10m)}
     scenario: upload-ideal-stock-amounts-csv
 
 scenarios:
@@ -50,6 +56,12 @@ scenarios:
 reporting:
   - module: passfail
     criteria:
-      Download Ideal Stock Amounts too slow: p90 of GetAnIdealStockAmountCSV>1100ms
-      Get Ideal Stock Amounts too slow: p90 of GetIdealStockAmounts>500ms
-      Upload Ideal Stock Amounts too slow: p90 of UploadIdealStockAmounts>3000ms
+      - 'GetAnIdealStockAmountCSV too high error rate: failures>1%, continue as failed, label=GetAnIdealStockAmountCSV, title="GetAnIdealStockAmountCSV error rate too high"'
+      - 'GetAnIdealStockAmountCSV is too slow: p90>2500ms, stop as failed, label=GetAnIdealStockAmountCSV, title="GetAnIdealStockAmountCSV response time too high"'
+      - 'GetAnIdealStockAmountCSV needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAnIdealStockAmountCSV, title="GetAnIdealStockAmountCSV needs improvement"'
+      - 'GetIdealStockAmounts too high error rate: failures>1%, continue as failed, label=GetIdealStockAmounts, title="GetIdealStockAmounts error rate too high"'
+      - 'GetIdealStockAmounts is too slow: p90>2500ms, stop as failed, label=GetIdealStockAmounts, title="GetIdealStockAmounts response time too high"'
+      - 'GetIdealStockAmounts needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetIdealStockAmounts, title="GetIdealStockAmounts needs improvement"'
+      - 'UploadIdealStockAmounts too high error rate: failures>1%, continue as failed, label=UploadIdealStockAmounts, title="UploadIdealStockAmounts error rate too high"'
+      - 'UploadIdealStockAmounts is too slow: p90>4000ms, stop as failed, label=UploadIdealStockAmounts, title="UploadIdealStockAmounts response time too high"'
+      - 'UploadIdealStockAmounts needs improvement: p90>3000ms and p90<=4000ms, continue as passed, label=UploadIdealStockAmounts, title="UploadIdealStockAmounts needs improvement"'