diff --git a/performance/config.yml b/performance/config.yml old mode 100644 new mode 100755 index 1b962cd93..8b14154e7 --- a/performance/config.yml +++ b/performance/config.yml @@ -1,4 +1,6 @@ modules: + local: + sequential: true jmeter: properties: base-uri: http://localhost @@ -15,6 +17,9 @@ modules: local: sequential: true reporting: + - module: console + summary: true + details: true - module: final-stats - module: passfail criteria: diff --git a/performance/test.sh b/performance/test.sh index deb208fdb..d59eae6fd 100755 --- a/performance/test.sh +++ b/performance/test.sh @@ -15,7 +15,7 @@ export BASE_URL="${URL}" docker run --rm -e BASE_URL -v $(pwd):/bzt-configs \ -v $(pwd)/../build/performance-artifacts:/tmp/artifacts \ -v $(pwd)/resources:/tmp/resources \ - blazemeter/taurus:1.10.3 \ + blazemeter/taurus:1.16.35 \ -o modules.jmeter.properties.base-uri="${BASE_URL}" \ -o reporting.2.dump-xml=/tmp/artifacts/stats.xml \ config.yml \ diff --git a/performance/tests/approvedProducts.yml b/performance/tests/approvedProducts.yml old mode 100644 new mode 100755 index b1f0c2734..65b943683 --- a/performance/tests/approvedProducts.yml +++ b/performance/tests/approvedProducts.yml @@ -1,6 +1,8 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: approved-products scenarios: @@ -17,6 +19,8 @@ scenarios: Authorization: Bearer ${access_token} reporting: - - module: passfail - criteria: - Get 1000 Approved Full Supply Products For Health Clinic too slow: p90 of Get1000ApprovedFullSupplyProductsForHealthClinic>3000ms + - module: passfail + criteria: + - 'Get1000ApprovedFullSupplyProductsForHealthClinic too high error rate: failures>1%, continue as failed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic error rate too high"' + - 'Get1000ApprovedFullSupplyProductsForHealthClinic is too slow: p90>5000ms, continue as failed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic response time too high"' + - 'Get1000ApprovedFullSupplyProductsForHealthClinic needs improvement: p90>3000ms and p90<=5000ms, continue as passed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic needs improvement"' diff --git a/performance/tests/approvedProductsForEssentialMedsAndDistrictHospital.yml b/performance/tests/approvedProductsForEssentialMedsAndDistrictHospital.yml old mode 100644 new mode 100755 index 2fd662561..ceffdc9bd --- a/performance/tests/approvedProductsForEssentialMedsAndDistrictHospital.yml +++ b/performance/tests/approvedProductsForEssentialMedsAndDistrictHospital.yml @@ -1,10 +1,9 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} - scenario: full-supply-approved-products - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(long-scenario-duration,60)} - scenario: non-full-supply-approved-products + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} + scenario: approved-products scenarios: full-supply-approved-products: @@ -25,7 +24,8 @@ scenarios: Authorization: Bearer ${access_token} reporting: - - module: passfail - criteria: - Get 1000 Approved Full Supply Products For District Hospital too slow: p90 of Get1000ApprovedFullSupplyProductsForDistrictHospital>2500ms - Get 9000 Approved Non Full Supply Products For District Hospital too slow: p90 of Get9000ApprovedNonFullSupplyProductsForDistrictHospital>25000ms + - module: passfail + criteria: + - 'p90>3000ms, continue as failed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic response time too high"' + - 'p90>1500ms and p90<=3000ms, continue as passed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic needs improvement"' + - 'failures>1%, continue as failed, label=Get1000ApprovedFullSupplyProductsForHealthClinic, title="Get1000ApprovedFullSupplyProductsForHealthClinic error rate too high"' diff --git a/performance/tests/facilities.yml b/performance/tests/facilities.yml old mode 100644 new mode 100755 index 4a1cb395f..bfd9ee3e9 --- a/performance/tests/facilities.yml +++ b/performance/tests/facilities.yml @@ -1,40 +1,64 @@ # /api/facilities/${id}/approvedProducts endpoint is tested in another config files execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facility-create - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-get-minimal-all - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-get-minimal-one-page - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-get-audit-log - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facility-update - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-get-one - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-get-by-boundary-all - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-get-by-boundary-one-page - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-delete-one - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-get-all - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-search-all - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,10s)} + hold-for: ${__P(scenario-duration,10m)} scenario: facilities-search-one-page scenarios: @@ -294,17 +318,52 @@ scenarios: Content-Type: application/json reporting: - - module: passfail - criteria: - Create Facility too slow: p90 of CreateFacility>500ms - Get All Minimal Facilities too slow: p90 of GetAllMinimalFacilities>1500ms - Get First Ten Minimal Facilities too slow: p90 of GetFirstTenMinimalFacilities>500ms - Get Facility Audit Log too slow: p90 of GetFacilityAuditLog>500ms - Update Facility too slow: p90 of UpdateFacility>500ms - Get Facility too slow: p90 of GetFacility>500ms - Get All Facilities By Boundary too slow: p90 of GetAllFacilitiesByBoundary>500ms - Get First ten Facilities By Boundary too slow: p90 of GetFirstTenFacilitiesByBoundary>500ms - Delete Facility too slow: p90 of DeleteFacility>500ms - Get All Facilities too slow: p90 of GetAllFacilities>2000ms - Find All Health Center Facilities too slow: p90 of FindAllHealthCenterFacilities>2000ms - Find First Ten Health Center Facilities too slow: p90 of FindFirstTenHealthCenterFacilities>500ms + - module: passfail + criteria: + - 'p90<=2000ms, continue as passed, label=CreateFacility, title="Create Facility is successful"' + - 'p90>2000ms and p90<=3000ms, continue as passed, label=CreateFacility, title="Create Facility needs improvement"' + - 'p90>3000ms, continue as failed, label=CreateFacility, title="Create Facility is too slow"' + + - 'p90<=1500ms, continue as passed, label=GetAllMinimalFacilities, title="Get All Minimal Facilities is successful"' + - 'p90>1500ms and p90<=2500ms, continue as passed, label=GetAllMinimalFacilities, title="Get All Minimal Facilities needs improvement"' + - 'p90>2500ms, continue as failed, label=GetAllMinimalFacilities, title="Get All Minimal Facilities is too slow"' + + - 'p90<=1500ms, continue as passed, label=GetFirstTenMinimalFacilities, title="Get First Ten Minimal Facilities is successful"' + - 'p90>1500ms and p90<=2500ms, continue as passed, label=GetFirstTenMinimalFacilities, title="Get First Ten Minimal Facilities needs improvement"' + - 'p90>2500ms, continue as failed, label=GetFirstTenMinimalFacilities, title="Get First Ten Minimal Facilities is too slow"' + + - 'p90<=3000ms, continue as passed, label=GetFacilityAuditLog, title="Get Facility Audit Log is successful"' + - 'p90>3000ms and p90<=5000ms, continue as passed, label=GetFacilityAuditLog, title="Get Facility Audit Log needs improvement"' + - 'p90>5000ms, continue as failed, label=GetFacilityAuditLog, title="Get Facility Audit Log is too slow"' + + - 'p90<=2500ms, continue as passed, label=UpdateFacility, title="Update Facility is successful"' + - 'p90>2500ms and p90<=4000ms, continue as passed, label=UpdateFacility, title="Update Facility needs improvement"' + - 'p90>4000ms, continue as failed, label=UpdateFacility, title="Update Facility is too slow"' + + - 'p90<=1500ms, continue as passed, label=GetFacility, title="Get Facility is successful"' + - 'p90>1500ms and p90<=2500ms, continue as passed, label=GetFacility, title="Get Facility needs improvement"' + - 'p90>2500ms, continue as failed, label=GetFacility, title="Get Facility is too slow"' + + - 'p90<=3000ms, continue as passed, label=GetAllFacilitiesByBoundary, title="Get All Facilities By Boundary is successful"' + - 'p90>3000ms and p90<=5000ms, continue as passed, label=GetAllFacilitiesByBoundary, title="Get All Facilities By Boundary needs improvement"' + - 'p90>5000ms, continue as failed, label=GetAllFacilitiesByBoundary, title="Get All Facilities By Boundary is too slow"' + + - 'p90<=3000ms, continue as passed, label=GetFirstTenFacilitiesByBoundary, title="Get First Ten Facilities By Boundary is successful"' + - 'p90>3000ms and p90<=5000ms, continue as passed, label=GetFirstTenFacilitiesByBoundary, title="Get First Ten Facilities By Boundary needs improvement"' + - 'p90>5000ms, continue as failed, label=GetFirstTenFacilitiesByBoundary, title="Get First Ten Facilities By Boundary is too slow"' + + - 'p90<=1500ms, continue as passed, label=DeleteFacility, title="Delete Facility is successful"' + - 'p90>1500ms and p90<=2500ms, continue as passed, label=DeleteFacility, title="Delete Facility needs improvement"' + - 'p90>2500ms, continue as failed, label=DeleteFacility, title="Delete Facility is too slow"' + + - 'p90<=1500ms, continue as passed, label=GetAllFacilities, title="Get All Facilities is successful"' + - 'p90>1500ms and p90<=2500ms, continue as passed, label=GetAllFacilities, title="Get All Facilities needs improvement"' + - 'p90>2500ms, continue as failed, label=GetAllFacilities, title="Get All Facilities is too slow"' + + - 'p90<=1500ms, continue as passed, label=FindAllHealthCenterFacilities, title="Find All Health Center Facilities is successful"' + - 'p90>1500ms and p90<=2500ms, continue as passed, label=FindAllHealthCenterFacilities, title="Find All Health Center Facilities needs improvement"' + - 'p90>2500ms, continue as failed, label=FindAllHealthCenterFacilities, title="Find All Health Center Facilities is too slow"' + + - 'p90<=1500ms, continue as passed, label=FindFirstTenHealthCenterFacilities, title="Find First Ten Health Center Facilities is successful"' + - 'p90>1500ms and p90<=2500ms, continue as passed, label=FindFirstTenHealthCenterFacilities, title="Find First Ten Health Center Facilities needs improvement"' + - 'p90>2500ms, continue as failed, label=FindFirstTenHealthCenterFacilities, title="Find First Ten Health Center Facilities is too slow"' diff --git a/performance/tests/facilityTypeApprovedProducts.yml b/performance/tests/facilityTypeApprovedProducts.yml old mode 100644 new mode 100755 index 82ce68dca..6de6f5b78 --- a/performance/tests/facilityTypeApprovedProducts.yml +++ b/performance/tests/facilityTypeApprovedProducts.yml @@ -1,6 +1,8 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: ftaps-get-first-10 scenarios: @@ -15,6 +17,8 @@ scenarios: Content-Type: application/json reporting: - - module: passfail - criteria: - 90% of get facility type approved products page of ten fail to get in 100ms milliseconds: p90 of GetAFacilityTypeApprovedProductsPageOfTen>500ms, continue as failed + - module: passfail + criteria: + - 'GetAFacilityTypeApprovedProductsPageOfTen too high error rate: failures>1%, continue as failed, label=GetAFacilityTypeApprovedProductsPageOfTen, title="GetAFacilityTypeApprovedProductsPageOfTen error rate too high"' + - 'GetAFacilityTypeApprovedProductsPageOfTen is too slow: p90>2500ms, stop as failed, label=GetAFacilityTypeApprovedProductsPageOfTen, title="GetAFacilityTypeApprovedProductsPageOfTen response time too high"' + - 'GetAFacilityTypeApprovedProductsPageOfTen needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAFacilityTypeApprovedProductsPageOfTen, title="GetAFacilityTypeApprovedProductsPageOfTen needs improvement"' diff --git a/performance/tests/facilityUpdateAsync.yml b/performance/tests/facilityUpdateAsync.yml old mode 100644 new mode 100755 index 3fbeaaf69..5c697608e --- a/performance/tests/facilityUpdateAsync.yml +++ b/performance/tests/facilityUpdateAsync.yml @@ -1,6 +1,8 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: facility-update-async scenarios: @@ -38,6 +40,11 @@ scenarios: body: ${facility} reporting: - - module: passfail - criteria: - Update Facility too slow: p90 of UpdateFacilityAsync>500ms + - module: passfail + criteria: + - 'UpdateFacilityAsync too high error rate: failures>1%, continue as failed, label=UpdateFacilityAsync, title="UpdateFacilityAsync error rate too high"' + - 'UpdateFacilityAsync is too slow: p90>4000ms, stop as failed, label=UpdateFacilityAsync, title="UpdateFacilityAsync response time too high"' + - 'UpdateFacilityAsync needs improvement: p90>2500ms and p90<=4000ms, continue as passed, label=UpdateFacilityAsync, title="UpdateFacilityAsync needs improvement"' + - 'GetGivenFacility too high error rate: failures>1%, continue as failed, label=GetGivenFacility, title="GetGivenFacility error rate too high"' + - 'GetGivenFacility is too slow: p90>2500ms, stop as failed, label=GetGivenFacility, title="GetGivenFacility response time too high"' + - 'GetGivenFacility needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetGivenFacility, title="GetGivenFacility needs improvement"' diff --git a/performance/tests/idealStockAmounts.yml b/performance/tests/idealStockAmounts.yml old mode 100644 new mode 100755 index b7a5d7eef..86b35f424 --- a/performance/tests/idealStockAmounts.yml +++ b/performance/tests/idealStockAmounts.yml @@ -1,12 +1,18 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: download-ideal-stock-amounts-csv - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-ideal-stock-amounts - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: upload-ideal-stock-amounts-csv scenarios: @@ -50,6 +56,12 @@ scenarios: reporting: - module: passfail criteria: - Download Ideal Stock Amounts too slow: p90 of GetAnIdealStockAmountCSV>1100ms - Get Ideal Stock Amounts too slow: p90 of GetIdealStockAmounts>500ms - Upload Ideal Stock Amounts too slow: p90 of UploadIdealStockAmounts>3000ms + - 'GetAnIdealStockAmountCSV too high error rate: failures>1%, continue as failed, label=GetAnIdealStockAmountCSV, title="GetAnIdealStockAmountCSV error rate too high"' + - 'GetAnIdealStockAmountCSV is too slow: p90>2500ms, stop as failed, label=GetAnIdealStockAmountCSV, title="GetAnIdealStockAmountCSV response time too high"' + - 'GetAnIdealStockAmountCSV needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAnIdealStockAmountCSV, title="GetAnIdealStockAmountCSV needs improvement"' + - 'GetIdealStockAmounts too high error rate: failures>1%, continue as failed, label=GetIdealStockAmounts, title="GetIdealStockAmounts error rate too high"' + - 'GetIdealStockAmounts is too slow: p90>2500ms, stop as failed, label=GetIdealStockAmounts, title="GetIdealStockAmounts response time too high"' + - 'GetIdealStockAmounts needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetIdealStockAmounts, title="GetIdealStockAmounts needs improvement"' + - 'UploadIdealStockAmounts too high error rate: failures>1%, continue as failed, label=UploadIdealStockAmounts, title="UploadIdealStockAmounts error rate too high"' + - 'UploadIdealStockAmounts is too slow: p90>4000ms, stop as failed, label=UploadIdealStockAmounts, title="UploadIdealStockAmounts response time too high"' + - 'UploadIdealStockAmounts needs improvement: p90>3000ms and p90<=4000ms, continue as passed, label=UploadIdealStockAmounts, title="UploadIdealStockAmounts needs improvement"' diff --git a/performance/tests/orderables.yml b/performance/tests/orderables.yml old mode 100644 new mode 100755 index 8c8cd041a..636bb1e80 --- a/performance/tests/orderables.yml +++ b/performance/tests/orderables.yml @@ -1,18 +1,28 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: orderables-get-first-10 - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: orderables-get-10-by-id - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: orderables-get-by-program-code-name - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: orderables-get-10000-by-program - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(long-scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(long-scenario-duration,10m)} scenario: orderables-search-10000-by-version-identities scenarios: @@ -92,10 +102,20 @@ scenarios: body: ${versionIdentities} reporting: - - module: passfail - criteria: - Get An Orderable Page Of Ten too slow: p90 of GetAnOrderablePageOfTen>500ms - Get An Orderable By Id Page Of Ten too slow: p90 of GetAnOrderableByIdPageOfTen>500ms - Get An Orderable By Program And Code And Name too slow: p90 of GetAnOrderableByProgramAndCodeAndName>500ms - Get 10000 Orderables By Program too slow: p90 of Get10000OrderablesByProgram>15000ms - Get 10000 Orderables By Version Identities too slow: p90 of Search10000OrderablesByVersionIdentities>15000ms \ No newline at end of file + - module: passfail + criteria: + - 'GetAnOrderablePageOfTen too high error rate: failures>1%, continue as failed, label=GetAnOrderablePageOfTen, title="GetAnOrderablePageOfTen error rate too high"' + - 'GetAnOrderablePageOfTen is too slow: p90>2500ms, stop as failed, label=GetAnOrderablePageOfTen, title="GetAnOrderablePageOfTen response time too high"' + - 'GetAnOrderablePageOfTen needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAnOrderablePageOfTen, title="GetAnOrderablePageOfTen needs improvement"' + - 'GetAnOrderableByIdPageOfTen too high error rate: failures>1%, continue as failed, label=GetAnOrderableByIdPageOfTen, title="GetAnOrderableByIdPageOfTen error rate too high"' + - 'GetAnOrderableByIdPageOfTen is too slow: p90>2500ms, stop as failed, label=GetAnOrderableByIdPageOfTen, title="GetAnOrderableByIdPageOfTen response time too high"' + - 'GetAnOrderableByIdPageOfTen needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAnOrderableByIdPageOfTen, title="GetAnOrderableByIdPageOfTen needs improvement"' + - 'GetAnOrderableByProgramAndCodeAndName too high error rate: failures>1%, continue as failed, label=GetAnOrderableByProgramAndCodeAndName, title="GetAnOrderableByProgramAndCodeAndName error rate too high"' + - 'GetAnOrderableByProgramAndCodeAndName is too slow: p90>2500ms, stop as failed, label=GetAnOrderableByProgramAndCodeAndName, title="GetAnOrderableByProgramAndCodeAndName response time too high"' + - 'GetAnOrderableByProgramAndCodeAndName needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAnOrderableByProgramAndCodeAndName, title="GetAnOrderableByProgramAndCodeAndName needs improvement"' + - 'Get10000OrderablesByProgram too high error rate: failures>1%, continue as failed, label=Get10000OrderablesByProgram, title="Get10000OrderablesByProgram error rate too high"' + - 'Get10000OrderablesByProgram is too slow: p90>15000ms, stop as failed, label=Get10000OrderablesByProgram, title="Get10000OrderablesByProgram response time too high"' + - 'Get10000OrderablesByProgram needs improvement: p90>5000ms and p90<=15000ms, continue as passed, label=Get10000OrderablesByProgram, title="Get10000OrderablesByProgram needs improvement"' + - 'Search10000OrderablesByVersionIdentities too high error rate: failures>1%, continue as failed, label=Search10000OrderablesByVersionIdentities, title="Search10000OrderablesByVersionIdentities error rate too high"' + - 'Search10000OrderablesByVersionIdentities is too slow: p90>15000ms, stop as failed, label=Search10000OrderablesByVersionIdentities, title="Search10000OrderablesByVersionIdentities response time too high"' + - 'Search10000OrderablesByVersionIdentities needs improvement: p90>5000ms and p90<=15000ms, continue as passed, label=Search10000OrderablesByVersionIdentities, title="Search10000OrderablesByVersionIdentities needs improvement"' diff --git a/performance/tests/processingPeriods.yml b/performance/tests/processingPeriods.yml old mode 100644 new mode 100755 index a5f4ffee1..0b2a58303 --- a/performance/tests/processingPeriods.yml +++ b/performance/tests/processingPeriods.yml @@ -1,22 +1,34 @@ execution: -- concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} - scenario: get-processing-period -- concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} - scenario: get-all-processing-periods -- concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} - scenario: processing-periods-search -- concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} - scenario: get-page-of-ten-processing-periods -- concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} - scenario: get-processing-period-duration -- concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} - scenario: get-processing-period-audit-log + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)}z + hold-for: ${__P(scenario-duration,10m)} + scenario: get-processing-period + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} + scenario: get-all-processing-periods + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} + scenario: processing-periods-search + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} + scenario: get-page-of-ten-processing-periods + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} + scenario: get-processing-period-duration + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} + scenario: get-processing-period-audit-log scenarios: get-processing-period: @@ -91,11 +103,23 @@ scenarios: Content-Type: application/json reporting: -- module: passfail - criteria: - Get Processing Period too slow: p90 of GetProcessingPeriod>50ms - Get All Processing Periods too slow: p90 of GetAllProcessingPeriods>50ms - Processing Periods Search too slow: p90 of ProcessingPeriodsSearch>50ms - Get Page of Ten Processing Periods too slow: p90 of GetPageOfTenProcessingPeriods>50ms - Get Processing Period Duration too slow: p90 of GetProcessingPeriodDuration>50ms - Get Processing Period Audit Log too slow: p90 of GetProcessingPeriodAuditLog>50ms \ No newline at end of file + - module: passfail + criteria: + - 'GetProcessingPeriod too high error rate: failures>1%, continue as failed, label=GetProcessingPeriod, title="GetProcessingPeriod error rate too high"' + - 'GetProcessingPeriod is too slow: p90>2500ms, stop as failed, label=GetProcessingPeriod, title="GetProcessingPeriod response time too high"' + - 'GetProcessingPeriod needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetProcessingPeriod, title="GetProcessingPeriod needs improvement"' + - 'GetAllProcessingPeriods too high error rate: failures>1%, continue as failed, label=GetAllProcessingPeriods, title="GetAllProcessingPeriods error rate too high"' + - 'GetAllProcessingPeriods is too slow: p90>2500ms, stop as failed, label=GetAllProcessingPeriods, title="GetAllProcessingPeriods response time too high"' + - 'GetAllProcessingPeriods needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAllProcessingPeriods, title="GetAllProcessingPeriods needs improvement"' + - 'ProcessingPeriodsSearch too high error rate: failures>1%, continue as failed, label=ProcessingPeriodsSearch, title="ProcessingPeriodsSearch error rate too high"' + - 'ProcessingPeriodsSearch is too slow: p90>2500ms, stop as failed, label=ProcessingPeriodsSearch, title="ProcessingPeriodsSearch response time too high"' + - 'ProcessingPeriodsSearch needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=ProcessingPeriodsSearch, title="ProcessingPeriodsSearch needs improvement"' + - 'GetPageOfTenProcessingPeriods too high error rate: failures>1%, continue as failed, label=GetPageOfTenProcessingPeriods, title="GetPageOfTenProcessingPeriods error rate too high"' + - 'GetPageOfTenProcessingPeriods is too slow: p90>2500ms, stop as failed, label=GetPageOfTenProcessingPeriods, title="GetPageOfTenProcessingPeriods response time too high"' + - 'GetPageOfTenProcessingPeriods needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetPageOfTenProcessingPeriods, title="GetPageOfTenProcessingPeriods needs improvement"' + - 'GetProcessingPeriodDuration too high error rate: failures>1%, continue as failed, label=GetProcessingPeriodDuration, title="GetProcessingPeriodDuration error rate too high"' + - 'GetProcessingPeriodDuration is too slow: p90>2500ms, stop as failed, label=GetProcessingPeriodDuration, title="GetProcessingPeriodDuration response time too high"' + - 'GetProcessingPeriodDuration needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetProcessingPeriodDuration, title="GetProcessingPeriodDuration needs improvement"' + - 'GetProcessingPeriodAuditLog too high error rate: failures>1%, continue as failed, label=GetProcessingPeriodAuditLog, title="GetProcessingPeriodAuditLog error rate too high"' + - 'GetProcessingPeriodAuditLog is too slow: p90>2500ms, stop as failed, label=GetProcessingPeriodAuditLog, title="GetProcessingPeriodAuditLog response time too high"' + - 'GetProcessingPeriodAuditLog needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetProcessingPeriodAuditLog, title="GetProcessingPeriodAuditLog needs improvement"' diff --git a/performance/tests/programs.yml b/performance/tests/programs.yml old mode 100644 new mode 100755 index 82de0a6de..8f6dbbfdc --- a/performance/tests/programs.yml +++ b/performance/tests/programs.yml @@ -1,15 +1,23 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: programs-get-one - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: programs-search - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: create-program - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: delete-program scenarios: @@ -70,9 +78,17 @@ scenarios: Authorization: Bearer ${access_token} reporting: - - module: passfail - criteria: - Get Program too slow: p90 of GetProgram>500ms - Get All Programs too slow: p90 of GetAllPrograms>500ms - Create Program too slow: p90 of CreateProgram>500ms - Delete Program too slow: p90 of DeleteProgram>500ms \ No newline at end of file + - module: passfail + criteria: + - 'GetProgram too high error rate: failures>1%, continue as failed, label=GetProgram, title="GetProgram error rate too high"' + - 'GetProgram is too slow: p90>2500ms, stop as failed, label=GetProgram, title="GetProgram response time too high"' + - 'GetProgram needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetProgram, title="GetProgram needs improvement"' + - 'GetAllPrograms too high error rate: failures>1%, continue as failed, label=GetAllPrograms, title="GetAllPrograms error rate too high"' + - 'GetAllPrograms is too slow: p90>2500ms, stop as failed, label=GetAllPrograms, title="GetAllPrograms response time too high"' + - 'GetAllPrograms needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAllPrograms, title="GetAllPrograms needs improvement"' + - 'CreateProgram too high error rate: failures>1%, continue as failed, label=CreateProgram, title="CreateProgram error rate too high"' + - 'CreateProgram is too slow: p90>3000ms, stop as failed, label=CreateProgram, title="CreateProgram response time too high"' + - 'CreateProgram needs improvement: p90>2000ms and p90<=3000ms, continue as passed, label=CreateProgram, title="CreateProgram needs improvement"' + - 'DeleteProgram too high error rate: failures>1%, continue as failed, label=DeleteProgram, title="DeleteProgram error rate too high"' + - 'DeleteProgram is too slow: p90>2500ms, stop as failed, label=DeleteProgram, title="DeleteProgram response time too high"' + - 'DeleteProgram needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=DeleteProgram, title="DeleteProgram needs improvement"' diff --git a/performance/tests/serviceAccounts.yml b/performance/tests/serviceAccounts.yml old mode 100644 new mode 100755 index c36112cdb..9da5206af --- a/performance/tests/serviceAccounts.yml +++ b/performance/tests/serviceAccounts.yml @@ -1,15 +1,23 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: service-account-create - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: service-account-get - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: service-account-update - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: service-account-delete scenarios: @@ -76,9 +84,17 @@ scenarios: Authorization: Bearer ${access_token} reporting: - - module: passfail - criteria: - Create Service Account too slow: p90 of CreateServiceAccount>500ms - Get Service Account too slow: p90 of GetServiceAccount>500ms - Update Service Account too slow: p90 of UpdateServiceAccount>500ms - Delete Service Account too slow: p90 of DeleteServiceAccount>500ms + - module: passfail + criteria: + - 'CreateServiceAccount too high error rate: failures>1%, continue as failed, label=CreateServiceAccount, title="CreateServiceAccount error rate too high"' + - 'CreateServiceAccount is too slow: p90>3000ms, stop as failed, label=CreateServiceAccount, title="CreateServiceAccount response time too high"' + - 'CreateServiceAccount needs improvement: p90>2000ms and p90<=3000ms, continue as passed, label=CreateServiceAccount, title="CreateServiceAccount needs improvement"' + - 'GetServiceAccount too high error rate: failures>1%, continue as failed, label=GetServiceAccount, title="GetServiceAccount error rate too high"' + - 'GetServiceAccount is too slow: p90>2500ms, stop as failed, label=GetServiceAccount, title="GetServiceAccount response time too high"' + - 'GetServiceAccount needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetServiceAccount, title="GetServiceAccount needs improvement"' + - 'UpdateServiceAccount too high error rate: failures>1%, continue as failed, label=UpdateServiceAccount, title="UpdateServiceAccount error rate too high"' + - 'UpdateServiceAccount is too slow: p90>4000ms, stop as failed, label=UpdateServiceAccount, title="UpdateServiceAccount response time too high"' + - 'UpdateServiceAccount needs improvement: p90>2500ms and p90<=4000ms, continue as passed, label=UpdateServiceAccount, title="UpdateServiceAccount needs improvement"' + - 'DeleteServiceAccount too high error rate: failures>1%, continue as failed, label=DeleteServiceAccount, title="DeleteServiceAccount error rate too high"' + - 'DeleteServiceAccount is too slow: p90>2500ms, stop as failed, label=DeleteServiceAccount, title="DeleteServiceAccount response time too high"' + - 'DeleteServiceAccount needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=DeleteServiceAccount, title="DeleteServiceAccount needs improvement"' diff --git a/performance/tests/supervisoryNodes.yml b/performance/tests/supervisoryNodes.yml old mode 100644 new mode 100755 index 8c5efd2c6..2f2d0c94f --- a/performance/tests/supervisoryNodes.yml +++ b/performance/tests/supervisoryNodes.yml @@ -1,12 +1,13 @@ execution: -# - concurrency: ${__P(scenario-concurrency,1)} -# hold-for: ${__P(scenario-duration,60)} -# scenario: get-supervisory-node - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-all-supervisory-nodes - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-page-of-ten-supervisory-nodes scenarios: @@ -27,7 +28,7 @@ scenarios: - include-scenario: get-user-token - url: ${__P(base-uri)}/api/supervisoryNodes method: GET - label: GetAllSupplyLines + label: GetAllSupervisoryNodes headers: Authorization: Bearer ${access_token} @@ -43,6 +44,9 @@ scenarios: reporting: - module: passfail criteria: -# Get Supervisory Node too slow: p90 of GetSupervisoryNode>15000ms - Get All Supervisory Nodes too slow: p90 of GetAllSupervisoryNodes>60000ms - Get Page of Ten Supervisory Nodes too slow: p90 of GetPageOfTenSupervisoryNodes>60000ms \ No newline at end of file + - 'GetAllSupervisoryNodes too high error rate: failures>1%, continue as failed, label=GetAllSupervisoryNodes, title="GetAllSupervisoryNodes error rate too high"' + - 'GetAllSupervisoryNodes is too slow: p90>2500ms, stop as failed, label=GetAllSupervisoryNodes, title="GetAllSupervisoryNodes response time too high"' + - 'GetAllSupervisoryNodes needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAllSupervisoryNodes, title="GetAllSupervisoryNodes needs improvement"' + - 'GetPageOfTenSupervisoryNodes too high error rate: failures>1%, continue as failed, label=GetPageOfTenSupervisoryNodes, title="GetPageOfTenSupervisoryNodes error rate too high"' + - 'GetPageOfTenSupervisoryNodes is too slow: p90>2500ms, stop as failed, label=GetPageOfTenSupervisoryNodes, title="GetPageOfTenSupervisoryNodes response time too high"' + - 'GetPageOfTenSupervisoryNodes needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetPageOfTenSupervisoryNodes, title="GetPageOfTenSupervisoryNodes needs improvement"' diff --git a/performance/tests/supplyLines.yml b/performance/tests/supplyLines.yml old mode 100644 new mode 100755 index 93e3ea22b..b0c2b5927 --- a/performance/tests/supplyLines.yml +++ b/performance/tests/supplyLines.yml @@ -1,24 +1,38 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-supply-line -# - concurrency: ${__P(scenario-concurrency,1)} -# hold-for: ${__P(scenario-duration,60)} -# scenario: get-all-supply-lines - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-all-supply-lines-expand - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + # - concurrency: ${__P(scenario-concurrency,10)} + # iterations: ${__P(scenario-iterations,10)} + # ramp-up: ${__P(scenario-ramp-up,1m)} + # hold-for: ${__P(scenario-duration,10m)} + # scenario: get-all-supply-lines + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-page-of-ten-supply-lines - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-page-of-ten-supply-lines-expand - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-supply-lines-by-supervisory-node-id - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: get-supply-lines-by-program-id-and-supplying-facility-id scenarios: @@ -99,10 +113,21 @@ scenarios: reporting: - module: passfail criteria: - Get Supply Line too slow: p90 of GetSupplyLine>60000ms -# Get All Supply Lines too slow: p90 of GetAll1000SupplyLines>1000ms - Get All Supply Lines with expand too slow: p90 of GetAll1000SupplyLinesExpand>2000ms - Get Page of Ten Supply Lines too slow: p90 of GetPageOfTenSupplyLines>1000ms - Get Page of Ten Supply Lines with expand too slow: p90 of GetPageOfTenSupplyLinesExpand>1000ms - Get Supply Lines by Supervisory Node too slow: p90 of GetSupplyLinesBySupervisoryNodeId>1000ms - Get Supply Lines by Program and Supplying Facility too slow: p90 of GetSupplyLinesByProgramIdAndSupplyingFacilityId>1000ms + - 'GetSupplyLine too high error rate: failures>1%, continue as failed, label=GetSupplyLine, title="GetSupplyLine error rate too high"' + - 'GetSupplyLine is too slow: p90>2500ms, stop as failed, label=GetSupplyLine, title="GetSupplyLine response time too high"' + - 'GetSupplyLine needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetSupplyLine, title="GetSupplyLine needs improvement"' + - 'GetAll1000SupplyLinesExpand too high error rate: failures>1%, continue as failed, label=GetAll1000SupplyLinesExpand, title="GetAll1000SupplyLinesExpand error rate too high"' + - 'GetAll1000SupplyLinesExpand is too slow: p90>2500ms, stop as failed, label=GetAll1000SupplyLinesExpand, title="GetAll1000SupplyLinesExpand response time too high"' + - 'GetAll1000SupplyLinesExpand needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAll1000SupplyLinesExpand, title="GetAll1000SupplyLinesExpand needs improvement"' + - 'GetPageOfTenSupplyLines too high error rate: failures>1%, continue as failed, label=GetPageOfTenSupplyLines, title="GetPageOfTenSupplyLines error rate too high"' + - 'GetPageOfTenSupplyLines is too slow: p90>2500ms, stop as failed, label=GetPageOfTenSupplyLines, title="GetPageOfTenSupplyLines response time too high"' + - 'GetPageOfTenSupplyLines needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetPageOfTenSupplyLines, title="GetPageOfTenSupplyLines needs improvement"' + - 'GetPageOfTenSupplyLinesExpand too high error rate: failures>1%, continue as failed, label=GetPageOfTenSupplyLinesExpand, title="GetPageOfTenSupplyLinesExpand error rate too high"' + - 'GetPageOfTenSupplyLinesExpand is too slow: p90>2500ms, stop as failed, label=GetPageOfTenSupplyLinesExpand, title="GetPageOfTenSupplyLinesExpand response time too high"' + - 'GetPageOfTenSupplyLinesExpand needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetPageOfTenSupplyLinesExpand, title="GetPageOfTenSupplyLinesExpand needs improvement"' + - 'GetSupplyLinesBySupervisoryNodeId too high error rate: failures>1%, continue as failed, label=GetSupplyLinesBySupervisoryNodeId, title="GetSupplyLinesBySupervisoryNodeId error rate too high"' + - 'GetSupplyLinesBySupervisoryNodeId is too slow: p90>2500ms, stop as failed, label=GetSupplyLinesBySupervisoryNodeId, title="GetSupplyLinesBySupervisoryNodeId response time too high"' + - 'GetSupplyLinesBySupervisoryNodeId needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetSupplyLinesBySupervisoryNodeId, title="GetSupplyLinesBySupervisoryNodeId needs improvement"' + - 'GetSupplyLinesByProgramIdAndSupplyingFacilityId too high error rate: failures>1%, continue as failed, label=GetSupplyLinesByProgramIdAndSupplyingFacilityId, title="GetSupplyLinesByProgramIdAndSupplyingFacilityId error rate too high"' + - 'GetSupplyLinesByProgramIdAndSupplyingFacilityId is too slow: p90>2500ms, stop as failed, label=GetSupplyLinesByProgramIdAndSupplyingFacilityId, title="GetSupplyLinesByProgramIdAndSupplyingFacilityId response time too high"' + - 'GetSupplyLinesByProgramIdAndSupplyingFacilityId needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetSupplyLinesByProgramIdAndSupplyingFacilityId, title="GetSupplyLinesByProgramIdAndSupplyingFacilityId needs improvement"' diff --git a/performance/tests/users.yml b/performance/tests/users.yml old mode 100644 new mode 100755 index 1d7e0f782..05c7f0dec --- a/performance/tests/users.yml +++ b/performance/tests/users.yml @@ -1,42 +1,68 @@ execution: - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-get-one - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-get-all - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-search-one-page - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-has-right - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-get-one-permission-strings - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-get-one-programs - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-get-one-facilities - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: create-user - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: delete-user - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-get-one-supported-programs - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-get-one-fulfillment-facilities - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-get-one-role-assignments - - concurrency: ${__P(scenario-concurrency,1)} - hold-for: ${__P(scenario-duration,60)} + - concurrency: ${__P(scenario-concurrency,10)} + iterations: ${__P(scenario-iterations,10)} + ramp-up: ${__P(scenario-ramp-up,1m)} + hold-for: ${__P(scenario-duration,10m)} scenario: users-search-by-right scenarios: @@ -95,13 +121,13 @@ scenarios: extract-jsonpath: user_id: $.id delete-user: - requests: - - include-scenario: create-user - - url: ${__P(base-uri)}/api/users/${user_id} - method: DELETE - label: DeleteUser - headers: - Authorization: Bearer ${access_token} + requests: + - include-scenario: create-user + - url: ${__P(base-uri)}/api/users/${user_id} + method: DELETE + label: DeleteUser + headers: + Authorization: Bearer ${access_token} users-has-right: variables: # administrator @@ -207,18 +233,29 @@ scenarios: Authorization: Bearer ${access_token} reporting: - - module: passfail - criteria: - Get User too slow: p90 of GetUser>500ms - Get All Users too slow: p90 of GetAllUsers>500ms - Get a User Page of Ten too slow: p90 of GetAUserPageOfTen>500ms - User Has Right too slow: p90 of UserHasRight>500ms - Get User Permission Strings too slow: p90 of GetUserPermissionStrings>500ms - Get User Programs too slow: p90 of GetUserPrograms>250ms - Get User Supported Programs too slow: p90 of GetUserSupportedPrograms>700ms - Get User Facilities too slow: p90 of GetUserFacilities>500ms - Get User Fulfillment Facilities too slow: p90 of GetUserFulfillmentFacilities>125000ms - Get User Role Assignments too slow: p90 of GetUserRoleAssignments>20000ms - Create User too slow: p90 of CreateUser>500ms - Delete User too slow: p90 of DeleteUser>500ms - Get Users By Right too slow: p90 of GetUsersByRight>500ms + - module: passfail + criteria: + - 'GetUser too high error rate: failures>1%, continue as failed, label=GetUser, title="GetUser error rate too high"' + - 'GetUser is too slow: p90>2500ms, stop as failed, label=GetUser, title="GetUser response time too high"' + - 'GetUser needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetUser, title="GetUser needs improvement"' + - 'GetAllUsers too high error rate: failures>1%, continue as failed, label=GetAllUsers, title="GetAllUsers error rate too high"' + - 'GetAllUsers is too slow: p90>2500ms, stop as failed, label=GetAllUsers, title="GetAllUsers response time too high"' + - 'GetAllUsers needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAllUsers, title="GetAllUsers needs improvement"' + - 'GetAUserPageOfTen too high error rate: failures>1%, continue as failed, label=GetAUserPageOfTen, title="GetAUserPageOfTen error rate too high"' + - 'GetAUserPageOfTen is too slow: p90>2500ms, stop as failed, label=GetAUserPageOfTen, title="GetAUserPageOfTen response time too high"' + - 'GetAUserPageOfTen needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetAUserPageOfTen, title="GetAUserPageOfTen needs improvement"' + - 'CreateUser too high error rate: failures>1%, continue as failed, label=CreateUser, title="CreateUser error rate too high"' + - 'CreateUser is too slow: p90>2000ms, stop as failed, label=CreateUser, title="CreateUser response time too high"' + - 'CreateUser needs improvement: p90>1500ms and p90<=2000ms, continue as passed, label=CreateUser, title="CreateUser needs improvement"' + - 'DeleteUser too high error rate: failures>1%, continue as failed, label=DeleteUser, title="DeleteUser error rate too high"' + - 'DeleteUser is too slow: p90>2500ms, stop as failed, label=DeleteUser, title="DeleteUser response time too high"' + - 'DeleteUser needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=DeleteUser, title="DeleteUser needs improvement"' + - 'GetUserPrograms too high error rate: failures>1%, continue as failed, label=GetUserPrograms, title="GetUserPrograms error rate too high"' + - 'GetUserPrograms is too slow: p90>1500ms, stop as failed, label=GetUserPrograms, title="GetUserPrograms response time too high"' + - 'GetUserPrograms needs improvement: p90>1000ms and p90<=1500ms, continue as passed, label=GetUserPrograms, title="GetUserPrograms needs improvement"' + - 'GetUserFacilities too high error rate: failures>1%, continue as failed, label=GetUserFacilities, title="GetUserFacilities error rate too high"' + - 'GetUserFacilities is too slow: p90>2500ms, stop as failed, label=GetUserFacilities, title="GetUserFacilities response time too high"' + - 'GetUserFacilities needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetUserFacilities, title="GetUserFacilities needs improvement"' + - 'GetUserRoleAssignments too high error rate: failures>1%, continue as failed, label=GetUserRoleAssignments, title="GetUserRoleAssignments error rate too high"' + - 'GetUserRoleAssignments is too slow: p90>2500ms, stop as failed, label=GetUserRoleAssignments, title="GetUserRoleAssignments response time too high"' + - 'GetUserRoleAssignments needs improvement: p90>1500ms and p90<=2500ms, continue as passed, label=GetUserRoleAssignments, title="GetUserRoleAssignments needs improvement"'