Add new servant benchmark: mysql-haskell #4550
Conversation
- Nesting the current/default test under `hasql` so we can have multiple other tests under the same servant framework. - Updated Dockerfile (which cannot be moved as per benchmark conventions) to reference the new source dir for `hasql`. - Decided to move `stack.yaml` into subdir as well so each test can pin itself or upgrade independently.
- Lifted most of the initial code from `hasql` benchmark before adopting it to replace `hasql` database libary with `mysql-haskell`. - Added dockerfile and updated benchmark config to include mysql-haskell bench. - Added top level `servant` README that explains the multi benchmark dir structure. - Git ignore `.stack_work` so devs can locally build haskell benchmarks without having to contend with stack artifacts.
Caches dependencies between local development iterations where only source code changes.
|
@jkarni and @winterland1989: would appreciate your feedback greatly |
|
@naushadh Great work on the other PR; saw your delta chart. This looks good, just need you to remove the license. We don't accept licenses for test implementations. |
|
Latest run (not completed) shows stats for https://tfb-status.techempower.com/results/0785634e-9677-41ed-9780-d293a3ed06fc |
|
It's cool though, decided to write a script to crunch stats like the visualization ui: https://gist.github.com/naushadh/f507030b58209ea0abdfccfd99c0eda4 |
$ pipenv run ./toolset/utils/score.py --results ~/Downloads/
results.2019-03-18-14-39-34-182.json --framework servant servant-mysql-haskell
|
|
@nbrady-techempower any idea why |
|
@naushadh Yes. It's because the metadata file that the visualization tool is using doesn't know about |
|
Gotcha; I assume that UI is closed source? I couldn't find a project under the TechEmpower org that might have it. |
|
Yeah it's closed source for now and maybe a little unruly. We've had it on the roadmap to create a new results web site and open source it here but unfortunately, or rather fortunately for paying business, we've had our resources tied up elsewhere. |
|
@nbrady-techempower since I've been adding (will continue) to add more frameworks and tune them, I want to ensure my script is producing scores in the same logic as the Web UI. I would appreciate it if you could give it a review; specifically the
|
|
This looks right to me and I've verified it against some visualized results. One thing I will point out is that we always divide totalRequests by 15 seconds (kind of related issue: #3947) So if I take the last run https://tfb-status.techempower.com/raw/results.2019-04-01-09-56-32-560.json your script will show |
servantbench, but it timed out before getting to the incomingservant-mysql-haskellbench: https://travis-ci.org/naushadh/FrameworkBenchmarks/jobs/505522644servant-mysql-haskellstrings in results.json toservantto get the visualization UI to render results. Attached results has this rename in place for easier visualization.{ "rawData": { "fortune": { "servant": [ { "latencyAvg": "5.24ms", "timeout": 16, "latencyMax": "85.09ms", "latencyStdev": "3.75ms", "totalRequests": 48876, "write": 0, "read": 0, "connect": 0, "startTime": 1552440127, "endTime": 1552440142 }, { "latencyAvg": "10.27ms", "latencyMax": "191.76ms", "latencyStdev": "8.82ms", "totalRequests": 50436, "startTime": 1552440144, "endTime": 1552440159 }, { "latencyAvg": "18.94ms", "timeout": 61, "latencyMax": "182.95ms", "latencyStdev": "10.04ms", "totalRequests": 52140, "write": 0, "read": 0, "connect": 0, "startTime": 1552440161, "endTime": 1552440176 }, { "latencyAvg": "44.11ms", "latencyMax": "1.06s", "latencyStdev": "56.12ms", "totalRequests": 50490, "startTime": 1552440178, "endTime": 1552440193 }, { "latencyAvg": "77.95ms", "latencyMax": "1.33s", "latencyStdev": "58.16ms", "totalRequests": 51034, "startTime": 1552440195, "endTime": 1552440210 }, { "latencyAvg": "160.48ms", "latencyMax": "2.40s", "latencyStdev": "90.36ms", "totalRequests": 48337, "startTime": 1552440212, "endTime": 1552440227 } ] }, "plaintext": { "servant": [ { "latencyAvg": "97.93ms", "latencyMax": "279.39ms", "latencyStdev": "49.03ms", "totalRequests": 316093, "startTime": 1552440262, "endTime": 1552440277 }, { "latencyAvg": "421.17ms", "latencyMax": "2.60s", "latencyStdev": "234.44ms", "totalRequests": 322165, "startTime": 1552440279, "endTime": 1552440294 }, { "latencyAvg": "1.76s", "latencyMax": "4.46s", "latencyStdev": "831.16ms", "totalRequests": 276083, "startTime": 1552440296, "endTime": 1552440311 }, { "latencyAvg": "4.62s", "timeout": 7066, "latencyMax": "8.00s", "latencyStdev": "1.82s", "totalRequests": 190435, "write": 0, "read": 0, "connect": 0, "startTime": 1552440313, "endTime": 1552440329 } ] }, "slocCounts": { "servant": 1099 }, "db": { "servant": [ { "latencyAvg": "4.89ms", "timeout": 16, "latencyMax": "165.57ms", "latencyStdev": "6.74ms", "totalRequests": 59121, "write": 0, "read": 0, "connect": 0, "startTime": 1552439620, "endTime": 1552439635 }, { "latencyAvg": "10.33ms", "latencyMax": "308.02ms", "latencyStdev": "16.12ms", "totalRequests": 60851, "startTime": 1552439637, "endTime": 1552439652 }, { "latencyAvg": "21.06ms", "timeout": 62, "latencyMax": "595.31ms", "latencyStdev": "39.28ms", "totalRequests": 60593, "write": 0, "read": 0, "connect": 0, "startTime": 1552439654, "endTime": 1552439669 }, { "latencyAvg": "52.14ms", "latencyMax": "713.35ms", "latencyStdev": "61.28ms", "totalRequests": 46320, "startTime": 1552439671, "endTime": 1552439686 }, { "latencyAvg": "120.44ms", "timeout": 17, "latencyMax": "2.81s", "latencyStdev": "209.14ms", "totalRequests": 46413, "write": 0, "read": 0, "connect": 0, "startTime": 1552439688, "endTime": 1552439704 }, { "latencyAvg": "350.37ms", "timeout": 10, "latencyMax": "7.94s", "latencyStdev": "826.14ms", "totalRequests": 43139, "write": 0, "read": 0, "connect": 0, "startTime": 1552439706, "endTime": 1552439721 } ] }, "update": { "servant": [ { "latencyAvg": "280.39ms", "timeout": 73, "latencyMax": "8.00s", "latencyStdev": "678.68ms", "totalRequests": 35620, "write": 0, "read": 0, "connect": 0, "startTime": 1552440008, "endTime": 1552440023 }, { "latencyAvg": "615.33ms", "timeout": 111, "latencyMax": "7.97s", "latencyStdev": "690.71ms", "totalRequests": 8997, "write": 0, "read": 0, "connect": 0, "startTime": 1552440025, "endTime": 1552440041 }, { "latencyAvg": "715.44ms", "timeout": 73, "latencyMax": "7.95s", "latencyStdev": "879.47ms", "totalRequests": 5151, "write": 0, "read": 0, "connect": 0, "startTime": 1552440043, "endTime": 1552440058 }, { "latencyAvg": "1.03s", "timeout": 66, "latencyMax": "7.93s", "latencyStdev": "763.32ms", "totalRequests": 3569, "write": 0, "read": 0, "connect": 0, "startTime": 1552440060, "endTime": 1552440075 }, { "latencyAvg": "1.30s", "timeout": 55, "latencyMax": "7.98s", "latencyStdev": "948.41ms", "totalRequests": 2619, "write": 0, "read": 0, "connect": 0, "startTime": 1552440077, "endTime": 1552440092 } ] }, "json": { "servant": [ { "latencyAvg": "1.35ms", "latencyMax": "44.93ms", "latencyStdev": "1.60ms", "totalRequests": 198128, "startTime": 1552439755, "endTime": 1552439770 }, { "latencyAvg": "2.81ms", "timeout": 30, "latencyMax": "65.07ms", "latencyStdev": "4.01ms", "totalRequests": 252932, "write": 0, "read": 0, "connect": 0, "startTime": 1552439772, "endTime": 1552439787 }, { "latencyAvg": "4.44ms", "latencyMax": "42.24ms", "latencyStdev": "4.70ms", "totalRequests": 265228, "startTime": 1552439789, "endTime": 1552439805 }, { "latencyAvg": "7.60ms", "timeout": 127, "latencyMax": "86.22ms", "latencyStdev": "6.10ms", "totalRequests": 260627, "write": 0, "read": 0, "connect": 0, "startTime": 1552439807, "endTime": 1552439822 }, { "latencyAvg": "13.68ms", "latencyMax": "80.96ms", "latencyStdev": "7.53ms", "totalRequests": 271798, "startTime": 1552439824, "endTime": 1552439839 }, { "latencyAvg": "28.46ms", "latencyMax": "356.70ms", "latencyStdev": "16.97ms", "totalRequests": 267016, "startTime": 1552439841, "endTime": 1552439856 } ] }, "commitCounts": { "servant": 0 }, "query": { "servant": [ { "latencyAvg": "158.95ms", "latencyMax": "3.25s", "latencyStdev": "173.31ms", "totalRequests": 53003, "startTime": 1552439890, "endTime": 1552439905 }, { "latencyAvg": "372.43ms", "latencyMax": "1.79s", "latencyStdev": "169.51ms", "totalRequests": 20573, "startTime": 1552439907, "endTime": 1552439922 }, { "latencyAvg": "611.07ms", "timeout": 37, "latencyMax": "7.69s", "latencyStdev": "488.14ms", "totalRequests": 11541, "write": 0, "read": 0, "connect": 0, "startTime": 1552439924, "endTime": 1552439939 }, { "latencyAvg": "994.43ms", "timeout": 92, "latencyMax": "7.90s", "latencyStdev": "1.07s", "totalRequests": 7688, "write": 0, "read": 0, "connect": 0, "startTime": 1552439941, "endTime": 1552439957 }, { "latencyAvg": "749.35ms", "timeout": 103, "latencyMax": "7.99s", "latencyStdev": "945.62ms", "totalRequests": 5639, "write": 0, "read": 0, "connect": 0, "startTime": 1552439959, "endTime": 1552439974 } ] }, "cached_query": {} }, "environmentDescription": "(unspecified, hostname = 5f7376a0680a)", "git": { "commitId": "ffec2c1c25f34deebb4017a438b854cb5852329d", "repositoryUrl": "[email protected]:naushadh/FrameworkBenchmarks.git", "branchName": "servant/mysql-haskell" }, "cachedQueryIntervals": [ 1, 10, 20, 50, 100 ], "uuid": "507e36a7-b6b2-4203-9d83-d31f9e9c76bc", "succeeded": { "fortune": [ "servant" ], "plaintext": [ "servant" ], "db": [ "servant" ], "update": [ "servant" ], "json": [ "servant" ], "query": [ "servant" ], "cached_query": [] }, "failed": { "fortune": [], "plaintext": [], "db": [], "update": [], "json": [], "query": [], "cached_query": [] }, "verify": { "servant": { "fortune": "pass", "plaintext": "pass", "db": "pass", "update": "pass", "json": "pass", "query": "pass" } }, "duration": 15, "frameworks": [ "servant" ], "pipelineConcurrencyLevels": [ 256, 1024, 4096, 16384 ], "completionTime": 1552440340615, "concurrencyLevels": [ 16, 32, 64, 128, 256, 512 ], "startTime": 1552439517671, "queryIntervals": [ 1, 5, 10, 15, 20 ], "completed": { "servant": "20190313012537" }, "name": "(unspecified, datetime = 2019-03-13 01:11:57)" }{ "rawData": { "fortune": { "servant": [ { "latencyAvg": "3.68ms", "timeout": 15, "latencyMax": "33.37ms", "latencyStdev": "2.59ms", "totalRequests": 69771, "write": 0, "read": 0, "connect": 0, "startTime": 1552441115, "endTime": 1552441130 }, { "latencyAvg": "6.70ms", "latencyMax": "55.29ms", "latencyStdev": "3.55ms", "totalRequests": 72269, "startTime": 1552441132, "endTime": 1552441147 }, { "latencyAvg": "14.34ms", "timeout": 47, "latencyMax": "131.48ms", "latencyStdev": "9.86ms", "totalRequests": 71155, "write": 0, "read": 0, "connect": 0, "startTime": 1552441149, "endTime": 1552441164 }, { "latencyAvg": "29.00ms", "latencyMax": "302.46ms", "latencyStdev": "19.78ms", "totalRequests": 70795, "startTime": 1552441166, "endTime": 1552441182 }, { "latencyAvg": "54.84ms", "timeout": 72, "latencyMax": "492.49ms", "latencyStdev": "37.10ms", "totalRequests": 72704, "write": 0, "read": 0, "connect": 0, "startTime": 1552441184, "endTime": 1552441199 }, { "latencyAvg": "115.98ms", "latencyMax": "1.16s", "latencyStdev": "55.56ms", "totalRequests": 67307, "startTime": 1552441201, "endTime": 1552441216 } ] }, "plaintext": { "servant": [ { "latencyAvg": "139.12ms", "latencyMax": "708.06ms", "latencyStdev": "100.49ms", "totalRequests": 333172, "startTime": 1552441250, "endTime": 1552441265 }, { "latencyAvg": "393.27ms", "latencyMax": "2.77s", "latencyStdev": "234.77ms", "totalRequests": 350484, "startTime": 1552441267, "endTime": 1552441283 }, { "latencyAvg": "2.34s", "timeout": 55, "latencyMax": "7.78s", "latencyStdev": "1.55s", "totalRequests": 251841, "write": 0, "read": 0, "connect": 0, "startTime": 1552441285, "endTime": 1552441300 }, { "latencyAvg": "4.57s", "timeout": 1859, "latencyMax": "7.70s", "latencyStdev": "2.05s", "totalRequests": 174522, "write": 0, "read": 0, "connect": 0, "startTime": 1552441302, "endTime": 1552441317 } ] }, "slocCounts": { "servant": 1099 }, "db": { "servant": [ { "latencyAvg": "3.29ms", "timeout": 16, "latencyMax": "66.44ms", "latencyStdev": "2.66ms", "totalRequests": 78119, "write": 0, "read": 0, "connect": 0, "startTime": 1552440609, "endTime": 1552440624 }, { "latencyAvg": "6.35ms", "latencyMax": "216.55ms", "latencyStdev": "5.41ms", "totalRequests": 78942, "startTime": 1552440626, "endTime": 1552440641 }, { "latencyAvg": "12.57ms", "timeout": 64, "latencyMax": "105.59ms", "latencyStdev": "6.02ms", "totalRequests": 78138, "write": 0, "read": 0, "connect": 0, "startTime": 1552440643, "endTime": 1552440658 }, { "latencyAvg": "25.01ms", "latencyMax": "293.58ms", "latencyStdev": "12.44ms", "totalRequests": 78711, "startTime": 1552440660, "endTime": 1552440675 }, { "latencyAvg": "50.96ms", "timeout": 31, "latencyMax": "1.10s", "latencyStdev": "37.50ms", "totalRequests": 78609, "write": 0, "read": 0, "connect": 0, "startTime": 1552440677, "endTime": 1552440692 }, { "latencyAvg": "137.10ms", "latencyMax": "2.31s", "latencyStdev": "200.75ms", "totalRequests": 74615, "startTime": 1552440694, "endTime": 1552440710 } ] }, "update": { "servant": [ { "latencyAvg": "550.82ms", "latencyMax": "1.67s", "latencyStdev": "219.87ms", "totalRequests": 13562, "startTime": 1552440997, "endTime": 1552441012 }, { "latencyAvg": "2.09s", "latencyMax": "4.49s", "latencyStdev": "416.70ms", "totalRequests": 3380, "startTime": 1552441014, "endTime": 1552441030 }, { "latencyAvg": "2.66s", "latencyMax": "6.74s", "latencyStdev": "1.04s", "totalRequests": 2616, "startTime": 1552441032, "endTime": 1552441047 }, { "latencyAvg": "4.22s", "latencyMax": "7.05s", "latencyStdev": "868.28ms", "totalRequests": 1533, "startTime": 1552441049, "endTime": 1552441064 }, { "latencyAvg": "4.70s", "timeout": 6, "latencyMax": "7.95s", "latencyStdev": "1.19s", "totalRequests": 1355, "write": 0, "read": 0, "connect": 0, "startTime": 1552441066, "endTime": 1552441081 } ] }, "json": { "servant": [ { "latencyAvg": "1.30ms", "latencyMax": "24.83ms", "latencyStdev": "1.23ms", "totalRequests": 196889, "startTime": 1552440744, "endTime": 1552440759 }, { "latencyAvg": "2.89ms", "timeout": 28, "latencyMax": "44.37ms", "latencyStdev": "4.01ms", "totalRequests": 249075, "write": 0, "read": 0, "connect": 0, "startTime": 1552440761, "endTime": 1552440776 }, { "latencyAvg": "4.58ms", "latencyMax": "44.69ms", "latencyStdev": "5.61ms", "totalRequests": 303806, "startTime": 1552440778, "endTime": 1552440793 }, { "latencyAvg": "7.37ms", "timeout": 128, "latencyMax": "73.26ms", "latencyStdev": "6.96ms", "totalRequests": 293819, "write": 0, "read": 0, "connect": 0, "startTime": 1552440795, "endTime": 1552440810 }, { "latencyAvg": "12.74ms", "latencyMax": "71.46ms", "latencyStdev": "7.69ms", "totalRequests": 292854, "startTime": 1552440812, "endTime": 1552440828 }, { "latencyAvg": "24.71ms", "latencyMax": "114.99ms", "latencyStdev": "9.59ms", "totalRequests": 295910, "startTime": 1552440830, "endTime": 1552440845 } ] }, "commitCounts": { "servant": 0 }, "query": { "servant": [ { "latencyAvg": "158.65ms", "latencyMax": "2.90s", "latencyStdev": "124.92ms", "totalRequests": 51712, "startTime": 1552440879, "endTime": 1552440894 }, { "latencyAvg": "428.62ms", "latencyMax": "2.84s", "latencyStdev": "126.83ms", "totalRequests": 17592, "startTime": 1552440896, "endTime": 1552440911 }, { "latencyAvg": "695.50ms", "latencyMax": "3.04s", "latencyStdev": "137.90ms", "totalRequests": 10716, "startTime": 1552440913, "endTime": 1552440929 }, { "latencyAvg": "1.02s", "latencyMax": "4.73s", "latencyStdev": "247.18ms", "totalRequests": 7307, "startTime": 1552440931, "endTime": 1552440946 }, { "latencyAvg": "1.30s", "latencyMax": "3.11s", "latencyStdev": "189.31ms", "totalRequests": 5625, "startTime": 1552440948, "endTime": 1552440963 } ] }, "cached_query": {} }, "environmentDescription": "(unspecified, hostname = ec65b9a95fbf)", "git": { "commitId": "ffec2c1c25f34deebb4017a438b854cb5852329d", "repositoryUrl": "[email protected]:naushadh/FrameworkBenchmarks.git", "branchName": "servant/mysql-haskell" }, "cachedQueryIntervals": [ 1, 10, 20, 50, 100 ], "uuid": "c3a79cb8-d252-4e69-b59c-b966389f1164", "succeeded": { "fortune": [ "servant" ], "plaintext": [ "servant" ], "db": [ "servant" ], "update": [ "servant" ], "json": [ "servant" ], "query": [ "servant" ], "cached_query": [] }, "failed": { "fortune": [], "plaintext": [], "db": [], "update": [], "json": [], "query": [], "cached_query": [] }, "verify": { "servant": { "fortune": "pass", "plaintext": "pass", "db": "pass", "update": "pass", "json": "pass", "query": "pass" } }, "duration": 15, "frameworks": [ "servant" ], "pipelineConcurrencyLevels": [ 256, 1024, 4096, 16384 ], "completionTime": 1552441330047, "concurrencyLevels": [ 16, 32, 64, 128, 256, 512 ], "startTime": 1552440493660, "queryIntervals": [ 1, 5, 10, 15, 20 ], "completed": { "servant": "20190313014205" }, "name": "(unspecified, datetime = 2019-03-13 01:28:13)" }