diff --git a/common/config/rush/pnpm-lock.yaml b/common/config/rush/pnpm-lock.yaml index 017975960..b20efde2a 100644 --- a/common/config/rush/pnpm-lock.yaml +++ b/common/config/rush/pnpm-lock.yaml @@ -302,6 +302,9 @@ dependencies: '@substrate/calc': specifier: ^0.2.8 version: 0.2.8 + '@types/clone': + specifier: ^2.1.4 + version: 2.1.4 '@types/deep-equal': specifier: ^1.0.4 version: 1.0.4 @@ -365,6 +368,9 @@ dependencies: camelcase: specifier: ^6.3.0 version: 6.3.0 + clone: + specifier: ^2.1.2 + version: 2.1.2 commander: specifier: ^11.1.0 version: 11.1.0 @@ -2548,6 +2554,10 @@ packages: '@types/node': 18.19.31 dev: false + /@types/clone@2.1.4: + resolution: {integrity: sha512-NKRWaEGaVGVLnGLB2GazvDaZnyweW9FJLLFL5LhywGJB3aqGMT9R/EUoJoSRP4nzofYnZysuDmrEJtJdAqUOtQ==} + dev: false + /@types/connect@3.4.38: resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} dependencies: @@ -3383,6 +3393,11 @@ packages: mimic-response: 1.0.1 dev: false + /clone@2.1.2: + resolution: {integrity: sha512-3Pe/CF1Nn94hyhIYpjtiLhdCoEoz0DqQ+988E9gmeEdQZlojxnOb74wctFyuwWQHzqyf9X7C7MG8juUpqBJT8w==} + engines: {node: '>=0.8'} + dev: false + /cluster-key-slot@1.1.2: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} @@ -3937,6 +3952,10 @@ packages: pure-rand: 6.1.0 dev: false + /fast-copy@3.0.2: + resolution: {integrity: sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==} + dev: false + /fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} dev: false @@ -5502,6 +5521,10 @@ packages: engines: {node: '>= 4'} dev: false + /rfdc@1.4.1: + resolution: {integrity: sha512-q1b3N5QkRUWUl7iyylaaj3kOpIT0N2i9MqIEQXP73GVsN9cw3fdx8X63cEmWhJGi2PPCF23Ijp7ktmd39rawIA==} + dev: false + /rollup@4.14.2: resolution: {integrity: sha512-WkeoTWvuBoFjFAhsEOHKRoZ3r9GfTyhh7Vff1zwebEFLEFjT1lG3784xEgKiTa7E+e70vsC81roVL2MP4tgEEQ==} engines: {node: '>=18.0.0', npm: '>=8.0.0'} @@ -6532,7 +6555,7 @@ packages: dev: false file:projects/astar-erc20.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-fqUQYw2QOvqPIa3TEHmaQSQIzf/TRnu92b/cwDihlmlfYV+UwTc/bLpCnBTfDbJWlQ5rPJmY7FJYTzK0zGYIXA==, tarball: file:projects/astar-erc20.tgz} + resolution: {integrity: sha512-nM8FzKINL+LRE3dJYPRcmDUYQylhViG8sv0ITeHvNrThiTwoRw3Is01eRofsmCanLrxKP7sedWQK7+LKLph1gQ==, tarball: file:projects/astar-erc20.tgz} id: file:projects/astar-erc20.tgz name: '@rush-temp/astar-erc20' version: 0.0.0 @@ -6566,7 +6589,7 @@ packages: dev: false file:projects/balances.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-SOIUJD5nBTiOJbTTG9/nx3HAQooJusXGXe6BI7vv1DeSc1UclmhexzZV3VYQim5NRST0DwbsNciDxgh6TqQMEQ==, tarball: file:projects/balances.tgz} + resolution: {integrity: sha512-sVXMA5Q+CTTaWSY0dP6Jy4wuAmKFA7YYb6Phg+Giep32Wo4MhuVqkzfHf2awB1pf+G3A6b0Yn/LmeQsyT6OCLQ==, tarball: file:projects/balances.tgz} id: file:projects/balances.tgz name: '@rush-temp/balances' version: 0.0.0 @@ -6617,7 +6640,7 @@ packages: dev: false file:projects/borsh-bench.tgz: - resolution: {integrity: sha512-4thNgEQETqf3DVQzlFeKdaEBDQDU4C+hLDMDCpzCcBwsbfcTGrh4bqxYZjKHKba2Sl0ZVg2MCiBUuOK6Ujo+wA==, tarball: file:projects/borsh-bench.tgz} + resolution: {integrity: sha512-0xnTzo1dLljgeDpBDXNZwF8r3oM/EyDvJVrdGQq4dq2QPCswCKdFCMubp071rmcJzeZJqbZ7e/FH3tb4bFCgBg==, tarball: file:projects/borsh-bench.tgz} name: '@rush-temp/borsh-bench' version: 0.0.0 dependencies: @@ -6655,7 +6678,7 @@ packages: dev: false file:projects/data-test.tgz: - resolution: {integrity: sha512-XOjgnQzgt0Awd6ZhQK3peOHMglZ9SsH+AHiiREw9Hfds8LCeTp4OhlflLe/QHK3JZInGTdKBHtyNBqkYfReq+g==, tarball: file:projects/data-test.tgz} + resolution: {integrity: sha512-DGiTWToHQch/bYReq5oI0mNGC0LM/WuMe8uwkU1a5sADzS+aX68lIypYVvIkRgpJEfcmOfsa1RgL2mata3AbJg==, tarball: file:projects/data-test.tgz} name: '@rush-temp/data-test' version: 0.0.0 dependencies: @@ -6669,7 +6692,7 @@ packages: dev: false file:projects/erc20-transfers.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-ch12dYQwXZ0AJw/99dxAR9eUhAMZXkOX4f20B7hqoEQavSFASWMOPZ9l6LB1lH3YnS5fBgrgn6dyIjLc9McCSA==, tarball: file:projects/erc20-transfers.tgz} + resolution: {integrity: sha512-AN4uyAnmQuT4N8ns6IAYjw2p6f1suGPLwX/Sj1CrIJ69zUuAK5qPd+dYEDfwKm0FiEbB5/pJya/CQCl3M3tvpw==, tarball: file:projects/erc20-transfers.tgz} id: file:projects/erc20-transfers.tgz name: '@rush-temp/erc20-transfers' version: 0.0.0 @@ -6703,7 +6726,7 @@ packages: dev: false file:projects/evm-abi.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-KGidYnAntwDgonJGlDFpUjhlPahyM1xgkj4cHOnpxVypF7wnahUo3AYG8A++9koD2KQBSmxsFsNaljj2Lgn5rA==, tarball: file:projects/evm-abi.tgz} + resolution: {integrity: sha512-YOB7H12nuCatlWP8NECCnDDpRL7l8wBxH2BauIZQ/4rGNf5zgemknH8DemVjO1wsk5aqx2Nm2lPUw6oF29e4OQ==, tarball: file:projects/evm-abi.tgz} id: file:projects/evm-abi.tgz name: '@rush-temp/evm-abi' version: 0.0.0 @@ -6733,7 +6756,7 @@ packages: dev: false file:projects/evm-codec.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-w1XelVKp/Sb6coGaQYXQw1ho/4UxFlZ2bc9xidnxatDY7yUNGn0TgBOI49OFEPvThDFYCuutY0j9Qre2aWRI4w==, tarball: file:projects/evm-codec.tgz} + resolution: {integrity: sha512-lcRV7X5lEKL8SNPHDvavJN1SAqUq/x+I6b2ArX49I0fLdmmxia0Hw5zKtA4TrbxTFnq9hIfQSgsLrHlzOS8VNg==, tarball: file:projects/evm-codec.tgz} id: file:projects/evm-codec.tgz name: '@rush-temp/evm-codec' version: 0.0.0 @@ -6762,7 +6785,7 @@ packages: dev: false file:projects/evm-processor.tgz: - resolution: {integrity: sha512-QjnluQmW9FI9zI51Kl3xpjK91z7O8taHttj2mMHO3J61beFmSbFgO1iIlUaXOm5f6AQecuxbXAN5+hBKkqm36w==, tarball: file:projects/evm-processor.tgz} + resolution: {integrity: sha512-RoOv+nclQlx/zDAnZmmyDfvG//UkB7b7zkir+M19TX944SsX4BbskPMYBqGX4oIweT+62cOYGD2GYPEBt+Nr9g==, tarball: file:projects/evm-processor.tgz} name: '@rush-temp/evm-processor' version: 0.0.0 dependencies: @@ -6771,7 +6794,7 @@ packages: dev: false file:projects/evm-typegen.tgz: - resolution: {integrity: sha512-HsCgpWkqSwWLDXiToIldaqNXQzUWuC1zviMI7Beb81eRL6Nur5KPU+JJkdRf24EkxOf2XNTCl0evhr8gwVLcdg==, tarball: file:projects/evm-typegen.tgz} + resolution: {integrity: sha512-12v7+7DuFV5DfWMjr7F9EUlsd3Fi2VbiZ8Br9WgclQP5Gg1UpZiZVF7J7fTV9reX1U+I7In+lHEKHh8yRx1DlA==, tarball: file:projects/evm-typegen.tgz} name: '@rush-temp/evm-typegen' version: 0.0.0 dependencies: @@ -6798,7 +6821,7 @@ packages: dev: false file:projects/fuel-data.tgz: - resolution: {integrity: sha512-dNmf9p1d7ilXI4tRNCaUHdDxXbIk3PyT+8bCidHbgofS8t3ZKVw+YdQ2HvRp5slbJ6NnHWAb1YcjYFLGlPp1RQ==, tarball: file:projects/fuel-data.tgz} + resolution: {integrity: sha512-VuXVfyKxlcvaYVTCI0RBJ9qYRwGT/XEGDXqzc3Pa4TttUquQFO0bWB47slxgPiBHYAIgd9/ZejW0DO+UM9k5Fw==, tarball: file:projects/fuel-data.tgz} name: '@rush-temp/fuel-data' version: 0.0.0 dependencies: @@ -6807,7 +6830,7 @@ packages: dev: false file:projects/fuel-dump.tgz: - resolution: {integrity: sha512-uYwx3hc4Vc/pskvTl2RhFp49KzH4YztbdoN+yPoYPzfQdWUgvOrX1/ZDWcWkR9ccjbOvbDBSr6St1EeLeigqjw==, tarball: file:projects/fuel-dump.tgz} + resolution: {integrity: sha512-9j6rLrBfJ5ko3as3QmZH5HnzGM05+CksWCTgZi0FOLMoB336JOgzLCGNTgR/BMTdgcccDjgKqIsvWB/OOSQFpg==, tarball: file:projects/fuel-dump.tgz} name: '@rush-temp/fuel-dump' version: 0.0.0 dependencies: @@ -6816,7 +6839,7 @@ packages: dev: false file:projects/fuel-indexer.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-Lpgs5YWKEUwa4RNULHVdGAkwD1ls4hig8fs18fS4Fs6w776hw7v0W1H1Pp+fByqX1o677tITg06nUULpc7pvhA==, tarball: file:projects/fuel-indexer.tgz} + resolution: {integrity: sha512-5/wI8rl+4dPJDnKYxprkRahZQBggGZ8PFIb++U/Ux8iviuzSgrcTp5QXpbBP0hNSMRItg+EXlEuZ9soH7a+7zg==, tarball: file:projects/fuel-indexer.tgz} id: file:projects/fuel-indexer.tgz name: '@rush-temp/fuel-indexer' version: 0.0.0 @@ -6847,7 +6870,7 @@ packages: dev: false file:projects/fuel-ingest.tgz: - resolution: {integrity: sha512-J4qwkpMsR9v3Tffp05tQWXnRkVKXz0Z22nItSqsz58H8xCWhAXOXs5vuc4AfSvwn/J7wDlR8P+PX1kyT7husjQ==, tarball: file:projects/fuel-ingest.tgz} + resolution: {integrity: sha512-S0MBibhUnPHGEZDmPXeg7Ty0n9KUy8unbwc0KwyFsM6wZPTtLxY7vvEI/fyRE776Zwv1PQC0v05AB7HCNDvrvQ==, tarball: file:projects/fuel-ingest.tgz} name: '@rush-temp/fuel-ingest' version: 0.0.0 dependencies: @@ -6856,7 +6879,7 @@ packages: dev: false file:projects/fuel-normalization.tgz: - resolution: {integrity: sha512-zBSLvem3QcfHry+1tXc/dxvGX/0lL6MtwYjIw8P3mtENMychDho9FwQvOwNt5gAg2qgvJEgSef3GW98ESUjspg==, tarball: file:projects/fuel-normalization.tgz} + resolution: {integrity: sha512-5OC0ntR4l4na/fJyUnjuN7pGx2eF5ice4qg/Ct9VtcqKt3oCVlO6hU2JORMwU1xQRFYq1QQK1pujEucrsnj+pQ==, tarball: file:projects/fuel-normalization.tgz} name: '@rush-temp/fuel-normalization' version: 0.0.0 dependencies: @@ -6865,7 +6888,7 @@ packages: dev: false file:projects/fuel-objects.tgz: - resolution: {integrity: sha512-GWl3yKfrwUeqbl/ov+0AXha8FTkye7hl/uD0jK4pEh9+8d8xGX4fkMclBx/YLNy9JCryPiCtX6kEXNE9sH/74w==, tarball: file:projects/fuel-objects.tgz} + resolution: {integrity: sha512-OJ8Vr2eQP+TmuJ3L2Asj8hxegmNn8aT21XGf+FChQ7SWYEj+VqMtMtpYAqjtG8U5Gyk7wuobahNHuVMVwZFBjg==, tarball: file:projects/fuel-objects.tgz} name: '@rush-temp/fuel-objects' version: 0.0.0 dependencies: @@ -6874,7 +6897,7 @@ packages: dev: false file:projects/fuel-stream.tgz: - resolution: {integrity: sha512-ETASw7qHqdaZlb3lVXkwLg/hnuH8BNlisS5DtM/BE/f2jk6s6fD+ruqa6XKPonNI0NbcJE+ajfUaODjBRNaP+g==, tarball: file:projects/fuel-stream.tgz} + resolution: {integrity: sha512-Mh5LbXoIpmruRVgdRvjZsWUBL6Rws5EMnloykA7f/GgTyqPNbpr1PAw8rqxN7BnD1wqJv38BptByeu3A2dA5pw==, tarball: file:projects/fuel-stream.tgz} name: '@rush-temp/fuel-stream' version: 0.0.0 dependencies: @@ -6901,7 +6924,7 @@ packages: dev: false file:projects/graphql-server.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-Em4HlNc1XsbxZlBDWgi/kyzr3bKi+aey58DMFrJIhpdbZWuUnFVWQUx2MEH5Gd8G+7CJUNOnM8sLI3Y5rcghPg==, tarball: file:projects/graphql-server.tgz} + resolution: {integrity: sha512-yQm2KmgdGJjxe5XLuqx2vUc9DJFc7+WIXiZR81nCke0HlCjZrlrZ5IV6TreN7TQdBNeQ2iAQEnyHxnCGmpFyOQ==, tarball: file:projects/graphql-server.tgz} id: file:projects/graphql-server.tgz name: '@rush-temp/graphql-server' version: 0.0.0 @@ -7005,7 +7028,7 @@ packages: dev: false file:projects/openreader.tgz(supports-color@8.1.1): - resolution: {integrity: sha512-1puqNG9tXqqbQFfGZ7BlcGGLqKiEd6QyjBVyjzkvaW+OQIe1Bwauo/bGl12rBmo/dmREuXfKcMRu3d38noDNdA==, tarball: file:projects/openreader.tgz} + resolution: {integrity: sha512-1yTnDvAUuWzyPC2bH2BZUtYhRMgK7Yh1MDnWwEyBjEhkhrJp/oqZ+8LbN8GpWREaQusvqQn+0U5PLiWqSjfNrw==, tarball: file:projects/openreader.tgz} id: file:projects/openreader.tgz name: '@rush-temp/openreader' version: 0.0.0 @@ -7050,7 +7073,7 @@ packages: dev: false file:projects/raw-archive-validator.tgz: - resolution: {integrity: sha512-q+OGPdLgI3fMid3VGwbwha6STqbMNcC51DlCT89DHIN5L8yJCeT6J9J3xhYRZcjdi/y5FDHH+q/X1VBnI74V2g==, tarball: file:projects/raw-archive-validator.tgz} + resolution: {integrity: sha512-cNBWqHaHWG16Os44defUv2B8FunEseiZFZU3OiNCh6FQXMi2lDWfIVUZ1ls920y/ZImJKBeSWiJ3SAM2b7tp7Q==, tarball: file:projects/raw-archive-validator.tgz} name: '@rush-temp/raw-archive-validator' version: 0.0.0 dependencies: @@ -7089,7 +7112,7 @@ packages: dev: false file:projects/shibuya-psp22.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-UrElinCriyBs2rDud5OJBuRr5TSOQEPviv/aaVCOiiPomNkWReqxRZ1dfKpC2Ab6WaTo+FJ2Kp7Vs48Uz/bF/Q==, tarball: file:projects/shibuya-psp22.tgz} + resolution: {integrity: sha512-1t8Hsg3mW5H7WUZGfcgmUAO08W1QcSQr3t8fkH9E1QhlngZFRh3zCmnETKAcINB5O5qqvzPXEqo9rR7cZv27cQ==, tarball: file:projects/shibuya-psp22.tgz} id: file:projects/shibuya-psp22.tgz name: '@rush-temp/shibuya-psp22' version: 0.0.0 @@ -7120,7 +7143,7 @@ packages: dev: false file:projects/solana-dump.tgz: - resolution: {integrity: sha512-j0NXfhUpCwqsp+WJN5lPFcKUCQNLg1sO48TlJiVRh3oqbrqbk/2YAL1HpbT7IYiUTwjBTDH80t/kq12iIWr0Og==, tarball: file:projects/solana-dump.tgz} + resolution: {integrity: sha512-YzB/1GR4S3BsjfuX2DrapDSm4fimLkLyx2oOxatzkl5Pwgl7IWNNY1MnGcKkvFnRhLZoV8QnCjS+R49BYtiwHw==, tarball: file:projects/solana-dump.tgz} name: '@rush-temp/solana-dump' version: 0.0.0 dependencies: @@ -7129,7 +7152,7 @@ packages: dev: false file:projects/solana-example.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-weRsJJ5qlielUcNWVf7Y/cbsnB1lPnwD3teFDjELKZaehmqAMX3C0rMNp1yLjokpgPk9z4avMmR0QcyjNQ1zZQ==, tarball: file:projects/solana-example.tgz} + resolution: {integrity: sha512-Ns7M8TC//cSLfJybpLFsop3TVKD33gDaCcfjs6VUh/UDsMpEPOmxyGEXmYzujPpmpc/7GD7iBXJ52kZ/po/fHg==, tarball: file:projects/solana-example.tgz} id: file:projects/solana-example.tgz name: '@rush-temp/solana-example' version: 0.0.0 @@ -7160,7 +7183,7 @@ packages: dev: false file:projects/solana-ingest.tgz: - resolution: {integrity: sha512-7cgYeih8AyL9y/y8nq61M44dK9XjxJwQPC1Ydb4Fp12AzwYL8T7ZhJmCVXBr+LIIYLi3USr4hHBgxg3fgxeZMw==, tarball: file:projects/solana-ingest.tgz} + resolution: {integrity: sha512-jfOG8LHbabxvfyr66/F9IvuwLO7XdSE4dEdlPZhrjoq+m7k0gtJ16QVTKDuhAA4U8qOqwtQicuzfPKy+df3iPQ==, tarball: file:projects/solana-ingest.tgz} name: '@rush-temp/solana-ingest' version: 0.0.0 dependencies: @@ -7169,7 +7192,7 @@ packages: dev: false file:projects/solana-normalization.tgz: - resolution: {integrity: sha512-O//+YPMmLqlNLdWYmga/ACWv49TMqzDMJB2iiJZRZzicUODmqFhhq7aY/ZQQe8NV/HrCS19cNomP7YQXHKfVHg==, tarball: file:projects/solana-normalization.tgz} + resolution: {integrity: sha512-y3093dWykFo9bW4AiRNqNSi1DFFvPjEnrr6aMN7Xgwrg86DmvVLhTLJRYk0MUiGp/nD4+WJKOIOd9qkzAy283w==, tarball: file:projects/solana-normalization.tgz} name: '@rush-temp/solana-normalization' version: 0.0.0 dependencies: @@ -7178,7 +7201,7 @@ packages: dev: false file:projects/solana-objects.tgz: - resolution: {integrity: sha512-LZrDlFBde7ywNHYsqlBnWktjLa4aPw2z8ARdoyfh3P3cMoBwtONGnUc33cVb2Jhp+tMlQ1v4mo4RJVqYXuHoGg==, tarball: file:projects/solana-objects.tgz} + resolution: {integrity: sha512-69rX5eue8et5dXGe4uInoFAfW/D5liK8cfcuQX34asOQXKP53i921uSBQDLvbvoxi4JkwGeQur7AjTeKXfLmWQ==, tarball: file:projects/solana-objects.tgz} name: '@rush-temp/solana-objects' version: 0.0.0 dependencies: @@ -7187,7 +7210,7 @@ packages: dev: false file:projects/solana-rpc-data.tgz: - resolution: {integrity: sha512-6wx7ff9zAv3F9wPMB24SuL4EEQYKASBuhMp/nQZvISkEBOFTFnQ1l5N9y+vUY1aMU7+MM6fsvHlLcmUSpd4p9Q==, tarball: file:projects/solana-rpc-data.tgz} + resolution: {integrity: sha512-OjV0jXCULMC0mll8sJlnUEs/WLflfyg8H6RdS0bw1qSzlUrxPWzEKYrP+axvlLwvdnCpZnceYHi4ePa0lqzFfA==, tarball: file:projects/solana-rpc-data.tgz} name: '@rush-temp/solana-rpc-data' version: 0.0.0 dependencies: @@ -7196,7 +7219,7 @@ packages: dev: false file:projects/solana-rpc.tgz: - resolution: {integrity: sha512-WIGmFLuy0IfwhHSITTAvY4ayHzUzgCDci2JvCf1OrOSRTvcOHvswF/Ubmh4BGz2hsIw1/vOndBvCTg2wQx3Pcg==, tarball: file:projects/solana-rpc.tgz} + resolution: {integrity: sha512-+UyXUQwPC520Mt3cwmiJ5CclgON5Fvc7Zw2HVyI/uhME8d2/iSNwjaEsFvYlSZanqfK2mp6IORum0QOZN/d+Tg==, tarball: file:projects/solana-rpc.tgz} name: '@rush-temp/solana-rpc' version: 0.0.0 dependencies: @@ -7205,7 +7228,7 @@ packages: dev: false file:projects/solana-stream.tgz: - resolution: {integrity: sha512-DAb2WWlb+uDB6ze1prkOsHKWXH9cAzkWxh2LubZIDlQTfi2cYkiwYBdvA6yYCUraW1177YVffKzN8I+9gGV6qA==, tarball: file:projects/solana-stream.tgz} + resolution: {integrity: sha512-jNA0VI23u7Z8mdSYSATMKKz5d8Y/ckdo2oRgON+a22cQSvIqRXGbX8yp2RGYK47YkL2pEhfZ6o1GX9il4bxmeA==, tarball: file:projects/solana-stream.tgz} name: '@rush-temp/solana-stream' version: 0.0.0 dependencies: @@ -7215,7 +7238,7 @@ packages: dev: false file:projects/solana-typegen.tgz: - resolution: {integrity: sha512-370CDKZNivGBMfcnWSmTnEuy4qEi1ZetBEtM9eJrWMBMfLkbcQ3oeKdPWpzzl0isSPy8YJxe1/rYKPfkymv0IQ==, tarball: file:projects/solana-typegen.tgz} + resolution: {integrity: sha512-LSlfR1IVH+qzkPzeDnDL7VHsbCNSf6ClQKsMxn4lXcwjYI6dbX2RcWJX+/ImZnWCYJ1k+RtRSN+n6akiAsKcBA==, tarball: file:projects/solana-typegen.tgz} name: '@rush-temp/solana-typegen' version: 0.0.0 dependencies: @@ -7248,7 +7271,7 @@ packages: dev: false file:projects/substrate-data-raw.tgz: - resolution: {integrity: sha512-o7h09auR6YZ/ZQViKLAM8fuEq+PFlk/DCQWH2ShJ1AwsGuZzAesha2oGdUNt56RxFtdAQk9hM6ySbDwuNI8aFw==, tarball: file:projects/substrate-data-raw.tgz} + resolution: {integrity: sha512-T9oA/jEt/x9WA4O38M/74iZ+IbAFzbhCfjX3dVThJIV66mpXhvhXHfN8Fj9kNe4jIjphtU7rWHZE6jvtNcfvGw==, tarball: file:projects/substrate-data-raw.tgz} name: '@rush-temp/substrate-data-raw' version: 0.0.0 dependencies: @@ -7257,7 +7280,7 @@ packages: dev: false file:projects/substrate-data.tgz: - resolution: {integrity: sha512-Omt2Tp505it+SDM9tReb6zEjWGvOzEab4mbbs6FpKiPSupM5+eMkCA1trn9rp/Pk+QUvkuv0CNn7r/HXb7sU2A==, tarball: file:projects/substrate-data.tgz} + resolution: {integrity: sha512-qQHwx9m0DMmQPIwS+4hqTdxToovLJAEukLYTqZQCyF/qnAkFC9ozmyfkuOSkz+chcAbrryouJHwxQgn6xdKMYQ==, tarball: file:projects/substrate-data.tgz} name: '@rush-temp/substrate-data' version: 0.0.0 dependencies: @@ -7268,7 +7291,7 @@ packages: dev: false file:projects/substrate-dump.tgz: - resolution: {integrity: sha512-Tn+o7bwXWDzCZbWQv7qAlxkpcNVjW6Sz7kKk7CXIKPwPwd3QunYTw60IasBuq0rJHTVVCew3GGCb5ipapXMjdQ==, tarball: file:projects/substrate-dump.tgz} + resolution: {integrity: sha512-FUJ19MipnL80R+xikPlYWNL85EWS5DFFMnvL09LgAxi2JjvfoBfvY2jUuUkezEpn0hOFyFrGFKozEnc3kEcAUw==, tarball: file:projects/substrate-dump.tgz} name: '@rush-temp/substrate-dump' version: 0.0.0 dependencies: @@ -7277,7 +7300,7 @@ packages: dev: false file:projects/substrate-ingest.tgz: - resolution: {integrity: sha512-N0gpgIiC5rj1kQIIwMqPMrIGwbtvQ3kMzX3s548N1ZMw7usv+MjgbYJSw0744o54ETZ8KOH2YTI1rZHcimmn7A==, tarball: file:projects/substrate-ingest.tgz} + resolution: {integrity: sha512-XmCLuG8FqnRe5pUUb1gM23NSJbtZxrryH1ubHztSHGhQwZPYRAC6kysLH2srxF99OkuxVCdWPywDtNCW0i0Ddg==, tarball: file:projects/substrate-ingest.tgz} name: '@rush-temp/substrate-ingest' version: 0.0.0 dependencies: @@ -7305,7 +7328,7 @@ packages: dev: false file:projects/substrate-processor.tgz: - resolution: {integrity: sha512-3fvboYXqkBYCSH13tnzyK522VPy+3hhHtSSYJPFFDVuSokA2/flhEciSiPsxc2vemX8BVdvcuanPaFP2dJG1eQ==, tarball: file:projects/substrate-processor.tgz} + resolution: {integrity: sha512-wZi9NLCMCfG3NgxjRylCKqUkV9tZ/re41pzUYpo+srZ+G/PQIz202lM5hcyi3QBvg9Z3/s/ix/18xI2iNrYoIA==, tarball: file:projects/substrate-processor.tgz} name: '@rush-temp/substrate-processor' version: 0.0.0 dependencies: @@ -7314,7 +7337,7 @@ packages: dev: false file:projects/substrate-runtime.tgz: - resolution: {integrity: sha512-ndbIQwyTh3t+Q5AiXk/oawuGVHqQWsCZcTfuPYx++EGYNRcFfYvM3kf9AvFYV0xuptoXTew+VnTHeWlnz0i+pQ==, tarball: file:projects/substrate-runtime.tgz} + resolution: {integrity: sha512-BQmKHPLpPHo90qwSymGADy8dhzW4n+upTa5fynp5DnRKiGm5cy/KYEI0bHj9h6mXfxidsNCpPauSmD6PzwPhmw==, tarball: file:projects/substrate-runtime.tgz} name: '@rush-temp/substrate-runtime' version: 0.0.0 dependencies: @@ -7327,7 +7350,7 @@ packages: dev: false file:projects/substrate-typegen.tgz: - resolution: {integrity: sha512-qXKZIoyz64qLhPMU063hgMeRtGvknlgU7T33O85IY1Fh4KKjaXz9xj8RbHVVso94EB0oktedgqT6JZyzXGtJYw==, tarball: file:projects/substrate-typegen.tgz} + resolution: {integrity: sha512-q6W2NzSV3iKpBeBoQ9qA4z1wawx10JZZ2wT5zIpYnl38NjD+1cn5Di3arUbWHsAo6FU33poRVP83Hs0ctH0Z/Q==, tarball: file:projects/substrate-typegen.tgz} name: '@rush-temp/substrate-typegen' version: 0.0.0 dependencies: @@ -7337,7 +7360,7 @@ packages: dev: false file:projects/typeorm-codegen.tgz: - resolution: {integrity: sha512-NHbOI/JYD7iz4i03UWvMKIZ0d7VbLZB5adSLmkK7nFKjkBFUlDjGoGrMI3CkVO3ToL4VA6s5oujD3nKAykCccA==, tarball: file:projects/typeorm-codegen.tgz} + resolution: {integrity: sha512-6cdUKkYHaSNAZtjWrOQCYCKWnHdWEDzy3GV3jxO1PpDd6ySXiujA5ZsPWOAMSINVs3bfu9tpwvTnvXRZTjjTqw==, tarball: file:projects/typeorm-codegen.tgz} name: '@rush-temp/typeorm-codegen' version: 0.0.0 dependencies: @@ -7347,7 +7370,7 @@ packages: dev: false file:projects/typeorm-config.tgz(pg@8.11.5)(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-wKT5CJd1nmlelB6/N9XlgRr89zsY3ZUZ0hxeWxflFK0CXHEpoCAuH9NOue6qpVdTFDDCSVuzgW2pHJhcbHLRwQ==, tarball: file:projects/typeorm-config.tgz} + resolution: {integrity: sha512-QfGFyFFGo+xz1wg5RR7Ez1YZ632lb960Jza+E40D+qHkGjs3R6/6nd4n5AVbPA/hseDGLFFgdYAGCSEYPzdtDQ==, tarball: file:projects/typeorm-config.tgz} id: file:projects/typeorm-config.tgz name: '@rush-temp/typeorm-config' version: 0.0.0 @@ -7377,7 +7400,7 @@ packages: dev: false file:projects/typeorm-migration.tgz(pg@8.11.5)(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-1qIAhun15pIWQR53FR4r9vGfopSAqEkhZwOYC1rlNoIzHd+szSGv5+T9VhrT3qd4nnIJsnTNMnu6D2aRRDEI5Q==, tarball: file:projects/typeorm-migration.tgz} + resolution: {integrity: sha512-pMPW0qPmab6ZlrE5VibdnG+OeyQWdrFy81mL+YkoiMhqHbvwLjGRauthM5Bj7hiGcrXSgRU74FUHQWaY5gC87Q==, tarball: file:projects/typeorm-migration.tgz} id: file:projects/typeorm-migration.tgz name: '@rush-temp/typeorm-migration' version: 0.0.0 @@ -7409,17 +7432,21 @@ packages: dev: false file:projects/typeorm-store.tgz(supports-color@8.1.1)(ts-node@10.9.2): - resolution: {integrity: sha512-sXhxNVktY6PiOA0HiRCYo8dJlsT4AH259wOZ3ledM0FnVRV+6XxQ6Btbd2Yw/9N74raYn5FeRyhSYcewH0Zjmg==, tarball: file:projects/typeorm-store.tgz} + resolution: {integrity: sha512-Uts/GfKPO3MjUUHEfH9Ilu4WW2MvcDUlTU/7dPLEB0JzEssKzczEzXo0w6gKYej4abSQ58m82SqWVVHyMIBD9w==, tarball: file:projects/typeorm-store.tgz} id: file:projects/typeorm-store.tgz name: '@rush-temp/typeorm-store' version: 0.0.0 dependencies: + '@types/clone': 2.1.4 '@types/mocha': 10.0.6 '@types/node': 18.19.31 '@types/pg': 8.11.5 + clone: 2.1.2 expect: 29.7.0 + fast-copy: 3.0.2 mocha: 10.4.0 pg: 8.11.5 + rfdc: 1.4.1 typeorm: 0.3.20(pg@8.11.5)(supports-color@8.1.1)(ts-node@10.9.2) typescript: 5.3.3 transitivePeerDependencies: @@ -7515,7 +7542,7 @@ packages: dev: false file:projects/util-internal-dump-cli.tgz: - resolution: {integrity: sha512-RSNg1RNaDTjBXo1gBwOQrhwSB2t1yHPPm6Dtt7dORMcWod2AubPtG+JRXodw5FsulKMKL+EQ2CYGjTNycboxrA==, tarball: file:projects/util-internal-dump-cli.tgz} + resolution: {integrity: sha512-xgG4eXDo6rW+fJukgdy7UofH0WOSwoJBYu894LYJq7BscGyFgA5vg4hPB7Oyngt1WOQUalKiolKFsIpPCXaqZQ==, tarball: file:projects/util-internal-dump-cli.tgz} name: '@rush-temp/util-internal-dump-cli' version: 0.0.0 dependencies: @@ -7559,7 +7586,7 @@ packages: dev: false file:projects/util-internal-ingest-cli.tgz: - resolution: {integrity: sha512-ACgY+iOdRYoLQhT1B1ys26kxz8WNBf1TP8trLw+yXUyM8tSZ6rg5VXM3raB89TO38vGBTOVC1Xw/2V9/lmiwzA==, tarball: file:projects/util-internal-ingest-cli.tgz} + resolution: {integrity: sha512-aUnjqkCZ1K036D1WsGDalurA8VO5LOChQxgqrTXNHZwIsNU92wERfGdHQ8IkfXDQ/n7TcL/MRd1fNjS6HzftWg==, tarball: file:projects/util-internal-ingest-cli.tgz} name: '@rush-temp/util-internal-ingest-cli' version: 0.0.0 dependencies: diff --git a/typeorm/typeorm-store/package.json b/typeorm/typeorm-store/package.json index 8596fb28d..f4e7ac3dc 100644 --- a/typeorm/typeorm-store/package.json +++ b/typeorm/typeorm-store/package.json @@ -19,13 +19,16 @@ }, "dependencies": { "@subsquid/typeorm-config": "^4.1.1", - "@subsquid/util-internal": "^3.2.0" + "@subsquid/util-internal": "^3.2.0", + "@subsquid/logger": "^1.3.3", + "fast-copy": "^3.0.2" }, "peerDependencies": { "typeorm": "^0.3.17", "@subsquid/big-decimal": "^1.0.0" }, "devDependencies": { + "@types/clone": "^2.1.4", "@types/mocha": "^10.0.6", "@types/node": "^18.18.14", "@types/pg": "^8.10.9", diff --git a/typeorm/typeorm-store/src/database.ts b/typeorm/typeorm-store/src/database.ts index 9cfe5ff82..b189e07f4 100644 --- a/typeorm/typeorm-store/src/database.ts +++ b/typeorm/typeorm-store/src/database.ts @@ -1,34 +1,96 @@ import {createOrmConfig} from '@subsquid/typeorm-config' -import {assertNotNull, last, maybeLast} from '@subsquid/util-internal' +import {assertNotNull, def, last, maybeLast} from '@subsquid/util-internal' import assert from 'assert' import {DataSource, EntityManager} from 'typeorm' -import {ChangeTracker, rollbackBlock} from './hot' +import {ChangeWriter, rollbackBlock} from './utils/changeWriter' import {DatabaseState, FinalTxInfo, HashAndHeight, HotTxInfo} from './interfaces' import {Store} from './store' +import {createLogger} from '@subsquid/logger' +import {StateManager} from './utils/stateManager' +import {sortMetadatasInCommitOrder} from './utils/commitOrder' +import {IsolationLevel} from './utils/tx' - -export type IsolationLevel = 'SERIALIZABLE' | 'READ COMMITTED' | 'REPEATABLE READ' - +export {IsolationLevel} export interface TypeormDatabaseOptions { + /** + * Support for storing the data on unfinalized + * blocks and the related rollbacks. + * See {@link https://docs.subsquid.io/sdk/resources/basics/unfinalized-blocks/} + * + * @defaultValue true + */ supportHotBlocks?: boolean + + /** + * PostgreSQL transaction isolation level + * See {@link https://www.postgresql.org/docs/current/transaction-iso.html} + * + * @defaultValue 'SERIALIZABLE' + */ isolationLevel?: IsolationLevel + + /** + * @defaultValue true + */ + batchWriteOperations?: boolean + + /** + * @defaultValue true + */ + cacheEntitiesByDefault?: boolean + + // FIXME: needs better name, means if we check db if entity is not found in the state + /** + * @defaultValue true + */ + syncOnGet?: boolean + + /** + * @defaultValue true + */ + resetOnCommit?: boolean + + /** + * Name of the database schema that the processor + * will use to track its state (height and hash of + * the highest indexed block). Set this if you run + * more than one processor against the same DB. + * + * @defaultValue 'squid_processor' + */ stateSchema?: string + + /** + * Directory with model definitions (at lib/model) + * and migrations (at db/migrations). + * + * @defaultValue process.cwd() + */ projectDir?: string } +const STATE_MANAGERS: WeakMap = new WeakMap() export class TypeormDatabase { - private statusSchema: string - private isolationLevel: IsolationLevel - private con?: DataSource - private projectDir: string + protected statusSchema: string + protected isolationLevel: IsolationLevel + protected batchWriteOperations: boolean + protected cacheEntitiesByDefault: boolean + protected syncOnGet: boolean + protected resetOnCommit: boolean + protected con?: DataSource + protected projectDir: string public readonly supportsHotBlocks: boolean constructor(options?: TypeormDatabaseOptions) { this.statusSchema = options?.stateSchema || 'squid_processor' this.isolationLevel = options?.isolationLevel || 'SERIALIZABLE' + this.batchWriteOperations = options?.batchWriteOperations ?? true + this.cacheEntitiesByDefault = options?.cacheEntitiesByDefault ?? true + this.syncOnGet = options?.syncOnGet ?? true + this.resetOnCommit = options?.resetOnCommit ?? true this.supportsHotBlocks = options?.supportHotBlocks !== false this.projectDir = options?.projectDir || process.cwd() } @@ -42,8 +104,8 @@ export class TypeormDatabase { await this.con.initialize() try { - return await this.con.transaction('SERIALIZABLE', em => this.initTransaction(em)) - } catch(e: any) { + return await this.con.transaction('SERIALIZABLE', (em) => this.initTransaction(em)) + } catch (e: any) { await this.con.destroy().catch(() => {}) // ignore error this.con = undefined throw e @@ -51,39 +113,37 @@ export class TypeormDatabase { } async disconnect(): Promise { - await this.con?.destroy().finally(() => this.con = undefined) + await this.con?.destroy().finally(() => (this.con = undefined)) } private async initTransaction(em: EntityManager): Promise { let schema = this.escapedSchema() - await em.query( - `CREATE SCHEMA IF NOT EXISTS ${schema}` - ) + await em.query(`CREATE SCHEMA IF NOT EXISTS ${schema}`) await em.query( `CREATE TABLE IF NOT EXISTS ${schema}.status (` + - `id int4 primary key, ` + - `height int4 not null, ` + - `hash text DEFAULT '0x', ` + - `nonce int4 DEFAULT 0`+ - `)` + `id int4 primary key, ` + + `height int4 not null, ` + + `hash text DEFAULT '0x', ` + + `nonce int4 DEFAULT 0` + + `)` ) - await em.query( // for databases created by prev version of typeorm store + await em.query( + // for databases created by prev version of typeorm store `ALTER TABLE ${schema}.status ADD COLUMN IF NOT EXISTS hash text DEFAULT '0x'` ) - await em.query( // for databases created by prev version of typeorm store - `ALTER TABLE ${schema}.status ADD COLUMN IF NOT EXISTS nonce int DEFAULT 0` - ) await em.query( - `CREATE TABLE IF NOT EXISTS ${schema}.hot_block (height int4 primary key, hash text not null)` + // for databases created by prev version of typeorm store + `ALTER TABLE ${schema}.status ADD COLUMN IF NOT EXISTS nonce int DEFAULT 0` ) + await em.query(`CREATE TABLE IF NOT EXISTS ${schema}.hot_block (height int4 primary key, hash text not null)`) await em.query( `CREATE TABLE IF NOT EXISTS ${schema}.hot_change_log (` + - `block_height int4 not null references ${schema}.hot_block on delete cascade, ` + - `index int4 not null, ` + - `change jsonb not null, ` + - `PRIMARY KEY (block_height, index)` + - `)` + `block_height int4 not null references ${schema}.hot_block on delete cascade, ` + + `index int4 not null, ` + + `change jsonb not null, ` + + `PRIMARY KEY (block_height, index)` + + `)` ) let status: (HashAndHeight & {nonce: number})[] = await em.query( @@ -94,9 +154,7 @@ export class TypeormDatabase { status.push({height: -1, hash: '0x', nonce: 0}) } - let top: HashAndHeight[] = await em.query( - `SELECT height, hash FROM ${schema}.hot_block ORDER BY height` - ) + let top: HashAndHeight[] = await em.query(`SELECT height, hash FROM ${schema}.hot_block ORDER BY height`) return assertStateInvariants({...status[0], top}) } @@ -110,15 +168,13 @@ export class TypeormDatabase { assert(status.length == 1) - let top: HashAndHeight[] = await em.query( - `SELECT hash, height FROM ${schema}.hot_block ORDER BY height` - ) + let top: HashAndHeight[] = await em.query(`SELECT hash, height FROM ${schema}.hot_block ORDER BY height`) return assertStateInvariants({...status[0], top}) } transact(info: FinalTxInfo, cb: (store: Store) => Promise): Promise { - return this.submit(async em => { + return this.submit(async (em) => { let state = await this.getState(em) let {prevHead: prev, nextHead: next} = info @@ -146,15 +202,21 @@ export class TypeormDatabase { }) } - transactHot2(info: HotTxInfo, cb: (store: Store, sliceBeg: number, sliceEnd: number) => Promise): Promise { - return this.submit(async em => { + transactHot2( + info: HotTxInfo, + cb: (store: Store, sliceBeg: number, sliceEnd: number) => Promise + ): Promise { + return this.submit(async (em) => { let state = await this.getState(em) let chain = [state, ...state.top] assertChainContinuity(info.baseHead, info.newBlocks) assert(info.finalizedHead.height <= (maybeLast(info.newBlocks) ?? info.baseHead).height) - assert(chain.find(b => b.hash === info.baseHead.hash), RACE_MSG) + assert( + chain.find((b) => b.hash === info.baseHead.hash), + RACE_MSG + ) if (info.newBlocks.length == 0) { assert(last(chain).hash === info.baseHead.hash, RACE_MSG) } @@ -169,7 +231,7 @@ export class TypeormDatabase { if (info.newBlocks.length) { let finalizedEnd = info.finalizedHead.height - info.newBlocks[0].height + 1 if (finalizedEnd > 0) { - await this.performUpdates(store => cb(store, 0, finalizedEnd), em) + await this.performUpdates((store) => cb(store, 0, finalizedEnd), em) } else { finalizedEnd = 0 } @@ -177,9 +239,9 @@ export class TypeormDatabase { let b = info.newBlocks[i] await this.insertHotBlock(em, b) await this.performUpdates( - store => cb(store, i, i + 1), + (store) => cb(store, i, i + 1), em, - new ChangeTracker(em, this.statusSchema, b.height) + new ChangeWriter(em, this.statusSchema, b.height) ) } } @@ -195,17 +257,14 @@ export class TypeormDatabase { } private deleteHotBlocks(em: EntityManager, finalizedHeight: number): Promise { - return em.query( - `DELETE FROM ${this.escapedSchema()}.hot_block WHERE height <= $1`, - [finalizedHeight] - ) + return em.query(`DELETE FROM ${this.escapedSchema()}.hot_block WHERE height <= $1`, [finalizedHeight]) } private insertHotBlock(em: EntityManager, block: HashAndHeight): Promise { - return em.query( - `INSERT INTO ${this.escapedSchema()}.hot_block (height, hash) VALUES ($1, $2)`, - [block.height, block.hash] - ) + return em.query(`INSERT INTO ${this.escapedSchema()}.hot_block (height, hash) VALUES ($1, $2)`, [ + block.height, + block.hash, + ]) } private async updateStatus(em: EntityManager, nonce: number, next: HashAndHeight): Promise { @@ -220,32 +279,29 @@ export class TypeormDatabase { // Will never happen if isolation level is SERIALIZABLE or REPEATABLE_READ, // but occasionally people use multiprocessor setups and READ_COMMITTED. - assert.strictEqual( - rowsChanged, - 1, - RACE_MSG - ) + assert.strictEqual(rowsChanged, 1, RACE_MSG) } private async performUpdates( cb: (store: Store) => Promise, em: EntityManager, - changeTracker?: ChangeTracker + changeWriter?: ChangeWriter ): Promise { - let running = true - - let store = new Store( - () => { - assert(running, `too late to perform db updates, make sure you haven't forgot to await on db query`) - return em - }, - changeTracker - ) + let store = new Store({ + em, + state: this.getStateManager(), + logger: this.getLogger(), + changes: changeWriter, + batchWriteOperations: this.batchWriteOperations, + syncOnGet: this.syncOnGet, + cacheEntitiesByDefault: this.cacheEntitiesByDefault, + }) try { await cb(store) + await store.sync(this.resetOnCommit) } finally { - running = false + store['isClosed'] = true } } @@ -256,7 +312,7 @@ export class TypeormDatabase { let con = this.con assert(con != null, 'not connected') return await con.transaction(this.isolationLevel, tx) - } catch(e: any) { + } catch (e: any) { if (e.code == '40001' && retries) { retries -= 1 } else { @@ -270,11 +326,28 @@ export class TypeormDatabase { let con = assertNotNull(this.con) return con.driver.escape(this.statusSchema) } -} + @def + private getLogger() { + return createLogger('sqd:typeorm-db') + } -const RACE_MSG = 'status table was updated by foreign process, make sure no other processor is running' + private getStateManager() { + let con = assertNotNull(this.con) + let stateManager = STATE_MANAGERS.get(con) + if (stateManager != null) return stateManager + stateManager = new StateManager({ + commitOrder: sortMetadatasInCommitOrder(con), + logger: this.getLogger(), + }) + STATE_MANAGERS.set(con, stateManager) + + return stateManager + } +} + +const RACE_MSG = 'status table was updated by foreign process, make sure no other processor is running' function assertStateInvariants(state: DatabaseState): DatabaseState { let height = state.height @@ -287,7 +360,6 @@ function assertStateInvariants(state: DatabaseState): DatabaseState { return state } - function assertChainContinuity(base: HashAndHeight, chain: HashAndHeight[]) { let prev = base for (let b of chain) { diff --git a/typeorm/typeorm-store/src/index.ts b/typeorm/typeorm-store/src/index.ts index 82d4fb142..ccb47792d 100644 --- a/typeorm/typeorm-store/src/index.ts +++ b/typeorm/typeorm-store/src/index.ts @@ -1,4 +1,4 @@ export * from './database' -export {EntityClass, FindManyOptions, FindOneOptions, Store} from './store' +export * from './store' export * from './decorators' -export * from './transformers' \ No newline at end of file +export * from './transformers' diff --git a/typeorm/typeorm-store/src/store.ts b/typeorm/typeorm-store/src/store.ts index d6ead6f58..727b681f7 100644 --- a/typeorm/typeorm-store/src/store.ts +++ b/typeorm/typeorm-store/src/store.ts @@ -1,20 +1,27 @@ -import assert from 'assert' -import {EntityManager, FindOptionsOrder, FindOptionsRelations, FindOptionsWhere} from 'typeorm' +import { + EntityManager, + EntityMetadata, + EntityNotFoundError, + FindOptionsOrder, + FindOptionsRelations, + FindOptionsWhere, +} from 'typeorm' import {EntityTarget} from 'typeorm/common/EntityTarget' +import {ChangeWriter} from './utils/changeWriter' +import {StateManager} from './utils/stateManager' +import {Logger} from '@subsquid/logger' +import {createFuture, Future} from '@subsquid/util-internal' +import {EntityLiteral, noNull, splitIntoBatches, traverseEntity} from './utils/misc' import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata' -import {ChangeTracker} from './hot' - - -export interface EntityClass { - new (): T -} +import assert from 'assert' +export {EntityTarget} -export interface Entity { +export interface GetOptions { id: string + relations?: FindOptionsRelations } - /** * Defines a special criteria to find specific entity. */ @@ -32,43 +39,77 @@ export interface FindOneOptions { /** * Indicates what relations of entity should be loaded (simplified left join form). */ - relations?: FindOptionsRelations; + relations?: FindOptionsRelations /** * Order, in which entities should be ordered. */ order?: FindOptionsOrder -} + cacheEntities?: boolean +} export interface FindManyOptions extends FindOneOptions { /** * Offset (paginated) where from entities should be taken. */ - skip?: number; + skip?: number /** * Limit (paginated) - max number of entities should be taken. */ - take?: number; + take?: number + + cache?: boolean } +export interface StoreOptions { + em: EntityManager + state: StateManager + changes?: ChangeWriter + logger?: Logger + batchWriteOperations: boolean + cacheEntitiesByDefault: boolean + syncOnGet: boolean +} /** * Restricted version of TypeORM entity manager for squid data handlers. */ export class Store { - constructor(private em: () => EntityManager, private changes?: ChangeTracker) {} + protected em: EntityManager + protected state: StateManager + protected changes?: ChangeWriter + protected logger?: Logger + + protected batchWriteOperations: boolean + protected cacheEntitiesByDefault: boolean + protected syncOnGet: boolean + + protected pendingCommit?: Future + protected isClosed = false + + constructor({em, changes, logger, state, ...opts}: StoreOptions) { + this.em = em + this.changes = changes + this.logger = logger?.child('store') + this.state = state + this.batchWriteOperations = opts.batchWriteOperations + this.cacheEntitiesByDefault = opts.cacheEntitiesByDefault + this.syncOnGet = opts.syncOnGet + } + + get _em() { + return this.em + } + + get _state() { + return this.state + } /** * Alias for {@link Store.upsert} */ - save(entity: E): Promise - save(entities: E[]): Promise - save(e: E | E[]): Promise { - if (Array.isArray(e)) { // please the compiler - return this.upsert(e) - } else { - return this.upsert(e) - } + async save(e: E | E[]): Promise { + return this.upsert(e) } /** @@ -76,59 +117,55 @@ export class Store { * * It always executes a primitive operation without cascades, relations, etc. */ - upsert(entity: E): Promise - upsert(entities: E[]): Promise - async upsert(e: E | E[]): Promise { - if (Array.isArray(e)) { - if (e.length == 0) return - let entityClass = e[0].constructor as EntityClass - for (let i = 1; i < e.length; i++) { - assert(entityClass === e[i].constructor, 'mass saving allowed only for entities of the same class') - } - await this.changes?.trackUpsert(entityClass, e) - await this.saveMany(entityClass, e) - } else { - let entityClass = e.constructor as EntityClass - await this.changes?.trackUpsert(entityClass, [e]) - await this.em().upsert(entityClass, e as any, ['id']) - } - } + async upsert(e: E | E[]): Promise { + return await this.performWrite(async () => { + let entities = Array.isArray(e) ? e : [e] + if (entities.length == 0) return - private async saveMany(entityClass: EntityClass, entities: any[]): Promise { - assert(entities.length > 0) - let em = this.em() - let metadata = em.connection.getMetadata(entityClass) - let fk = metadata.columns.filter(c => c.relationMetadata) - if (fk.length == 0) return this.upsertMany(em, entityClass, entities) - let currentSignature = this.getFkSignature(fk, entities[0]) - let batch = [] - for (let e of entities) { - let sig = this.getFkSignature(fk, e) - if (sig === currentSignature) { - batch.push(e) - } else { - await this.upsertMany(em, entityClass, batch) - currentSignature = sig - batch = [e] + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) + this.state.upsert(md, entity) } - } - if (batch.length) { - await this.upsertMany(em, entityClass, batch) - } + }) } private getFkSignature(fk: ColumnMetadata[], entity: any): bigint { let sig = 0n for (let i = 0; i < fk.length; i++) { let bit = fk[i].getEntityValue(entity) === undefined ? 0n : 1n - sig |= (bit << BigInt(i)) + sig |= bit << BigInt(i) } return sig } - private async upsertMany(em: EntityManager, entityClass: EntityClass, entities: any[]): Promise { + private async _upsert(metadata: EntityMetadata, entities: EntityLiteral[]): Promise { + this.logger?.debug(`upsert ${entities.length} ${metadata.name} entities`) + await this.changes?.writeUpsert(metadata, entities) + + let fk = metadata.columns.filter((c) => c.relationMetadata) + if (fk.length == 0) return this.upsertMany(metadata.target, entities) + let signatures = entities + .map((e) => ({entity: e, value: this.getFkSignature(fk, e)})) + .sort((a, b) => (a.value > b.value ? -1 : b.value > a.value ? 1 : 0)) + let currentSignature = signatures[0].value + let batch: EntityLiteral[] = [] + for (let s of signatures) { + if (s.value === currentSignature) { + batch.push(s.entity) + } else { + await this.upsertMany(metadata.target, batch) + currentSignature = s.value + batch = [s.entity] + } + } + if (batch.length) { + await this.upsertMany(metadata.target, batch) + } + } + + private async upsertMany(target: EntityTarget, entities: EntityLiteral[]) { for (let b of splitIntoBatches(entities, 1000)) { - await em.upsert(entityClass, b as any, ['id']) + await this.em.upsert(target, b as any, ['id']) } } @@ -138,114 +175,251 @@ export class Store { * * Executes a primitive INSERT operation without cascades, relations, etc. */ - insert(entity: E): Promise - insert(entities: E[]): Promise - async insert(e: E | E[]): Promise { - if (Array.isArray(e)) { - if (e.length == 0) return - let entityClass = e[0].constructor as EntityClass - for (let i = 1; i < e.length; i++) { - assert(entityClass === e[i].constructor, 'mass saving allowed only for entities of the same class') - } - await this.changes?.trackInsert(entityClass, e) - for (let b of splitIntoBatches(e, 1000)) { - await this.em().insert(entityClass, b as any) + async insert(e: E | E[]): Promise { + return await this.performWrite(async () => { + const entities = Array.isArray(e) ? e : [e] + if (entities.length == 0) return + + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) + this.state.insert(md, entity) } - } else { - let entityClass = e.constructor as EntityClass - await this.changes?.trackInsert(entityClass, [e]) - await this.em().insert(entityClass, e as any) + }) + } + + private async _insert(metadata: EntityMetadata, entities: EntityLiteral[]) { + this.logger?.debug(`insert ${entities.length} ${metadata.name} entities`) + await this.changes?.writeInsert(metadata, entities) + await this.insertMany(metadata.target, entities) + } + + private async insertMany(target: EntityTarget, entities: EntityLiteral[]) { + for (let b of splitIntoBatches(entities, 1000)) { + await this.em.insert(target, b) } } /** * Deletes a given entity or entities from the database. * - * Unlike {@link EntityManager.remove} executes a primitive DELETE query without cascades, relations, etc. + * Executes a primitive DELETE query without cascades, relations, etc. */ - remove(entity: E): Promise - remove(entities: E[]): Promise - remove(entityClass: EntityClass, id: string | string[]): Promise - async remove(e: E | E[] | EntityClass, id?: string | string[]): Promise{ - if (id == null) { - if (Array.isArray(e)) { - if (e.length == 0) return - let entityClass = e[0].constructor as EntityClass - for (let i = 1; i < e.length; i++) { - assert(entityClass === e[i].constructor, 'mass deletion allowed only for entities of the same class') + async delete(e: E | E[]): Promise + async delete(target: EntityTarget, id: string | string[]): Promise + async delete(e: E | E[] | EntityTarget, id?: string | string[]): Promise { + return await this.performWrite(async () => { + if (id == null) { + const entities = Array.isArray(e) ? e : [e as E] + if (entities.length == 0) return + + for (const entity of entities) { + const md = this.getEntityMetadata(entity.constructor) + + this.state.delete(md, entity.id) } - let ids = e.map(i => i.id) - await this.changes?.trackDelete(entityClass, ids) - await this.em().delete(entityClass, ids) } else { - let entity = e as E - let entityClass = entity.constructor as EntityClass - await this.changes?.trackDelete(entityClass, [entity.id]) - await this.em().delete(entityClass, entity.id) + const ids = Array.isArray(id) ? id : [id] + if (ids.length == 0) return + + const md = this.getEntityMetadata(e as EntityTarget) + for (const id of ids) { + this.state.delete(md, id) + } } - } else { - let entityClass = e as EntityClass - await this.changes?.trackDelete(entityClass, Array.isArray(id) ? id : [id]) - await this.em().delete(entityClass, id) - } + }) } - async count(entityClass: EntityClass, options?: FindManyOptions): Promise { - return this.em().count(entityClass, options) + private async _delete(metadata: EntityMetadata, ids: string[]) { + this.logger?.debug(`delete ${metadata.name} ${ids.length} entities`) + await this.changes?.writeDelete(metadata, ids) + await this.em.delete(metadata.target, ids) // NOTE: should be split by chunks too? } - async countBy(entityClass: EntityClass, where: FindOptionsWhere | FindOptionsWhere[]): Promise { - return this.em().countBy(entityClass, where) + async count(target: EntityTarget, options?: FindManyOptions): Promise { + return await this.performRead(async () => { + return await this.em.count(target, options) + }) } - async find(entityClass: EntityClass, options?: FindManyOptions): Promise { - return this.em().find(entityClass, options) + async countBy( + target: EntityTarget, + where: FindOptionsWhere | FindOptionsWhere[] + ): Promise { + return await this.count(target, {where}) } - async findBy(entityClass: EntityClass, where: FindOptionsWhere | FindOptionsWhere[]): Promise { - return this.em().findBy(entityClass, where) + async find(target: EntityTarget, options: FindManyOptions): Promise { + return await this.performRead(async () => { + const {cache, ...opts} = options + + const res = await this.em.find(target, opts) + if (cache ?? this.cacheEntitiesByDefault) { + const metadata = this.getEntityMetadata(target) + for (const e of res) { + this.cacheEntity(metadata, e) + } + } + + return res + }) } - async findOne(entityClass: EntityClass, options: FindOneOptions): Promise { - return this.em().findOne(entityClass, options).then(noNull) + async findBy( + target: EntityTarget, + where: FindOptionsWhere | FindOptionsWhere[], + cache?: boolean + ): Promise { + return await this.find(target, {where, cache}) } - async findOneBy(entityClass: EntityClass, where: FindOptionsWhere | FindOptionsWhere[]): Promise { - return this.em().findOneBy(entityClass, where).then(noNull) + async findOne( + target: EntityTarget, + options: FindOneOptions + ): Promise { + return await this.performRead(async () => { + const {cacheEntities, ...opts} = options + + const res = await this.em.findOne(target, opts).then(noNull) + if (cacheEntities ?? this.cacheEntitiesByDefault) { + const metadata = this.getEntityMetadata(target) + const idOrEntity = res || getIdFromWhere(options.where) + this.cacheEntity(metadata, idOrEntity) + } + + return res + }) } - async findOneOrFail(entityClass: EntityTarget, options: FindOneOptions): Promise { - return this.em().findOneOrFail(entityClass, options) + async findOneBy( + target: EntityTarget, + where: FindOptionsWhere | FindOptionsWhere[], + cacheEntities?: boolean + ): Promise { + return await this.findOne(target, {where, cacheEntities}) } - async findOneByOrFail(entityClass: EntityTarget, where: FindOptionsWhere | FindOptionsWhere[]): Promise { - return this.em().findOneByOrFail(entityClass, where) + async findOneOrFail(target: EntityTarget, options: FindOneOptions): Promise { + const res = await this.findOne(target, options) + if (res == null) throw new EntityNotFoundError(target, options.where) + + return res } - get(entityClass: EntityClass, optionsOrId: FindOneOptions | string): Promise { - if (typeof optionsOrId == 'string') { - return this.findOneBy(entityClass, {id: optionsOrId} as any) - } else { - return this.findOne(entityClass, optionsOrId) + async findOneByOrFail( + target: EntityTarget, + where: FindOptionsWhere | FindOptionsWhere[], + cache?: boolean + ): Promise { + const res = await this.findOneBy(target, where, cache) + if (res == null) throw new EntityNotFoundError(target, where) + + return res + } + + async get(target: EntityTarget, id: string): Promise + async get(target: EntityTarget, options: GetOptions): Promise + async get( + target: EntityTarget, + idOrOptions: string | GetOptions + ): Promise { + const {id, relations} = parseGetOptions(idOrOptions) + const metadata = this.getEntityMetadata(target) + + let entity = this.state.get(metadata, id, relations) + if (entity !== undefined || !this.syncOnGet) return noNull(entity) + + return await this.findOne(target, {where: {id} as any, relations, cacheEntities: true}) + } + + async getOrFail(target: EntityTarget, id: string): Promise + async getOrFail(target: EntityTarget, options: GetOptions): Promise + async getOrFail(target: EntityTarget, idOrOptions: string | GetOptions): Promise { + const options = parseGetOptions(idOrOptions) + + let e = await this.get(target, options) + if (e == null) throw new EntityNotFoundError(target, options.id) + + return e + } + + reset(): void { + this.state.reset() + } + + async sync(reset?: boolean): Promise { + await this.pendingCommit?.promise() + + this.pendingCommit = createFuture() + try { + await this.state.performUpdate(async ({upserts, inserts, deletes, extraUpserts}) => { + for (const {metadata, entities} of upserts) { + await this._upsert(metadata, entities) + } + + for (const {metadata, entities} of inserts) { + await this._insert(metadata, entities) + } + + for (const {metadata, ids} of deletes) { + await this._delete(metadata, ids) + } + + for (const {metadata, entities} of extraUpserts) { + await this._upsert(metadata, entities) + } + }) + + if (reset) { + this.reset() + } + } finally { + this.pendingCommit.resolve() + this.pendingCommit = undefined + } + } + + private async performRead(cb: () => Promise): Promise { + this.assertNotClosed() + await this.sync() + return await cb() + } + + private async performWrite(cb: () => Promise): Promise { + this.assertNotClosed() + await this.pendingCommit?.promise() + await cb() + if (!this.batchWriteOperations) { + await this.sync() } } -} + private assertNotClosed() { + assert(!this.isClosed, `too late to perform db updates, make sure you haven't forgot to await on db query`) + } -function* splitIntoBatches(list: T[], maxBatchSize: number): Generator { - if (list.length <= maxBatchSize) { - yield list - } else { - let offset = 0 - while (list.length - offset > maxBatchSize) { - yield list.slice(offset, offset + maxBatchSize) - offset += maxBatchSize + private cacheEntity(metadata: EntityMetadata, entityOrId?: E | string) { + if (entityOrId == null) { + return + } else if (typeof entityOrId === 'string') { + this.state.settle(metadata, entityOrId) + } else { + traverseEntity(metadata, entityOrId, (e, md) => this.state.persist(md, e)) } - yield list.slice(offset) + } + + private getEntityMetadata(target: EntityTarget) { + return this.em.connection.getMetadata(target) } } +function parseGetOptions(idOrOptions: string | GetOptions): GetOptions { + if (typeof idOrOptions === 'string') { + return {id: idOrOptions} + } else { + return idOrOptions + } +} -function noNull(val: null | undefined | T): T | undefined { - return val == null ? undefined : val +function getIdFromWhere(where?: FindOptionsWhere) { + return typeof where?.id === 'string' ? where.id : undefined } diff --git a/typeorm/typeorm-store/src/test/database.test.ts b/typeorm/typeorm-store/src/test/database.test.ts index 48ffbf1c4..cb2c38613 100644 --- a/typeorm/typeorm-store/src/test/database.test.ts +++ b/typeorm/typeorm-store/src/test/database.test.ts @@ -206,8 +206,8 @@ describe('TypeormDatabase', function() { ] }, async (store, block) => { expect(block).toEqual({height: 2, hash: 'c-2'}) - expect(await store.find(Data)).toEqual([a1]) - await store.remove(a1) + expect(await store.find(Data, {})).toEqual([a1]) + await store.delete(a1) }) expect(await em.find(Data)).toEqual([]) diff --git a/typeorm/typeorm-store/src/test/lib/model.ts b/typeorm/typeorm-store/src/test/lib/model.ts index 8cd68bfc3..20cbfd0ef 100644 --- a/typeorm/typeorm-store/src/test/lib/model.ts +++ b/typeorm/typeorm-store/src/test/lib/model.ts @@ -26,7 +26,7 @@ export class Order { @ManyToOne(() => Item, {nullable: true}) item!: Item - @Column({nullable: false}) + @Column('int4') qty!: number } diff --git a/typeorm/typeorm-store/src/test/store.test.ts b/typeorm/typeorm-store/src/test/store.test.ts index 212f46c73..99eac4689 100644 --- a/typeorm/typeorm-store/src/test/store.test.ts +++ b/typeorm/typeorm-store/src/test/store.test.ts @@ -4,24 +4,26 @@ import {Equal} from 'typeorm' import {Store} from '../store' import {Item, Order} from './lib/model' import {getEntityManager, useDatabase} from './util' +import {sortMetadatasInCommitOrder} from '../utils/commitOrder' +import {StateManager} from '../utils/stateManager' describe("Store", function() { - describe(".save()", function() { + describe(".upsert()", function() { useDatabase([ `CREATE TABLE item (id text primary key , name text)` ]) it("saving of a single entity", async function() { let store = await createStore() - await store.save(new Item('1', 'a')) - await expect(getItems()).resolves.toEqual([{id: '1', name: 'a'}]) + await store.upsert(new Item('1', 'a')) + await expect(getItems(store)).resolves.toEqual([{id: '1', name: 'a'}]) }) it("saving of multiple entities", async function() { let store = await createStore() - await store.save([new Item('1', 'a'), new Item('2', 'b')]) - await expect(getItems()).resolves.toEqual([ + await store.upsert([new Item('1', 'a'), new Item('2', 'b')]) + await expect(getItems(store)).resolves.toEqual([ {id: '1', name: 'a'}, {id: '2', name: 'b'} ]) @@ -33,18 +35,18 @@ describe("Store", function() { for (let i = 0; i < 20000; i++) { items.push(new Item(''+i)) } - await store.save(items) + await store.upsert(items) expect(await store.count(Item)).toEqual(items.length) }) it("updates", async function() { let store = await createStore() - await store.save(new Item('1', 'a')) - await store.save([ + await store.upsert(new Item('1', 'a')) + await store.upsert([ new Item('1', 'foo'), new Item('2', 'b') ]) - await expect(getItems()).resolves.toEqual([ + await expect(getItems(store)).resolves.toEqual([ {id: '1', name: 'foo'}, {id: '2', name: 'b'} ]) @@ -61,29 +63,29 @@ describe("Store", function() { it("removal by passing an entity", async function() { let store = await createStore() - await store.remove(new Item('1')) - await expect(getItemIds()).resolves.toEqual(['2', '3']) + await store.delete(new Item('1')) + await expect(getItemIds(store)).resolves.toEqual(['2', '3']) }) it("removal by passing an array of entities", async function() { let store = await createStore() - await store.remove([ + await store.delete([ new Item('1'), new Item('3') ]) - await expect(getItemIds()).resolves.toEqual(['2']) + await expect(getItemIds(store)).resolves.toEqual(['2']) }) it("removal by passing an id", async function() { let store = await createStore() - await store.remove(Item, '1') - await expect(getItemIds()).resolves.toEqual(['2', '3']) + await store.delete(Item, '1') + await expect(getItemIds(store)).resolves.toEqual(['2', '3']) }) it("removal by passing an array of ids", async function() { let store = await createStore() - await store.remove(Item, ['1', '2']) - await expect(getItemIds()).resolves.toEqual(['3']) + await store.delete(Item, ['1', '2']) + await expect(getItemIds(store)).resolves.toEqual(['3']) }) }) @@ -97,11 +99,11 @@ describe("Store", function() { `INSERT INTO "order" (id, item_id, qty) values ('2', '2', 3)` ]) - it(".save() doesn't clear reference (single row update)", async function() { + it(".upsert() doesn't clear reference (single row update)", async function() { let store = await createStore() let order = assertNotNull(await store.get(Order, '1')) order.qty = 5 - await store.save(order) + await store.upsert(order) let newOrder = await store.findOneOrFail(Order, { where: {id: Equal('1')}, relations: { @@ -112,7 +114,7 @@ describe("Store", function() { expect(newOrder.item.id).toEqual('1') }) - it(".save() doesn't clear reference (multi row update)", async function() { + it(".upsert() doesn't clear reference (multi row update)", async function() { let store = await createStore() let orders = await store.find(Order, {order: {id: 'ASC'}}) let items = await store.find(Item, {order: {id: 'ASC'}}) @@ -120,7 +122,7 @@ describe("Store", function() { orders[0].qty = 5 orders[1].qty = 1 orders[1].item = items[0] - await store.save(orders) + await store.upsert(orders) let newOrders = await store.find(Order, { relations: { @@ -152,19 +154,24 @@ describe("Store", function() { }) -export function createStore(): Promise { - return getEntityManager().then( - em => new Store(() => em) - ) +export async function createStore(): Promise { + const em = await getEntityManager() + return new Store({ + em, + state: new StateManager({ + commitOrder: sortMetadatasInCommitOrder(em.connection), + }), + batchWriteOperations: true, + cacheEntitiesByDefault: true, + syncOnGet: true, + }) } -export async function getItems(): Promise { - let em = await getEntityManager() - return em.find(Item) +export async function getItems(store: Store): Promise { + return store.find(Item, {where: {}}) } - -export function getItemIds(): Promise { - return getItems().then(items => items.map(it => it.id).sort()) +export function getItemIds(store: Store): Promise { + return getItems(store).then((items) => items.map((it) => it.id).sort()) } diff --git a/typeorm/typeorm-store/src/transformers.ts b/typeorm/typeorm-store/src/transformers.ts index 15c3b86f2..0f68d6f41 100644 --- a/typeorm/typeorm-store/src/transformers.ts +++ b/typeorm/typeorm-store/src/transformers.ts @@ -1,3 +1,4 @@ +import {BigDecimal} from '@subsquid/big-decimal' import {ValueTransformer} from 'typeorm' export const bigintTransformer: ValueTransformer = { @@ -18,23 +19,11 @@ export const floatTransformer: ValueTransformer = { }, } -const decimal = { - get BigDecimal(): any { - throw new Error('Package `@subsquid/big-decimal` is not installed') - }, -} - -try { - Object.defineProperty(decimal, 'BigDecimal', { - value: require('@subsquid/big-decimal').BigDecimal, - }) -} catch (e) {} - export const bigdecimalTransformer: ValueTransformer = { to(x?: any) { return x?.toString() }, from(s?: any): any | undefined { - return s == null ? undefined : decimal.BigDecimal(s) + return s == null ? undefined : BigDecimal(s) }, } diff --git a/typeorm/typeorm-store/src/utils/cacheMap.ts b/typeorm/typeorm-store/src/utils/cacheMap.ts new file mode 100644 index 000000000..bdb74bc62 --- /dev/null +++ b/typeorm/typeorm-store/src/utils/cacheMap.ts @@ -0,0 +1,97 @@ +import {EntityMetadata} from 'typeorm' +import {EntityLiteral} from './misc' +import {Logger} from '@subsquid/logger' +import clone from 'fast-copy' + +export class CachedEntity { + constructor(public value: E | null = null) {} +} + +export class CacheMap { + private map: Map> = new Map() + private logger?: Logger + + constructor(logger?: Logger) { + this.logger = logger?.child('cache') + } + + get(metadata: EntityMetadata, id: string) { + return this.getEntityCache(metadata)?.get(id) + } + + has(metadata: EntityMetadata, id: string): boolean { + const cacheMap = this.getEntityCache(metadata) + const cachedEntity = cacheMap.get(id) + return !!cachedEntity?.value + } + + settle(metadata: EntityMetadata, id: string): void { + const cacheMap = this.getEntityCache(metadata) + + if (cacheMap.has(id)) return + + cacheMap.set(id, new CachedEntity()) + this.logger?.debug(`added empty entity ${metadata.name} ${id}`) + } + + delete(metadata: EntityMetadata, id: string): void { + const cacheMap = this.getEntityCache(metadata) + cacheMap.set(id, new CachedEntity()) + this.logger?.debug(`deleted entity ${metadata.name} ${id}`) + } + + clear(): void { + this.logger?.debug(`cleared`) + this.map.clear() + } + + add(metadata: EntityMetadata, entity: E, isNew = false): void { + const cacheMap = this.getEntityCache(metadata) + + let cached = cacheMap.get(entity.id) + if (cached == null) { + cached = new CachedEntity() + cacheMap.set(entity.id, cached) + } + + let cachedEntity = cached.value + if (cachedEntity == null) { + cachedEntity = cached.value = metadata.create() as E + cachedEntity.id = entity.id + this.logger?.debug(`added entity ${metadata.name} ${entity.id}`) + } + + for (const column of metadata.nonVirtualColumns) { + const objectColumnValue = column.getEntityValue(entity) + if (isNew || objectColumnValue !== undefined) { + column.setEntityValue(cachedEntity, clone(objectColumnValue ?? null)) + } + } + + for (const relation of metadata.relations) { + if (!relation.isOwning) continue + + const inverseEntity = relation.getEntityValue(entity) as EntityLiteral | null | undefined + const inverseMetadata = relation.inverseEntityMetadata + + if (inverseEntity != null) { + const mockEntity = inverseMetadata.create() + Object.assign(mockEntity, {id: inverseEntity.id}) + + relation.setEntityValue(cachedEntity, mockEntity) + } else if (isNew || inverseEntity === null) { + relation.setEntityValue(cachedEntity, null) + } + } + } + + private getEntityCache(metadata: EntityMetadata): Map> { + let map = this.map.get(metadata) + if (map == null) { + map = new Map() + this.map.set(metadata, map) + } + + return map as Map> + } +} diff --git a/typeorm/typeorm-store/src/hot.ts b/typeorm/typeorm-store/src/utils/changeWriter.ts similarity index 85% rename from typeorm/typeorm-store/src/hot.ts rename to typeorm/typeorm-store/src/utils/changeWriter.ts index 33030331b..09606ca0e 100644 --- a/typeorm/typeorm-store/src/hot.ts +++ b/typeorm/typeorm-store/src/utils/changeWriter.ts @@ -1,8 +1,7 @@ import {assertNotNull} from '@subsquid/util-internal' -import type {EntityManager, EntityMetadata} from 'typeorm' +import type {EntityManager, EntityMetadata, EntityTarget} from 'typeorm' import {ColumnMetadata} from 'typeorm/metadata/ColumnMetadata' -import {Entity, EntityClass} from './store' - +import {EntityLiteral} from './misc' export interface RowRef { table: string @@ -37,33 +36,30 @@ export interface ChangeRow { } -export class ChangeTracker { +export class ChangeWriter { private index = 0 constructor( - private em: EntityManager, + protected em: EntityManager, private statusSchema: string, private blockHeight: number ) { this.statusSchema = this.escape(this.statusSchema) } - trackInsert(type: EntityClass, entities: Entity[]): Promise { - let meta = this.getEntityMetadata(type) + writeInsert(metadata: EntityMetadata, entities: EntityLiteral[]): Promise { return this.writeChangeRows(entities.map(e => { return { kind: 'insert', - table: meta.tableName, + table: metadata.tableName, id: e.id } })) } - async trackUpsert(type: EntityClass, entities: Entity[]): Promise { - let meta = this.getEntityMetadata(type) - + async writeUpsert(metadata: EntityMetadata, entities: EntityLiteral[]): Promise { let touchedRows = await this.fetchEntities( - meta, + metadata, entities.map(e => e.id) ).then( entities => new Map( @@ -76,35 +72,34 @@ export class ChangeTracker { if (fields) { return { kind: 'update', - table: meta.tableName, + table: metadata.tableName, id: e.id, fields } } else { return { kind: 'insert', - table: meta.tableName, + table: metadata.tableName, id: e.id, } } })) } - async trackDelete(type: EntityClass, ids: string[]): Promise { - let meta = this.getEntityMetadata(type) - let deletedEntities = await this.fetchEntities(meta, ids) + async writeDelete(metadata: EntityMetadata, ids: string[]): Promise { + let deletedEntities = await this.fetchEntities(metadata, ids) return this.writeChangeRows(deletedEntities.map(e => { let {id, ...fields} = e return { kind: 'delete', - table: meta.tableName, + table: metadata.tableName, id: id, fields } })) } - private async fetchEntities(meta: EntityMetadata, ids: string[]): Promise { + private async fetchEntities(meta: EntityMetadata, ids: string[]): Promise { let entities = await this.em.query( `SELECT * FROM ${this.escape(meta.tableName)} WHERE id = ANY($1::text[])`, [ids] @@ -133,7 +128,7 @@ export class ChangeTracker { return entities } - private writeChangeRows(changes: ChangeRecord[]): Promise { + private async writeChangeRows(changes: ChangeRecord[]): Promise { let height = new Array(changes.length) let index = new Array(changes.length) let change = new Array(changes.length) @@ -149,11 +144,7 @@ export class ChangeTracker { sql += ' SELECT block_height, index, change::jsonb' sql += ' FROM unnest($1::int[], $2::int[], $3::text[]) AS i(block_height, index, change)' - return this.em.query(sql, [height, index, change]).then(() => {}) - } - - private getEntityMetadata(type: EntityClass): EntityMetadata { - return this.em.connection.getMetadata(type) + await this.em.query(sql, [height, index, change]) } private escape(name: string): string { diff --git a/typeorm/typeorm-store/src/utils/commitOrder.ts b/typeorm/typeorm-store/src/utils/commitOrder.ts new file mode 100644 index 000000000..84fce502c --- /dev/null +++ b/typeorm/typeorm-store/src/utils/commitOrder.ts @@ -0,0 +1,74 @@ +import {DataSource, EntityMetadata} from 'typeorm' +import {RelationMetadata} from 'typeorm/metadata/RelationMetadata' + +enum NodeState { + Unvisited, + Visiting, + Visited, +} + +const COMMIT_ORDERS: WeakMap = new WeakMap() + +export function sortMetadatasInCommitOrder(connection: DataSource): EntityMetadata[] { + let commitOrder = COMMIT_ORDERS.get(connection) + if (commitOrder != null) return commitOrder + + let states: Map = new Map() + + function visit(node: EntityMetadata) { + if (states.get(node.name) !== NodeState.Unvisited) return + + states.set(node.name, NodeState.Visiting) + + for (let edge of node.relations) { + if (edge.foreignKeys.length === 0) continue + + let target = edge.inverseEntityMetadata + let targetState = states.get(target.name) + + if (targetState === NodeState.Unvisited) { + visit(target) + } else if (targetState === NodeState.Visiting) { + let inverseEdge = target.relations.find((r) => r.inverseEntityMetadata === node) + if (inverseEdge != null) { + let edgeWeight = getWeight(edge) + let inverseEdgeWeight = getWeight(inverseEdge) + + if (edgeWeight > inverseEdgeWeight) { + for (let r of target.relations) { + visit(r.inverseEntityMetadata) + } + + states.set(target.name, NodeState.Visited) + commitOrder?.push(target) + } + } + } + } + + let nodeState = states.get(node.name) + + if (nodeState !== NodeState.Visited) { + states.set(node.name, NodeState.Visited) + commitOrder?.push(node) + } + } + + commitOrder = [] + + for (let node of connection.entityMetadatas) { + if (!states.has(node.name)) { + states.set(node.name, NodeState.Unvisited) + } + + visit(node) + } + + COMMIT_ORDERS.set(connection, commitOrder) + + return commitOrder +} + +function getWeight(edge: RelationMetadata) { + return edge.isNullable ? 0 : 1 +} diff --git a/typeorm/typeorm-store/src/utils/misc.ts b/typeorm/typeorm-store/src/utils/misc.ts new file mode 100644 index 000000000..353fd50b2 --- /dev/null +++ b/typeorm/typeorm-store/src/utils/misc.ts @@ -0,0 +1,68 @@ +import {EntityMetadata, FindOptionsRelations, ObjectLiteral} from 'typeorm' + +export interface EntityLiteral extends ObjectLiteral { + id: string +} + +export function* splitIntoBatches(list: T[], maxBatchSize: number): Generator { + if (list.length <= maxBatchSize) { + yield list + } else { + let offset = 0 + while (list.length - offset > maxBatchSize) { + yield list.slice(offset, offset + maxBatchSize) + offset += maxBatchSize + } + yield list.slice(offset) + } +} + +export function mergeRelations( + a: FindOptionsRelations, + b: FindOptionsRelations +): FindOptionsRelations { + const mergedObject: FindOptionsRelations = {} + + for (const key in a) { + mergedObject[key] = a[key] + } + + for (const key in b) { + const bValue = b[key] + const value = mergedObject[key] + if (typeof bValue === 'object') { + mergedObject[key] = ( + typeof value === 'object' ? mergeRelations(value as any, bValue as any) : bValue + ) as any + } else { + mergedObject[key] = value || bValue + } + } + + return mergedObject +} + +export function traverseEntity( + metadata: EntityMetadata, + entity: EntityLiteral, + cb: (e: EntityLiteral, metadata: EntityMetadata) => void +) { + for (const relation of metadata.relations) { + const inverseEntity = relation.getEntityValue(entity) + if (inverseEntity == null) continue + + if (relation.isOneToMany || relation.isManyToMany) { + for (const ie of inverseEntity) { + traverseEntity(relation.inverseEntityMetadata, ie, cb) + } + } else { + traverseEntity(relation.inverseEntityMetadata, inverseEntity, cb) + } + } + + cb(entity, metadata) +} + +export function noNull(val: null | undefined | T): T | undefined { + return val == null ? undefined : val +} diff --git a/typeorm/typeorm-store/src/utils/stateManager.ts b/typeorm/typeorm-store/src/utils/stateManager.ts new file mode 100644 index 000000000..5386050db --- /dev/null +++ b/typeorm/typeorm-store/src/utils/stateManager.ts @@ -0,0 +1,303 @@ +import {Logger} from '@subsquid/logger' +import {EntityManager, EntityMetadata, FindOptionsRelations} from 'typeorm' +import {CacheMap} from './cacheMap' +import assert from 'assert' +import {EntityLiteral} from './misc' +import {unexpectedCase} from '@subsquid/util-internal' +import clone from 'fast-copy' + +export enum ChangeType { + Insert = 'insert', + Upsert = 'upsert', + Delete = 'delete', +} + +export type ChangeSets = { + upserts: {metadata: EntityMetadata; entities: EntityLiteral[]}[] + inserts: {metadata: EntityMetadata; entities: EntityLiteral[]}[] + deletes: {metadata: EntityMetadata; ids: string[]}[] + extraUpserts: {metadata: EntityMetadata; entities: EntityLiteral[]}[] +} + +export class StateManager { + protected cacheMap: CacheMap + protected stateMap: Map> + protected commitOrder: EntityMetadata[] + protected logger?: Logger + + constructor({commitOrder, logger}: {commitOrder: EntityMetadata[]; logger?: Logger}) { + this.cacheMap = new CacheMap(this.logger) + this.stateMap = new Map() + this.commitOrder = commitOrder + this.logger = logger?.child('state') + } + + get( + metadata: EntityMetadata, + id: string, + relationMask?: FindOptionsRelations + ): E | null | undefined { + const cached = this.cacheMap.get(metadata, id) + + if (cached == null) { + return undefined + } else if (cached.value == null) { + return null + } else { + const entity = cached.value + const clonedEntity = metadata.create() + + for (const column of metadata.nonVirtualColumns) { + const objectColumnValue = column.getEntityValue(entity) + if (objectColumnValue !== undefined) { + column.setEntityValue(clonedEntity, clone(objectColumnValue)) + } + } + + if (relationMask != null) { + for (const relation of metadata.relations) { + const inverseMask = relationMask[relation.propertyName] + if (!inverseMask) continue + + const inverseEntityMock = relation.getEntityValue(entity) as EntityLiteral + + if (inverseEntityMock === null) { + relation.setEntityValue(clonedEntity, null) + } else { + const cachedInverseEntity = + inverseEntityMock != null + ? this.get( + relation.inverseEntityMetadata, + inverseEntityMock.id, + typeof inverseMask === 'boolean' ? undefined : inverseMask + ) + : undefined + + if (cachedInverseEntity === undefined) { + return undefined // unable to build whole relation chain + } else { + relation.setEntityValue(clonedEntity, cachedInverseEntity) + } + } + } + } + + return clonedEntity + } + } + + insert(metadata: EntityMetadata, entity: EntityLiteral): void { + const prevType = this.getState(metadata, entity.id) + switch (prevType) { + case undefined: + this.setState(metadata, entity.id, ChangeType.Insert) + this.cacheMap.add(metadata, entity, true) + break + case ChangeType.Insert: + case ChangeType.Upsert: + throw new Error(`Entity ${metadata.name} ${entity.id} is already marked as ${prevType}`) + case ChangeType.Delete: + this.setState(metadata, entity.id, ChangeType.Upsert) + this.cacheMap.add(metadata, entity, true) + break + default: + throw unexpectedCase(prevType) + } + } + + upsert(metadata: EntityMetadata, entity: EntityLiteral): void { + const prevType = this.getState(metadata, entity.id) + switch (prevType) { + case undefined: + case ChangeType.Insert: + case ChangeType.Upsert: + this.setState(metadata, entity.id, ChangeType.Upsert) + this.cacheMap.add(metadata, entity) + break + case ChangeType.Delete: + this.setState(metadata, entity.id, ChangeType.Upsert) + this.cacheMap.add(metadata, entity, true) + break + default: + throw unexpectedCase(prevType) + } + } + + delete(metadata: EntityMetadata, id: string): void { + const prevType = this.getState(metadata, id) + switch (prevType) { + case undefined: + case ChangeType.Upsert: + this.setState(metadata, id, ChangeType.Delete) + this.cacheMap.delete(metadata, id) + break + case ChangeType.Insert: + this.getChanges(metadata).delete(id) + this.cacheMap.delete(metadata, id) + break + case ChangeType.Delete: + this.logger?.debug(`entity ${metadata.name} ${id} is already marked as ${ChangeType.Delete}`) + break + default: + throw unexpectedCase(prevType) + } + } + + persist(metadata: EntityMetadata, entity: EntityLiteral) { + this.getChanges(metadata).delete(entity.id) // reset state + this.cacheMap.add(metadata, entity) + } + + settle(metadata: EntityMetadata, id: string) { + this.cacheMap.settle(metadata, id) + } + + isInserted(metadata: EntityMetadata, id: string) { + return this.getState(metadata, id) === ChangeType.Insert + } + + isUpserted(metadata: EntityMetadata, id: string) { + return this.getState(metadata, id) === ChangeType.Upsert + } + + isDeleted(metadata: EntityMetadata, id: string) { + return this.getState(metadata, id) === ChangeType.Delete + } + + isExists(metadata: EntityMetadata, id: string) { + return this.cacheMap.has(metadata, id) + } + + reset(): void { + this.logger?.debug(`reset`) + this.stateMap.clear() + this.cacheMap.clear() + } + + async performUpdate(cb: (cs: ChangeSets) => Promise) { + const changeSets: ChangeSets = { + inserts: [], + upserts: [], + deletes: [], + extraUpserts: [], + } + + for (const metadata of this.commitOrder) { + const entityChanges = this.stateMap.get(metadata) + if (entityChanges == null || entityChanges.size == 0) continue + + const inserts: EntityLiteral[] = [] + const upserts: EntityLiteral[] = [] + const deletes: string[] = [] + const extraUpserts: EntityLiteral[] = [] + + for (const [id, type] of entityChanges) { + const cached = this.cacheMap.get(metadata, id) + + switch (type) { + case ChangeType.Insert: { + assert(cached?.value != null, `unable to insert entity ${metadata.name} ${id}`) + + const {entity, extraUpsert} = this.extractExtraUpsert(metadata, cached.value) + inserts.push(entity) + if (extraUpsert != null) { + extraUpserts.push(extraUpsert) + } + + break + } + case ChangeType.Upsert: { + assert(cached?.value != null, `unable to upsert entity ${metadata.name} ${id}`) + + + const {entity, extraUpsert} = this.extractExtraUpsert(metadata, cached.value) + upserts.push(entity) + if (extraUpsert != null) { + extraUpserts.push(extraUpsert) + } + + break + } + case ChangeType.Delete: { + deletes.push(id) + break + } + } + } + + if (upserts.length) { + changeSets.upserts.push({metadata, entities: upserts}) + } + + if (inserts.length) { + changeSets.inserts.push({metadata, entities: inserts}) + } + + if (deletes.length) { + changeSets.deletes.push({metadata, ids: deletes}) + } + + if (extraUpserts.length) { + changeSets.extraUpserts.push({metadata, entities: extraUpserts}) + } + } + + await cb(changeSets) + + this.stateMap.clear() + } + + private extractExtraUpsert(metadata: EntityMetadata, entity: E) { + const commitOrderIndex = this.commitOrder.indexOf(metadata) + + let extraUpsert: E | undefined + for (const relation of metadata.relations) { + if (relation.foreignKeys.length == 0) continue + + const inverseEntity = relation.getEntityValue(entity) + if (inverseEntity == null) continue + + const inverseMetadata = relation.inverseEntityMetadata + if (metadata === inverseMetadata && inverseEntity.id === entity.id) continue + + const invCommitOrderIndex = this.commitOrder.indexOf(inverseMetadata) + if (invCommitOrderIndex < commitOrderIndex) continue + + const isInverseInserted = this.isInserted(inverseMetadata, inverseEntity.id) + if (!isInverseInserted) continue + + if (extraUpsert == null) { + extraUpsert = entity + entity = metadata.create() as E + Object.assign(entity, extraUpsert) + } + + relation.setEntityValue(entity, undefined) + } + + return { + entity, + extraUpsert + } + } + + private setState(metadata: EntityMetadata, id: string, type: ChangeType): this { + this.getChanges(metadata).set(id, type) + this.logger?.debug(`entity ${metadata.name} ${id} marked as ${type}`) + return this + } + + private getState(metadata: EntityMetadata, id: string): ChangeType | undefined { + return this.getChanges(metadata).get(id) + } + + private getChanges(metadata: EntityMetadata): Map { + let map = this.stateMap.get(metadata) + if (map == null) { + map = new Map() + this.stateMap.set(metadata, map) + } + + return map + } +} diff --git a/typeorm/typeorm-store/src/tx.ts b/typeorm/typeorm-store/src/utils/tx.ts similarity index 93% rename from typeorm/typeorm-store/src/tx.ts rename to typeorm/typeorm-store/src/utils/tx.ts index 5c8f204be..d2ae10f4d 100644 --- a/typeorm/typeorm-store/src/tx.ts +++ b/typeorm/typeorm-store/src/utils/tx.ts @@ -1,5 +1,7 @@ import type {DataSource, EntityManager} from "typeorm" -import type {IsolationLevel} from "./database" + + +export type IsolationLevel = 'SERIALIZABLE' | 'READ COMMITTED' | 'REPEATABLE READ' export interface Tx {