Integrate on-chain NAV data into historical hash price calculations (#…

…555) * seperate processing logic into tokenservice * refactor processing functions * use flow api source in calculating historical data, remove legacy osmosis only fetchers * fix lints * fix test to not use legacy fetcher call, remove protected * remove old test, add source checks * reorder asserts in test * fix percent change to return predicable decimal places, add tests * fix lints * add test, fix query to limit 1 * add change log entry * change changelog entry description
provenance-io · Oct 3, 2024 · 7f16bcb · 7f16bcb
1 parent db73ab3
commit 7f16bcb
Show file tree

Hide file tree

Showing 14 changed files with 285 additions and 101 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -35,6 +35,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ### Features
 * Add hash price support for nav values from on chain events [#543](https://github.com/provenance-io/explorer-service/pull/543)
+* Integrate on-chain NAV data into historical hash price calculations [#555](https://github.com/provenance-io/explorer-service/pull/555) 
 
 ### Improvements
 

diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt b/service/src/main/kotlin/io/provenance/explorer/domain/entities/ExplorerCache.kt
@@ -269,11 +269,13 @@ class TokenHistoricalDailyRecord(id: EntityID<DateTime>) : Entity<DateTime>(id)
             TokenHistoricalDailyRecord.wrapRows(query).map { it.data }.toList()
         }
 
-        fun lastKnownPriceForDate(date: DateTime) = transaction {
+        fun lastKnownPriceForDate(date: DateTime): BigDecimal = transaction {
             TokenHistoricalDailyRecord
                 .find { TokenHistoricalDailyTable.timestamp lessEq date }
                 .orderBy(Pair(TokenHistoricalDailyTable.timestamp, SortOrder.DESC))
-                .firstOrNull()?.data?.quote?.get(USD_UPPER)?.close ?: BigDecimal.ZERO
+                .limit(1)
+                .map { it.data.quote[USD_UPPER]?.close ?: BigDecimal.ZERO }
+                .singleOrNull() ?: BigDecimal.ZERO
         }
 
         fun getLatestDateEntry(): TokenHistoricalDailyRecord? = transaction {

diff --git a/service/src/main/kotlin/io/provenance/explorer/domain/extensions/CoinExtensions.kt b/service/src/main/kotlin/io/provenance/explorer/domain/extensions/CoinExtensions.kt
@@ -48,8 +48,13 @@ fun Double.toPercentage() = "${this * 100}%"
 
 fun List<Int>.avg() = this.sum() / this.size
 
-fun BigDecimal.percentChange(orig: BigDecimal) =
-    ((this.minus(orig)).divide(orig, orig.scale(), RoundingMode.HALF_EVEN)).multiply(BigDecimal(100))
+fun BigDecimal.percentChange(orig: BigDecimal): BigDecimal {
+    if (orig.compareTo(BigDecimal.ZERO) == 0) return BigDecimal.ZERO
+    return this.subtract(orig)
+        .divide(orig, 6, RoundingMode.HALF_EVEN)
+        .multiply(BigDecimal(100))
+        .setScale(1, RoundingMode.HALF_EVEN)
+}
 
 fun Int.padToDecString() =
     BigDecimal(this).multiply(BigDecimal("1e${(UTILITY_TOKEN_BASE_DECIMAL_PLACES - 1) * 2}")).toPlainString()

diff --git a/...ce/src/main/kotlin/io/provenance/explorer/domain/models/explorer/HistoricalPriceModels.kt b/...ce/src/main/kotlin/io/provenance/explorer/domain/models/explorer/HistoricalPriceModels.kt
@@ -8,7 +8,8 @@ data class HistoricalPrice(
     val low: BigDecimal,
     val close: BigDecimal,
     val open: BigDecimal,
-    val volume: BigDecimal
+    val volume: BigDecimal,
+    val source: String
 )
 
 fun HistoricalPrice.toCsv(): List<String> = listOf(

diff --git a/service/src/main/kotlin/io/provenance/explorer/grpc/flow/FlowApiGrpcClient.kt b/service/src/main/kotlin/io/provenance/explorer/grpc/flow/FlowApiGrpcClient.kt
@@ -48,7 +48,7 @@ class FlowApiGrpcClient(flowApiChannelUri: URI) {
         priceDenoms.forEach { requestBuilder.addPriceDenoms(it) }
         val request = requestBuilder.build()
         try {
-            logger().info("getMarkerNavByPriceDenoms $request")
+            logger().debug("getMarkerNavByPriceDenoms $request")
             val response: NavEventResponse = navService.getNavEvents(request)
             return@runBlocking response.navEventsList
         } catch (e: Exception) {

diff --git a/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt b/service/src/main/kotlin/io/provenance/explorer/service/TokenService.kt
@@ -32,8 +32,10 @@ import io.provenance.explorer.domain.models.explorer.TokenHistoricalDataRequest
 import io.provenance.explorer.domain.models.toCsv
 import io.provenance.explorer.grpc.v1.AccountGrpcClient
 import io.provenance.explorer.model.AssetHolder
+import io.provenance.explorer.model.CmcHistoricalQuote
 import io.provenance.explorer.model.CmcLatestDataAbbrev
 import io.provenance.explorer.model.CmcLatestQuoteAbbrev
+import io.provenance.explorer.model.CmcQuote
 import io.provenance.explorer.model.RichAccount
 import io.provenance.explorer.model.TokenDistribution
 import io.provenance.explorer.model.TokenDistributionAmount
@@ -51,6 +53,7 @@ import kotlinx.coroutines.flow.toList
 import kotlinx.coroutines.runBlocking
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.joda.time.DateTime
+import org.joda.time.Interval
 import org.springframework.stereotype.Service
 import java.math.BigDecimal
 import java.math.RoundingMode
@@ -65,12 +68,8 @@ class TokenService(
 ) {
     protected val logger = logger(TokenService::class)
 
-    private val historicalPriceFetchers: List<HistoricalPriceFetcher> by lazy {
-        historicalPriceFetcherFactory.createNhashFetchers()
-    }
-
-    private val deprecatedHistoricalPricingFetchers: List<HistoricalPriceFetcher> by lazy {
-        historicalPriceFetcherFactory.createOsmosisPriceFetcher()
+    protected val historicalPriceFetchers: List<HistoricalPriceFetcher> by lazy {
+        historicalPriceFetcherFactory.createNhashPricingFetchers()
     }
 
     fun getTokenDistributionStats() = transaction { TokenDistributionAmountsRecord.getStats() }
@@ -245,11 +244,67 @@ class TokenService(
         return@runBlocking allPrices
     }
 
-    fun fetchLegacyHistoricalPriceData(fromDate: DateTime?): List<HistoricalPrice> = runBlocking {
-        val allPrices = deprecatedHistoricalPricingFetchers.flatMap { fetcher ->
-            fetcher.fetchHistoricalPrice(fromDate)
+    fun processHistoricalData(startDate: DateTime, today: DateTime, historicalPrices: List<HistoricalPrice>): List<CmcHistoricalQuote> {
+        val baseMap = Interval(startDate, today)
+            .let { int -> generateSequence(int.start) { dt -> dt.plusDays(1) }.takeWhile { dt -> dt < int.end } }
+            .map { it to emptyList<HistoricalPrice>() }.toMap().toMutableMap()
+
+        var prevPrice = TokenHistoricalDailyRecord.lastKnownPriceForDate(startDate)
+
+        baseMap.putAll(
+            historicalPrices
+                .filter { DateTime(it.time * 1000).startOfDay() != today }
+                .groupBy { DateTime(it.time * 1000).startOfDay() }
+        )
+
+        return baseMap.map { (k, v) ->
+            val high = v.maxByOrNull { it.high.toThirdDecimal() }
+            val low = v.minByOrNull { it.low.toThirdDecimal() }
+            val open = v.minByOrNull { DateTime(it.time * 1000) }?.open ?: prevPrice
+            val close = v.maxByOrNull { DateTime(it.time * 1000) }?.close ?: prevPrice
+            val closeDate = k.plusDays(1).minusMillis(1)
+            val usdVolume = v.sumOf { it.volume.toThirdDecimal() }.stripTrailingZeros()
+            CmcHistoricalQuote(
+                time_open = k,
+                time_close = closeDate,
+                time_high = if (high != null) DateTime(high.time * 1000) else k,
+                time_low = if (low != null) DateTime(low.time * 1000) else k,
+                quote = mapOf(
+                    USD_UPPER to
+                        CmcQuote(
+                            open = open,
+                            high = high?.high ?: prevPrice,
+                            low = low?.low ?: prevPrice,
+                            close = close,
+                            volume = usdVolume,
+                            market_cap = close.multiply(
+                                totalSupply().divide(UTILITY_TOKEN_BASE_MULTIPLIER)
+                            ).toThirdDecimal(),
+                            timestamp = closeDate
+                        )
+                )
+            ).also { prevPrice = close }
+        }
+    }
+
+    fun updateAndSaveTokenHistoricalData(startDate: DateTime, endDate: DateTime) {
+        val historicalPrices = fetchHistoricalPriceData(startDate) ?: return
+        val processedData = processHistoricalData(startDate, endDate, historicalPrices)
+        processedData.forEach { record ->
+            val source = historicalPriceFetchers.joinToString(separator = ",") { it.getSource() }
+            TokenHistoricalDailyRecord.save(record.time_open.startOfDay(), record, source)
+        }
+    }
+
+    fun updateAndSaveLatestTokenData(startDate: DateTime, today: DateTime) {
+        val list = fetchHistoricalPriceData(startDate)?.sortedBy { it.time }
+
+        list?.let {
+            val latestData = processLatestTokenData(it, today)
+            latestData?.let { data ->
+                cacheLatestTokenData(data)
+            }
         }
-        return@runBlocking allPrices
     }
 
     fun processLatestTokenData(list: List<HistoricalPrice>, today: DateTime): CmcLatestDataAbbrev? {
@@ -265,7 +320,7 @@ class TokenService(
         )
     }
 
-    fun cacheLatestTokenData(data: CmcLatestDataAbbrev) {
+    protected fun cacheLatestTokenData(data: CmcLatestDataAbbrev) {
         CacheUpdateRecord.updateCacheByKey(
             CacheKeys.UTILITY_TOKEN_LATEST.key,
             VANILLA_MAPPER.writeValueAsString(data)
@@ -277,7 +332,7 @@ class TokenService(
         val baseFileName = filters.getFileNameBase()
 
         val fileList = runBlocking {
-            val data = fetchLegacyHistoricalPriceData(filters.fromDate)
+            val data = fetchHistoricalPriceData(filters.fromDate)
             listOf(
                 CsvData(
                     "TokenHistoricalData",

diff --git a/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt b/service/src/main/kotlin/io/provenance/explorer/service/async/ScheduledTaskService.kt
@@ -36,12 +36,8 @@ import io.provenance.explorer.domain.extensions.height
 import io.provenance.explorer.domain.extensions.monthToQuarter
 import io.provenance.explorer.domain.extensions.startOfDay
 import io.provenance.explorer.domain.extensions.toDateTime
-import io.provenance.explorer.domain.extensions.toThirdDecimal
-import io.provenance.explorer.domain.models.HistoricalPrice
 import io.provenance.explorer.grpc.extensions.getMsgSubTypes
 import io.provenance.explorer.grpc.extensions.getMsgType
-import io.provenance.explorer.model.CmcHistoricalQuote
-import io.provenance.explorer.model.CmcQuote
 import io.provenance.explorer.model.base.USD_UPPER
 import io.provenance.explorer.service.AccountService
 import io.provenance.explorer.service.AssetService
@@ -65,7 +61,6 @@ import org.jetbrains.exposed.sql.select
 import org.jetbrains.exposed.sql.transactions.transaction
 import org.joda.time.DateTime
 import org.joda.time.DateTimeZone
-import org.joda.time.Interval
 import org.joda.time.LocalDate
 import org.springframework.scheduling.annotation.Scheduled
 import org.springframework.stereotype.Service
@@ -296,67 +291,19 @@ class ScheduledTaskService(
     @Scheduled(cron = "0 0 1 * * ?") // Every day at 1 am
     fun updateTokenHistorical() {
         val today = DateTime.now().startOfDay()
-        var startDate = today.minusMonths(1)
-        val latest = TokenHistoricalDailyRecord.getLatestDateEntry()
-        if (latest != null) {
-            startDate = latest.timestamp.minusDays(1).startOfDay()
-        }
-        val dlobRes = tokenService.fetchLegacyHistoricalPriceData(startDate) ?: return
-        logger.info("Updating token historical data starting from $startDate with ${dlobRes.size} buy records for roll-up.")
-
-        val baseMap = Interval(startDate, today)
-            .let { int -> generateSequence(int.start) { dt -> dt.plusDays(1) }.takeWhile { dt -> dt < int.end } }
-            .map { it to emptyList<HistoricalPrice>() }.toMap().toMutableMap()
-        var prevPrice = TokenHistoricalDailyRecord.lastKnownPriceForDate(startDate)
-
-        baseMap.putAll(
-            dlobRes
-                .filter { DateTime(it.time * 1000).startOfDay() != today }
-                .groupBy { DateTime(it.time * 1000).startOfDay() }
-        )
-        baseMap.forEach { (k, v) ->
-            val high = v.maxByOrNull { it.high.toThirdDecimal() }
-            val low = v.minByOrNull { it.low.toThirdDecimal() }
-            val open = v.minByOrNull { DateTime(it.time * 1000) }?.open ?: prevPrice
-            val close = v.maxByOrNull { DateTime(it.time * 1000) }?.close ?: prevPrice
-            val closeDate = k.plusDays(1).minusMillis(1)
-            val usdVolume = v.sumOf { it.volume.toThirdDecimal() }.stripTrailingZeros()
-            val record = CmcHistoricalQuote(
-                time_open = k,
-                time_close = closeDate,
-                time_high = if (high != null) DateTime(high.time * 1000) else k,
-                time_low = if (low != null) DateTime(low.time * 1000) else k,
-                quote = mapOf(
-                    USD_UPPER to
-                        CmcQuote(
-                            open = open,
-                            high = high?.high ?: prevPrice,
-                            low = low?.low ?: prevPrice,
-                            close = close,
-                            volume = usdVolume,
-                            market_cap = close.multiply(
-                                tokenService.totalSupply().divide(UTILITY_TOKEN_BASE_MULTIPLIER)
-                            ).toThirdDecimal(),
-                            timestamp = closeDate
-                        )
-                )
-            ).also { prevPrice = close }
-            TokenHistoricalDailyRecord.save(record.time_open.startOfDay(), record, "osmosis")
-        }
+        val defaultStartDate = today.minusMonths(1)
+
+        val latestEntryDate = TokenHistoricalDailyRecord.getLatestDateEntry()?.timestamp?.startOfDay()
+        val startDate = latestEntryDate?.minusDays(1) ?: defaultStartDate
+
+        tokenService.updateAndSaveTokenHistoricalData(startDate, today)
     }
 
     @Scheduled(cron = "0 0/5 * * * ?") // Every 5 minutes
     fun updateTokenLatest() {
         val today = DateTime.now().withZone(DateTimeZone.UTC)
         val startDate = today.minusDays(1)
-        val list = tokenService.fetchHistoricalPriceData(startDate)?.sortedBy { it.time }
-
-        list?.let {
-            val latestData = tokenService.processLatestTokenData(it, today)
-            latestData?.let { data ->
-                tokenService.cacheLatestTokenData(data)
-            }
-        }
+        tokenService.updateAndSaveLatestTokenData(startDate, today)
     }
 
     // Remove once the ranges have been updated

diff --git a/...ce/src/main/kotlin/io/provenance/explorer/service/pricing/fetchers/FlowApiPriceFetcher.kt b/...ce/src/main/kotlin/io/provenance/explorer/service/pricing/fetchers/FlowApiPriceFetcher.kt
@@ -14,6 +14,9 @@ class FlowApiPriceFetcher(
     private val flowApiGrpcClient: FlowApiGrpcClient
 ) : HistoricalPriceFetcher {
 
+    override fun getSource(): String {
+        return "flow-api"
+    }
     override fun fetchHistoricalPrice(fromDate: DateTime?): List<HistoricalPrice> {
         val onChainNavEvents = getMarkerNavByPriceDenoms(fromDate, 17800)
         return onChainNavEvents.map { navEvent ->
@@ -25,7 +28,8 @@ class FlowApiPriceFetcher(
                 low = pricePerHash,
                 close = pricePerHash,
                 open = pricePerHash,
-                volume = pricePerHash.multiply(volumeHash)
+                volume = pricePerHash.multiply(volumeHash),
+                source = getSource()
             )
         }
     }

diff --git a/...src/main/kotlin/io/provenance/explorer/service/pricing/fetchers/HistoricalPriceFetcher.kt b/...src/main/kotlin/io/provenance/explorer/service/pricing/fetchers/HistoricalPriceFetcher.kt
@@ -4,5 +4,7 @@ import io.provenance.explorer.domain.models.HistoricalPrice
 import org.joda.time.DateTime
 
 interface HistoricalPriceFetcher {
+
+    fun getSource(): String
     fun fetchHistoricalPrice(fromDate: DateTime?): List<HistoricalPrice>
 }
diff --git a/...n/kotlin/io/provenance/explorer/service/pricing/fetchers/HistoricalPriceFetcherFactory.kt b/...n/kotlin/io/provenance/explorer/service/pricing/fetchers/HistoricalPriceFetcherFactory.kt
@@ -6,16 +6,10 @@ import io.provenance.explorer.grpc.flow.FlowApiGrpcClient
 class HistoricalPriceFetcherFactory(
     private val flowApiGrpcClient: FlowApiGrpcClient
 ) {
-    fun createNhashFetchers(): List<HistoricalPriceFetcher> {
+    fun createNhashPricingFetchers(): List<HistoricalPriceFetcher> {
         return listOf(
             OsmosisPriceFetcher(),
             FlowApiPriceFetcher(UTILITY_TOKEN, listOf("uusd.trading", "uusdc.figure.se", "uusdt.figure.se"), flowApiGrpcClient)
         )
     }
-
-    fun createOsmosisPriceFetcher(): List<HistoricalPriceFetcher> {
-        return listOf(
-            OsmosisPriceFetcher()
-        )
-    }
 }
diff --git a/...ce/src/main/kotlin/io/provenance/explorer/service/pricing/fetchers/OsmosisPriceFetcher.kt b/...ce/src/main/kotlin/io/provenance/explorer/service/pricing/fetchers/OsmosisPriceFetcher.kt
@@ -21,6 +21,11 @@ import java.net.URLEncoder
 class OsmosisPriceFetcher : HistoricalPriceFetcher {
 
     val logger = logger(OsmosisPriceFetcher::class)
+
+    override fun getSource(): String {
+        return "osmosis"
+    }
+
     override fun fetchHistoricalPrice(fromDate: DateTime?): List<HistoricalPrice> {
         val osmosisHistoricalPrices = fetchOsmosisData(fromDate)
         return osmosisHistoricalPrices.map { osmosisPrice ->
@@ -30,7 +35,8 @@ class OsmosisPriceFetcher : HistoricalPriceFetcher {
                 low = osmosisPrice.low,
                 close = osmosisPrice.close,
                 open = osmosisPrice.open,
-                volume = osmosisPrice.volume
+                volume = osmosisPrice.volume,
+                source = getSource()
             )
         }
     }

diff --git a/service/src/test/kotlin/io/provenance/explorer/domain/extensions/CoinExtensionsKtTest.kt b/service/src/test/kotlin/io/provenance/explorer/domain/extensions/CoinExtensionsKtTest.kt
@@ -0,0 +1,29 @@
+package io.provenance.explorer.domain.extensions
+
+import org.junit.jupiter.api.Assertions.assertEquals
+import org.junit.jupiter.api.Test
+import java.math.BigDecimal
+
+class CoinExtensionsKtTest {
+
+    @Test
+    fun `test percentChange extension`() {
+        val increaseFrom100To120 = BigDecimal("120").percentChange(BigDecimal("100"))
+        assertEquals(BigDecimal("20.0"), increaseFrom100To120, "Percent change calculation is incorrect")
+
+        val decreaseFrom100To80 = BigDecimal("80").percentChange(BigDecimal("100"))
+        assertEquals(BigDecimal("-20.0"), decreaseFrom100To80, "Percent change calculation is incorrect")
+
+        val noChangeAt100 = BigDecimal("100").percentChange(BigDecimal("100"))
+        assertEquals(BigDecimal("0.0"), noChangeAt100, "Percent change calculation is incorrect")
+
+        val smallIncreaseFrom100To100_01 = BigDecimal("100.01").percentChange(BigDecimal("100"))
+        assertEquals(BigDecimal("0.0"), smallIncreaseFrom100To100_01, "Percent change calculation is incorrect")
+
+        val rounding = BigDecimal("1.600").percentChange(BigDecimal("1.4"))
+        assertEquals(BigDecimal("14.3"), rounding, "Percent change calculation is incorrect")
+
+        val divisionByZero = BigDecimal("100").percentChange(BigDecimal("0"))
+        assertEquals(BigDecimal.ZERO, divisionByZero, "Percent change calculation is incorrect when dividing by zero")
+    }
+}