feat: sort treasury stuff (#659)

* fix: deduplicate internal transfers

* feat: sort treasury stuff

* chore: ignore nft from treasury

scripts/accountant/sort.py

@@ -5,13 +5,15 @@
 import pandas as pd
 from pony.orm import db_session
 from tqdm import tqdm
+from yearn.entities import deduplicate_internal_transfers
 from yearn.treasury import accountant
 
 pd.set_option('display.max_rows', 1000)
 pd.set_option('display.max_colwidth', None)
 
 @db_session
 def main():
+    deduplicate_internal_transfers()
     txs = accountant.unsorted_txs()
     start_ct, start = len(txs), time()
     accountant.sort_txs(txs)

yearn/entities.py

@@ -553,7 +553,21 @@ class PartnerHarvestEvent(db.Entity):
 
 db.generate_mapping(create_tables=True)
 
-
+@db_session
+def deduplicate_internal_transfers():
+    db.execute(
+        """
+        WITH counted AS (
+            SELECT treasury_tx_id, ROW_NUMBER() over(partition BY CHAIN, TIMESTAMP, block, hash, log_index, token_id, "from", "to", amount, gas_used, gas_price ORDER BY treasury_tx_id ASC) number
+            FROM treasury_txs
+        )
+        DELETE FROM treasury_txs WHERE treasury_tx_id IN (
+            SELECT treasury_tx_id FROM counted WHERE NUMBER > 1
+        )
+        """
+    )
+
+# views
 @db_session
 def create_txgroup_parentage_view() -> None:
     try:

yearn/treasury/_setup.py

@@ -25,6 +25,7 @@
         "0x528Ff33Bf5bf96B5392c10bc4748d9E9Fb5386B2", # PRM
         "0x53fFFB19BAcD44b82e204d036D579E86097E5D09", # BGBG
         "0x57b9d10157f66D8C00a815B5E289a152DeDBE7ed", # 环球股
+        "0x1d41cf24dF81E3134319BC11c308c5589A486166", # Strangers NFT from @marcoworms <3
     },
     Network.Arbitrum: {
         "0x89b0f9dB18FD079063cFA91F174B300C1ce0003C", # AIELON

yearn/treasury/accountant/expenses/__init__.py

@@ -42,7 +42,6 @@
     testing = ux.create_child("Testing", HashMatcher(general.hashes["website"]["ux"]["testing"]).contains)
 
     grants.create_child("Vault Management Dashboard", HashMatcher(["0xc59b692bff01c3a364d9a1fc629dfd743c1e421f7eaee7efdca86f23d0a8a7ad"]).contains) # These is also a stream for these guys TODO figure out how to account for streams as they stream
-    grants.create_child("V3 Development", people.is_v3_team)
     grants.create_child("V3 Support", HashMatcher([["0x213979422ec4154eb0aa0db4b03f48e1669c08fa055ab44e4006fa7d90bb8547", Filter('log_index', 534)]]).contains) # These is also a stream for these guys TODO figure out how to account for streams as they stream
     grants.create_child("Frontend Support", people.is_frontend_support)
     grants.create_child("yGift Team Grant", people.is_ygift_grant)
@@ -65,15 +64,9 @@
     infrastructure_txgroup.create_child("Wonderland Jobs", infrastructure.is_wonderland)
     infrastructure_txgroup.create_child("Unspecified Infra", infrastructure.is_generic)
 
-    # Previously these weren't very granularly categorized but now with the new BR system we can split out each grant
-    grants.create_child("yETH [BR#xxx]", people.is_yeth)
+    # Previously grants weren't very granularly categorized but now with the new BR system we can split out each grant
     grants.create_child("Yearn Exporter [BR#xxx]", people.is_yearn_exporter)
     grants.create_child("Xopowo [BR#xxx]", people.is_xopowo)
-    grants.create_child("S2 Team [BR#xxx]", people.is_s2_team)
-    grants.create_child("yCreative [BR#xxx]", people.is_ycreative)
-    grants.create_child("ySecurity [BR#xxx]", people.is_ysecurity)
-    grants.create_child("Zootroop [BR#xxx]", people.is_zootroop)
-    grants.create_child("Corn [BR#xxx]", people.is_corn)
     grants.create_child("Tapir [BR#xxx]", people.is_tapir)
     grants.create_child("Hipsterman", people.is_hipsterman)
     grants.create_child("Worms", people.is_worms)
@@ -82,3 +75,7 @@
     grants.create_child("ySupport [BR#xxx]", people.is_ysupport)
     grants.create_child("Rantom [BR#xxx]", people.is_rantom)
     grants.create_child("Dinobots [BR#xxx]", people.is_dinobots)
+    grants.create_child("TxCreator [BR#xxx]", people.is_tx_creator)
+
+    for team in people.yteams:
+        grants.create_child(team.txgroup_name, team.is_team_comp)

yearn/treasury/accountant/expenses/people.py

@@ -1,11 +1,80 @@
 
+from typing import List, Optional
 
+from brownie import convert
+from msgspec import UNSET, Struct
 from pony.orm import commit
 
 from yearn.entities import TreasuryTx
 from yearn.treasury.accountant.classes import Filter, HashMatcher, IterFilter
 
 
+class BudgetRequest(Struct):
+    number: int = 0
+    dai: Optional[float] = UNSET
+    yfi: Optional[float] = UNSET
+    # TODO: maybe refactor this somewhere else? its only here for non-comp 
+    usdc: Optional[float] = UNSET
+
+    def __post_init__(self) -> None:
+        if self.number:
+            _budget_requests[self.number] = self
+
+    @property
+    def url(self) -> str:
+        if not self.number:
+            raise ValueError(f"{self}.no is unassigned")
+        return f"https://github.com/yearn/budget/issues/{self.number}"
+
+
+class yTeam(Struct):
+    label: str
+    address: str
+    brs: List[BudgetRequest] = []
+
+    def __post_init__(self) -> None:
+        self.address = convert.to_address(self.address)
+
+    @property
+    def txgroup_name(self) -> str:
+        # TODO: BR numbers
+        return f"{self.label} [BR#xxx]"
+
+    def is_team_comp(self, tx: TreasuryTx) -> bool:
+        token = tx._symbol.lower()
+        brs_for_token = [br for br in self.brs if hasattr(br, token)]
+        if not brs_for_token:
+            return False
+        amount = float(tx.amount)
+        return any(amount == getattr(br, token) for br in brs_for_token) and tx.to_address.address == self.address
+
+
+# TODO: add BR numbers
+yteams = [
+    yTeam(
+        "V3 Development", 
+        "0x3333333368A1ed686392C1534e747F3e15CA286C", 
+        brs=[
+            BudgetRequest(dai=35_000, yfi=1.5),
+            # 2nd request
+            BudgetRequest(dai=40_000, yfi=1.5),
+            # extra fundus # TODO: figure out a better way to handle these
+            BudgetRequest(dai=25_000),
+            BudgetRequest(usdc=65_000),
+        ]
+    ),
+    yTeam("S2 Team", "0x00520049162aa47AdA264E2f77DA6749dfaa6218", brs=[BudgetRequest(dai=39_500, yfi=2.25)]),
+    yTeam("yCreative", "0x402449F51afbFC864D44133A975980179C6cD24C", brs=[BudgetRequest(dai=15_500, yfi=1.65), BudgetRequest(dai=46_500, yfi=4.95)]),
+    yTeam("Corn", "0xF6411852b105042bb8bbc6Dd50C0e8F30Af63337", brs=[BudgetRequest(dai=10_000, yfi=1.5), BudgetRequest(dai=30_000, yfi=4.5)]),
+    yTeam("ySecurity", "0x4851C7C7163bdF04A22C9e12Ab77e184a5dB8F0E", brs=[BudgetRequest(dai=20_667, yfi=2.5)]),
+    yTeam("Zootroop", "0xBd5CA40C66226F53378AE06bc71784CAd6016087", brs=[BudgetRequest(dai=34_500, yfi=1.5), BudgetRequest(dai=36_500, yfi=2)]),
+    yTeam("yETH", "0xeEEEEeeeEe274C3CCe13f77C85d8eBd9F7fd4479", brs=[BudgetRequest(dai=20_000, yfi=4.5)]),
+    yTeam("yLockers", "0xAAAA00079535300dDba84A89eb92434250f83ea7", brs=[BudgetRequest(dai=20_600, yfi=2), BudgetRequest(dai=60_500, yfi=6)]),
+    yTeam("yDiscount", "0x54991866A907891c9B85478CC1Fb0560B17D2b1D", brs=[BudgetRequest(yfi=1)]),
+]
+
+# old
+# TODO refactor all of this
 def is_team_payment(tx: TreasuryTx) -> bool:
     hashes = [
         "0x0704961379ad0ee362c9a2dfd22fdfa7d9a2ed5a94a99dbae1f83853a2d597bc",
@@ -105,6 +174,11 @@ def is_other_grant(tx: TreasuryTx) -> bool:
         "0xa96246a18846cd6966fc87571313a2021d8e1e38466132e2b7fe79acbbd2c589",
         ["0x47bcb48367b5c724780b40a19eed7ba4f623de619e98c30807f52be934d28faf", Filter('_symbol', "DAI")],
         ["0xa0aa1d29e11589350db4df09adc496102a1d9db8a778fc7108d2fb99fb40b6d0", IterFilter("log_index", [391, 394])],
+        "0xf8fe392a865b36b908bd8a4247d7d40a3d8cfef13da9e8bc657aee46bd82f9dd",
+        ["0xfc07ee04d44f8e481f58339b7b8c998d454e4ec427b8021c4e453c8eeee6a9b9", Filter('_symbol', "ETH")],
+        ["0xfc07ee04d44f8e481f58339b7b8c998d454e4ec427b8021c4e453c8eeee6a9b9", Filter('log_index', 218)],
+        ["0x925d77f797779f087a44b2e871160fb194f9cdf949019b5c3c3617f86a0d97fb", IterFilter('log_index', [150, 151])],
+        "0x4bda2a3b21ff999a2ef1cbbaffaa21a713293f8e175adcc4d67e862f0717d6ef",
     ]
     disperse_hashes = [
         "0x1e411c81fc83abfe260f58072c74bfa91e38e27e0066da07ea06f75d9c8f4a00",
@@ -146,14 +220,6 @@ def is_docs_grant(tx: TreasuryTx) -> bool:
     return tx in HashMatcher([
         "0x99f8e351a15e7ce7f0acbae2dea52c56cd93ef97b0a5981f79a68180ff777f00",
     ])
-
-def is_yeth(tx: TreasuryTx) -> bool:
-    if tx.to_address.address == "0xeEEEEeeeEe274C3CCe13f77C85d8eBd9F7fd4479":
-        if tx._symbol == "YFI" and tx.amount == 4.5:
-            return True
-        elif tx._symbol == "DAI" and tx.amount == 20_000:
-            return True
-    return False
 
 def is_yearn_exporter(tx: TreasuryTx) -> bool:
     if tx.to_address.address == "0xcD63C69f08bdDa7Fe96a87A2Ca3f56f3a6990a75":
@@ -166,87 +232,30 @@ def is_yearn_exporter(tx: TreasuryTx) -> bool:
         return True        
     return False
 
-# TODO figure out why these didn't sort
-_xopowo_hashes = [
-    "0xea1ce33495faf84892826ce4d25b9010ef1b035215f38e537164237cff07c459",
-    "0x1969a5ebdedc96057feaa7a156adbdfd2e452868d0bb8258df767f12db26895d",
-    "0xa8eb68338106aa60aba95406ed201b098821884c8c1893c93b400ee994a7facc",
-]
-
 def is_xopowo(tx: TreasuryTx) -> bool:
     if tx.to_address.address == "0x4F909396A75FE9d59F584156A851B3770f3F438a":
-        print(tx.amount)
-        print(5.1)
-        if tx._symbol == "YFI" and (tx.amount == 5.1 or tx.hash in _xopowo_hashes):
-            return True
+        if tx._symbol == "YFI":
+            return float(tx.amount) == 5.1 or tx in HashMatcher([
+                # usual amount chunked in two parts
+                "0x699887b634006c45a5a0ccd66cdc0f5396e82635dd17e7e4e0934014d454d81c",
+                "0xfc07ee04d44f8e481f58339b7b8c998d454e4ec427b8021c4e453c8eeee6a9b9",
+            ])
         elif tx._symbol == "DAI" and tx.amount == 71_500:
             return True
     return False
 
-def is_v3_team(tx: TreasuryTx) -> bool:
-    if tx.to_address.address == "0x3333333368A1ed686392C1534e747F3e15CA286C":
-        return True
-        # make sure this is good
-        if tx._symbol == "YFI" and tx.amount == 1.5:
-            return True
-        elif tx._symbol == "DAI" and tx.amount == 35_000:
-            return True
-    return False
-
-def is_s2_team(tx: TreasuryTx) -> bool:
-    if tx.to_address.address == "0x00520049162aa47AdA264E2f77DA6749dfaa6218":
-        if tx._symbol == "YFI" and tx.amount == 2.25:
-            return True
-        elif tx._symbol == "DAI" and tx.amount == 39_500:
-            return True
-    return False
-
-# TODO figure out why this didn't sort
-_ycreative_hashes = [
-    "0x1969a5ebdedc96057feaa7a156adbdfd2e452868d0bb8258df767f12db26895d",
-    "0xea1ce33495faf84892826ce4d25b9010ef1b035215f38e537164237cff07c459",
-    "0x53cbb8189b9ce895f0fc9b5fb0660f47c406c5d1879e053fe0bec3ff3321676c",
-]
-
-def is_ycreative(tx: TreasuryTx) -> bool:
-    if tx.to_address.address == "0x402449F51afbFC864D44133A975980179C6cD24C":
-        if tx._symbol == "YFI" and (tx.amount == 1.65 or tx.hash in _ycreative_hashes):
-            return True
-        elif tx._symbol == "DAI" and tx.amount == 15_500:
-            return True
-    return False
-
-def is_ysecurity(tx: TreasuryTx) -> bool:
-    if tx.to_address.address == "0x4851C7C7163bdF04A22C9e12Ab77e184a5dB8F0E":
-        if tx._symbol == "YFI" and tx.amount == 2.5:
-            return True
-        elif tx._symbol == "DAI" and tx.amount == 20_667:
-            return True
-    return False
-
-def is_zootroop(tx: TreasuryTx) -> bool:
-    if tx.to_address.address == "0xBd5CA40C66226F53378AE06bc71784CAd6016087":
-        if tx._symbol == "YFI" and tx.amount == 1.5:
-            return True
-        elif tx._symbol == "DAI" and tx.amount == 34_500:
-            return True
-    return False
-
-def is_corn(tx: TreasuryTx) -> bool:
-    if tx.to_address.address == "0xF6411852b105042bb8bbc6Dd50C0e8F30Af63337":
-        if tx._symbol == "YFI" and tx.amount == 1.5:
-            return True
-        elif tx._symbol == "DAI" and tx.amount == 10_000:
-            return True
-    return False
 
 # TODO: Refactor this whole thing
 
 def is_tapir(tx: TreasuryTx) -> bool:
     return tx.to_address.address == "0x80c9aC867b2D36B7e8D74646E074c460a008C0cb" and tx._symbol == "DAI" and tx.amount == 4_000
 
 def is_hipsterman(tx: TreasuryTx) -> bool:
-    return tx.to_address.address == "0xE53D3f2B99FE0Ed6C05977bC0547127836f0D78d" and tx._symbol == "DAI" and tx.amount == 3_500
+    if tx.to_address.address == "0xE53D3f2B99FE0Ed6C05977bC0547127836f0D78d" and tx._symbol == "DAI" and tx.amount in [3_500, 7000]:
+        return True
+    elif tx.hash == "0xfe0ce0947c405f22c99eab63f7e529d95eab4274f2c468deaa1da50adaeb4450":
+        tx.value_usd *= -1
+        return True
 
 def is_worms(tx: TreasuryTx) -> bool:
     return tx.to_address.address == "0xB1d693B77232D88a3C9467eD5619FfE79E80BCCc"
@@ -267,4 +276,10 @@ def is_rantom(tx: TreasuryTx) -> bool:
     return tx.to_address.address == "0x254b42CaCf7290e72e2C84c0337E36E645784Ce1"
 
 def is_tx_creator(tx: TreasuryTx) -> bool:
-    return tx.to_address.address == "0x4007c53A48DefaB0b9D2F05F34df7bd3088B3299"
+    return tx.to_address.address == "0x4007c53A48DefaB0b9D2F05F34df7bd3088B3299"
+
+def is_dinobots(tx: TreasuryTx) -> bool:
+    return tx.token.symbol == "DAI" and tx._from_nickname == "Yearn yChad Multisig" and tx._to_nickname == "yMechs Multisig" and int(tx.amount) == 47_500
+
+# TODO: add rejected BRs
+_budget_requests = {}

yearn/treasury/accountant/expenses/security.py

@@ -17,20 +17,7 @@ def is_yacademy_audit(tx: TreasuryTx) -> bool:
     return tx in HashMatcher(hashes)
 
 def is_chainsec_audit(tx: TreasuryTx) -> bool:
-    return tx in HashMatcher({
-        Network.Mainnet: [
-            "0x7672b9d10b968c58525cff566a60bc8d44a6633f51a712e0eb00ecf88f86aef3",
-            "0x4a77efb6234793b6316e11b6ef7f345f26d3d1b7b8edb8efffe1c0dc4cdfb0e0",
-            ["0x44fdf3172c73b410400718badc7801a7fc496227b5325d90ed840033e16d8366", Filter('log_index', 390)],
-            "0xfdca5bfef0061fa0cea28c04b08ac239f6cb3d708f23548cded80411575ae7ce",
-            ["0xbe95bd4f46e2933953d726a231625852caf8e707bbc16fbda33d20e7ea1f3e6a", Filter('log_index', 310)],
-            "0xea937a5c5298dd5b60c8e0f193798855b2641c64ced0f92b1d9fdef673ae508d",
-            "0x410c8858b77f5810cbf7c3352d8858ec6d2e783ef7dfdefb16da5cf323fe610c",
-            "0x314243dd4b344857456b7aecfd011c9fcd03f93a7ce0cd3ac60f265cfbe3d534",
-            "0x74909f152767fc6223b6308b46085e0da7403078510ca2b751291fac64ca91d2",
-            ["0xa0aa1d29e11589350db4df09adc496102a1d9db8a778fc7108d2fb99fb40b6d0", Filter('log_index', 395)],
-        ],
-    }.get(chain.id, []))
+    return chain.id == Network.Mainnet and tx._symbol in ["USDC", "USDT"] and tx.to_address.address == "0x8bAf5eaF92E37CD9B1FcCD676918A9B3D4F87Dc7"
 
 def is_debaub_audit(tx: TreasuryTx) -> bool:
     return tx in HashMatcher([
@@ -39,7 +26,7 @@ def is_debaub_audit(tx: TreasuryTx) -> bool:
 
 def is_decurity_audit(tx: TreasuryTx) -> bool:
     return tx in HashMatcher([
-        "0x3d1a0b2bb71f3b7118b1a4d5f9e78962809044616ae04659ef383141df678b4f"
+        "0x3d1a0b2bb71f3b7118b1a4d5f9e78962809044616ae04659ef383141df678b4f",
     ])
 
 def is_statemind_audit(tx: TreasuryTx) -> bool:

yearn/treasury/accountant/ignore/__init__.py

@@ -160,6 +160,7 @@ def is_keep_crv(tx: TreasuryTx) -> bool:
     ignore_txgroup.create_child("Testing with contributor funds", general.is_ycrv_for_testing)
     ignore_txgroup.create_child("Deploy Vesting Package", general.is_vest_factory)
     ignore_txgroup.create_child("Ignore yMechs", general.is_ignore_ymechs)
+    ignore_txgroup.create_child("Maker DSR", maker.is_dsr)
 elif chain.id == Network.Fantom:
     ignore_txgroup.create_child("OTCTrader", general.is_otc_trader)
 

yearn/treasury/accountant/ignore/maker.py

@@ -25,4 +25,8 @@ def is_dai(tx: TreasuryTx) -> bool:
         if tx.from_address.address == ZERO_ADDRESS:
             return True
         elif tx.to_address.address == "0xd42e1Cb8b98382df7Db43e0F09dFE57365659D16": # DSProxy
-            return True
+            return True
+
+def is_dsr(tx: TreasuryTx) -> bool:
+    """sending DAI to or receiving DAI back from Maker's DSR module"""
+    return "Contract: DsrManager" in [tx._to_nickname, tx._from_nickname]

yearn/treasury/accountant/other_expenses/__init__.py

@@ -30,6 +30,7 @@
     other_expense_txgroup.create_child("Warroom Games 2023 Prizes", general.is_warroom_games)
     other_expense_txgroup.create_child("yfi.eth", general.is_yfi_dot_eth)
     other_expense_txgroup.create_child("Fund Vyper Compiler Audit Context", general.is_yyper_contest)
+    other_expense_txgroup.create_child("Reimburse yETH Applications", general.is_reimburse_yeth_applications)
 
 # Bugs
 if chain.id == Network.Mainnet:

yearn/treasury/accountant/other_expenses/general.py

@@ -113,4 +113,7 @@ def is_yfi_dot_eth(tx: TreasuryTx) -> bool:
 
 def is_yyper_contest(tx: TreasuryTx) -> bool:
     """Grant for a vyper compiler audit context, vyper-context.eth"""
-    return tx in HashMatcher([["0xb8bb3728fdfb49d7c86c08dba8e3586e3761f13d2c88fa6fab80227b6a3f4519", Filter('log_index', 202)]])
+    return tx in HashMatcher([["0xb8bb3728fdfb49d7c86c08dba8e3586e3761f13d2c88fa6fab80227b6a3f4519", Filter('log_index', 202)]])
+
+def is_reimburse_yeth_applications(tx: TreasuryTx) -> bool:
+    return tx in HashMatcher([["0x846d475425a1a70469b8674b6f15568c83a14ed3251cafa006811722af676f44", Filter('_symbol', 'ETH')]])

yearn/treasury/accountant/revenue/__init__.py

@@ -3,7 +3,7 @@
 from y.networks import Network
 
 from yearn.treasury.accountant.classes import TopLevelTxGroup
-from yearn.treasury.accountant.revenue import (bribes, farming, fees,
+from yearn.treasury.accountant.revenue import (bribes, dinobots, farming, fees,
                                                keepcoins, seasolver, yacademy)
 
 REVENUE_LABEL = "Protocol Revenue"
@@ -23,6 +23,8 @@
 seasolver_txgroup.create_child("Positive Slippage", seasolver.is_seasolver_slippage_revenue)
 seasolver_txgroup.create_child("CowSwap Incentives", seasolver.is_cowswap_incentive)
 
+dinobots_txgroup = revenue_txgroup.create_child("Dinobots", dinobots.is_dinobots_rev)
+
 bribes_txgroup = revenue_txgroup.create_child("Bribes")
 bribes_txgroup.create_child("yCRV Bribes", check=bribes.is_ycrv_bribe)
 bribes_txgroup.create_child("yBribe Fees", check=bribes.is_ybribe_fees)

yearn/treasury/accountant/revenue/dinobots.py

@@ -0,0 +1,5 @@
+
+from yearn.entities import TreasuryTx
+
+def is_dinobots_rev(tx: TreasuryTx) -> bool:
+    return tx.hash in ["0xf1ce47dc8a44fc37ffc580960f22bbe23896cf31bb0484f0cd5d23fc53c4d5fe", "0xa09e8ed14967023a672d332eb620204b20d63c574bc1b7575fc710cdfb794271"]