From c8932a6b321ed6419f0553106a599d81e3cb2382 Mon Sep 17 00:00:00 2001 From: LexLuthr Date: Wed, 2 Oct 2024 21:04:29 +0400 Subject: [PATCH] add indexer remove all --- api/api.go | 1 + api/proxy_gen.go | 13 +++++++ build/openrpc/boost.json.gz | Bin 3229 -> 3292 bytes cmd/boostd/index.go | 27 ++++++++++++++ cmd/migrate-curio/ipni.go | 49 +++++++++++++++++++++++++ documentation/en/api-v1-methods.md | 17 ++++++++- indexprovider/wrapper.go | 57 ++++++++++++++++++++++++++--- node/impl/boost.go | 4 ++ 8 files changed, 162 insertions(+), 6 deletions(-) create mode 100644 cmd/migrate-curio/ipni.go diff --git a/api/api.go b/api/api.go index 91f4fd425..6f7a0b943 100644 --- a/api/api.go +++ b/api/api.go @@ -26,6 +26,7 @@ type Boost interface { Net // MethodGroup: Boost + BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) //perm:admin BoostIndexerAnnounceAllDeals(ctx context.Context) error //perm:admin BoostIndexerListMultihashes(ctx context.Context, contextID []byte) ([]multihash.Multihash, error) //perm:admin BoostIndexerAnnounceLatest(ctx context.Context) (cid.Cid, error) //perm:admin diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 8f3ce38a0..b98bf78bb 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -60,6 +60,8 @@ type BoostStruct struct { BoostIndexerListMultihashes func(p0 context.Context, p1 []byte) ([]multihash.Multihash, error) `perm:"admin"` + BoostIndexerRemoveAll func(p0 context.Context) ([]cid.Cid, error) `perm:"admin"` + BoostLegacyDealByProposalCid func(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) `perm:"admin"` BoostOfflineDealWithData func(p0 context.Context, p1 uuid.UUID, p2 string, p3 bool) (*ProviderDealRejectionInfo, error) `perm:"admin"` @@ -355,6 +357,17 @@ func (s *BoostStub) BoostIndexerListMultihashes(p0 context.Context, p1 []byte) ( return *new([]multihash.Multihash), ErrNotSupported } +func (s *BoostStruct) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) { + if s.Internal.BoostIndexerRemoveAll == nil { + return *new([]cid.Cid), ErrNotSupported + } + return s.Internal.BoostIndexerRemoveAll(p0) +} + +func (s *BoostStub) BoostIndexerRemoveAll(p0 context.Context) ([]cid.Cid, error) { + return *new([]cid.Cid), ErrNotSupported +} + func (s *BoostStruct) BoostLegacyDealByProposalCid(p0 context.Context, p1 cid.Cid) (legacytypes.MinerDeal, error) { if s.Internal.BoostLegacyDealByProposalCid == nil { return *new(legacytypes.MinerDeal), ErrNotSupported diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index ba155d660c236bdf109eb97a5ab34dcce3787a36..e51ae0ab4c6d0ba3f84331a9d563759eb00621a9 100644 GIT binary patch literal 3292 zcmV<23?uU&iwFP!00000|Li?$Q`usvJuDK zdn~E60eA5M=bl7ogG1CC^s#|&L3ori+{Nuir_sby13Mtc-12bu@*Xqds8t`S4JN0o zabR=v#*~{7I>zht_(qZ`}Xace`O<|nP|&HZ-ULvrSMI;jpmkaxlLqpCgDyt z4#WQ&$Za8$T8%?$ZKJKrtKmd~fj|f@w$c4p_Epjm@)~_6K`>&^SfAR+)UF3|)eB5Z<8?l{1theli=Ii?;8iQ2#hc8MTPNEBZv;tnS}F863a!|3Zd z;=8R3p(8TQVWK{mV{l0Nx&zuWiTV~WNk^?LL|fm-)EpzqENT)7Y7fV^B{kKz zJ1#MAz?$tMCoXIw!YoA4shTw=-dNEM?ug^sph&}f+rBcenu@!Z*pq^?YkV~|uz-Jj zD!{^BDSS{zbp@zRE!-Uu+k@$qn&kPmRAh=M9CAp8GPB_vBBwCIe=)oH_xZutWWS8z z|KGpItKz2_Q)dqu^N5?C+Ib$&PgIWmArjzl3CO1G1{>HHHtyoFl&;s^-MtQ`G)(U7 zjv`s!aRvV`m~wZhq;z*^^Wm;TJPBf#xOCSQ{BH7}YhaCQ{(mR!=02WIjRJ*mPXEnP zvAj}H%Zn8QWwNwZP!RQ+p}Dd;h_K983AKzrMH#0oYp=qw0`fB78EQ!SIT%)SA!s%w zg@3EvNueLjc-AVQ^|nX_920L<@joVBvFHyux68@?G4b?rA5wEA*$+{y<^Flez2cb+ zU82EayRLRhVWZXoVb5ER+dt>Xv08itBulR4G8`|Cz$Ae45s z0zr$Y7lOrIJZiSuojq*e_|)mK6EXxlJoC9ELVj?$IaXj>&6a^bFl&C`TyjCKVIZgp z0}#=dxS4`}$ZeZQ5DJsvix828;b}a6POljWMd2P_1U%6;@DY&&ce4S|bZX$f_uFV> zQwGUNfiT|<*Q5_W01%35xYa)D{L1+spZ>l3_t(Gwx`pFErTb~W=ZP=B9GHLH9Upu= zZy$Rf_@|3Iae8w5>z{Ijjo`W!)HBXyO#40?a-UgB!3~I1hYzg@x+S)+_~=3q=8Y7U z1`h9gudOIICX^}U6pWwt6bB6ap-sJUib>AFlsq{8I|KhV z8hIdNb~yodEywYv!!ejQE?0hQx0U|5jA=gzkwyOuWFSDqmmC}T^T9-dcLpZVTP4Hq z4E&LJQq7`pn{aB6G43-HE_i?NxgU-yQ74uasKrn)KuD^@Ffw^=YUn-#;yzL@Gvm5Q zHI0soRz#nt^Mt83zZa}fJ`Pbdw1|WqNgYU5MJ~v6fqS)hYxP3~;%Ee|!s-|?2P_)= zV@MehlMl?~mO4Tg4~Yl+dkWRe;X(l@CSJ0Z50%rQ)~GBR<;V&fSmeO8IQHIRyvX={e!nmsogo$C)(r{A7e`mGN6>yW<=`RkCs z4*BblzYh89kiQQ3>yW<=`RkDXMTY!8HJ63_RDiki@9VS0d4YW#JS8*3>sFC|I-oiO zbsoxDpN)8Nq(6hU!Qei9mIsgL^XO6~ru-5_10`e1o^5lJ1Qb5P-Dd0)DD@t?{-KQa zENd>*D-+-4hc!oFgM{8lg6KQy8e-Ww1Bc(j7v=%6!8+vbB>v|7hPuifkDc??Jo~xB zmU6>Wr?2V&B$x3c4W)+GI*V4#-&|@t6Ar3wjomQq?6oA0dzVz^+4`II9z9b>i$9CC)aGQf;SLx=zZ7*^2`^&@pt();!m-THhi z*LLuABJ|m}vjJ2zm?~n4xJ@~e&zskN?v&@X6@yC+l)NxAzlLQSZAGO;g|Ve(za{Mr z;)<@y@uvP*4<32eWj-$N20Y%%{#))Y*zUTk#~b6?K_UUFK7l`MkUoQSy97 z{oKZep4)h$k^Q^9YLR`Iyj|rPs(fI@k5R1=)JJB?fQ@?NpZ>ok8V zukub55v%rK&4r^)ny;$dLXeN3*JijsNGiRSyVoLjt-S80xjc6)rMoHFQ?T+JjmxW$ zzUvC3+T+m#GCI(M`x!BDO8szfIiD!Kn(v~v$jm7QU$^U7twvsp8foWsNYzOSToZHE za@Sl~4;HT^TResqNuQ^VR-UdL*6QvxtGmv!MzWDepHNTENCInA`78j+_m_f~J)o`2 zD`$2Svvp;)-oj=}UoN59W@QDKJ>(?K=H}Z1tu>z$YrX`^%n9PaN)p$$IVRav&AQC3dhdtPDJ*EG!Ewl<%DF zmWa7_pXwr?f6h8ydAg3YPD^STB15RWS!D`V>nc)h6=_mjmTHdBv#gIkv%J($T&IzY z1Q)P+Md^&3E3OZVmAiZG==!1(RBYf=ki;UABqQfG@x}{X9CYQEr{HCkkLU|r)Ah28Ehy%=uDxgAGx9xn(M_tpwRcOXO@=mn--ayj4O5U2bJT% z)u^6&&_F4CC7prXHVQAnKyC*q6|`PsAPp9qYZ0VBqB@c~h&n<2GP{R(nO(xTZadpb zseBE}LJ7*MKL;k*bbipQE$ikDFSU6?=jWW}4W~rhfcylq=iOA(&gqrlx6afr{pd> z7Ukb&sYex?dy+VAl5V_nsX{7FHo`xjd6ZbH`mlLLAr6mPVMUB;84qus8l_gu(X`}> z)xtq)u8<&!Zl;tQo!oMn=?ffI4KZy@@w`%3FU&5y%W1Uo8&fXn0Woj-OteKq&v0yodW8e`(Lo$96R9{z4~v&77wY2iAd4Z7V4K_{ha9=Tk`n za)1>6TAnKnIRMucjC?$wnvB#xj&JnG@z*Bpt(*t&23FJ|rmB2L%c}LHn=M;kZj=4% zZgU&pk9N+3VgoBI{yPF<#|pA4t7IV**nw2?aw#)q0k6+jT4duyWL^IBG&TIwN@Ldg aVGl- literal 3229 zcmV;O3}W*iiwFP!00000|Li?$Q`4yMq$u4yB0^J*Bo+|bU z(St2{Wtoy?^5374FR>HL@g+c^c*;AOsiN=j7_paeLGRunM(wcQ7b!sc zf}vLozD1C>4fI}!Ph6h)uFHi4OZA+qUs2~g&|7X>;kNyS&A+g(U%z4ld%oikF~fa) zJ_P}Y2tbTWGy{nUn<0}s4lwDV2^UCCsfR+Mwy=R+B8U?b#TSaWHu+Vsh2>Lx_DFYF7z%4M**0(7&r-(9(nnZ${!|`oNP4(@L zOU!Gq7PH8i3)_e=3lVgpdQFKpRdj<};_(lFn)FAc1^;{FBpq@e5yU(O9I;2)m~ zuy9`rAJkG^0cuhU_b0^mV1B7OdA=QJS3#GS$(`Lv zB+EOl;Qs|v?v9m|?v8Cf-gSs4LF^Kj?z)2C&i-=+taZiz@3`CE!}GaOpb*aJzga4l zR|;x)v0|W1mevXiqFyt!Ypa6@%Y2p4$oNx~amupxDr_qtFY}$ThNPc@W2I5%G#ef6TdEPWF$8rz=g-5#ZM5~ua}+JT)=2Oh zYJpG#&m<}E^y4Y?GR>%6NupB-Ke#k67GKVdZ2uJg+tK`r|0&m!N{FVeW2Q1z>C(Us zxgCZ;J8ul^P?q`>_a{P-*#@$hzxx@y*LE!?~EyNpL?K0L|wH9(un`CN^b| ztP}|I-Ed5L@go4CsD?Y;!``o)|MBtP+kb!k`>z`~`ct|e-wZtQ`Imk3kK3dD4`+x`ek)FnZ+`t#POuSdw}N`gxr}K)WMl3#ODVV!k!tawH9o0b(oPj3VjJF5JN3e(!Y^<)(x(g`9%%dQWk{$RFF(o2HoL3{1&`<-axXZ=!U{`V)e>#|gdF^uLxAr^fkL#HBy%1USPeBF(M10Ayfj{lfBzUJ_0=-c({MNu9 zh$mGq3O5O-4jAJ;GvS;MN1ujas}gl$S%F#%1p|bnN(>{D=cbzOG9d0E^|COot5nnI zxM)Q5dA>}TYW)Yn2<77tMMH~7*pbwMWK`sW%vZQqi?>!kR3MH<&?=0M5p%$z!5?GF zh?u=+Cb!fQ`glw{c(bQa%`7eyfMVh$WBFJ)9ct`Gt1TRqnJ%lQszOM#Wf^XXG;h^O z5?fHy7h_Qau<7ft3iU^rz@m;v48As#Q&8)n zoDJE87f1RFX!jW0r`Ph}@q8X#s>GC^gJ_^+OgXS^Zjyk)hq&L4eFCN4L)Sl)(ScgFiB{W@f`%%JU6$?dDF+_h<>Tosm0BdzON zM&t*k39d{qJO$!~sw6FQRRv`MR7co&ihPs#V;IQSZ`T<}-*IMj?PEN)&^+FkVT>od zN4uC>!{eF7hc}3tj9`;et&&pJB|0A=kNLo^Lz6gb5@$`~>=`A_?jgzb-F^3hEY$tF z%S{faar0lKoBvaLjhjDY7Tkh}MQ32!Y6q(x99siP69R09?tHE{p94wn+gEh!^RayF z6XiOI`{di%cq?j5)x1Q^seI4p&6+=V%Clz8SX2WgFDlJ1dD%u=Q6*AgD5{xn&20#A z0a)emRKKhT4^*3~pdTTGZK{Hrs-UJSs7Y=#$&Dtt@g$QQO;u1+71UG({dgIrm0et2UdLQYJ;FYn&XjT`(#DUk*lXGi;c>A!78uU?v+=0 z$BKwmuWZeQqs_6`Rc;~3N6>2v+&5NdjoiItx$ERzhMLQBM^d`?C3^~1o}+Pb8Pa!C zVN`QGnm|SidT>7@CQhjb2N%nU(#z#b*H)Q1#o(J>lGdn^m!d|xc^9hcBn7UBS+_bi z7uJKtE6Eg(p+(Z`X`+>Q*2Sf> z*onosvRZFtvbFD`&`eWX0cIEZ2tL$zcqfAXoedgFYS7j9D&zm{e^V$g`F;0)?e41A zSksYwq9VVy?hdTPZ12xDy8|l+Qn7bq2Lr?Uq7%x(;-W&S&zWwG*lXvhF82BBY`mVQ zYmRl=OUn=$Lgn=;Q}))>qG@Z30rl(B7UZlsJ}i0c z?{%Z?iwfDWflojZi%62RoY};it{ffL6>*(_=aDN-wOc@qQ}93pkJ5u}IE6_-;^+~y zzDzD%b+>x%eTnP3dj>uw--5Smq^jfwH-y?`Y{LgGGUzT>&jke;l(TahNK{VETa50( zlrRSNFenTdGAb#Ne1q*g1<&Qo+Jqz2QhJLLfY4(I3htu9iG#uErl%3od|kC>A!jS{ ztKIZvGtaY*ZJD|1+*44tq@zS3&5ATu#U{;8njKmw0qu8b)uQVVD#g4z3-WI?XzB!; zI>D!2C%ED(RNj-63sJpbY*JA)>faGkk$0A@cBv7y>K~UKjS^*JtCqABYo$D;a;zA5 zVO47NmOPIN8(P+Yq?{RzqD}cJFYZ3B-s^SuaYcovN*(+`jsE}aO z`%drEYfg;+s3*pIZ*ne%Iw9g3=mdQt+klSoqlz&^K9#w4aN#eLk zy7AVf3aL2R2>*QIQDUj;k=2Sq94@uOh#1u}9^NuFN{v{eX~_}w!a-`*NRUK3Q_77_ zcDYRV6%MP0m^P+(Ua6}e%r5+#bCbw#Ou3}{#Ju(^)>WoQ=3W*U)llYM8j=RG=_Mg6 z2<1P?d$@~1)?O6&*p6m%OUZHXZO$zvBWvHMwiT3reBfe{^Qj~OIY0`Zmgh=C4!}(X zBOlMFCL_(m@q2wZ{=%evubp#pab!jJOjMQcXjrw2uolBMm)m6i2MJ|f8h>CjmpH>>P(I>}y-90>?U(Wv* P00960pbD7=FOUEL_B30P diff --git a/cmd/boostd/index.go b/cmd/boostd/index.go index 0e4b0fd69..35dc919dd 100644 --- a/cmd/boostd/index.go +++ b/cmd/boostd/index.go @@ -22,6 +22,7 @@ var indexProvCmd = &cli.Command{ indexProvAnnounceLatestHttp, indexProvAnnounceDealRemovalAd, indexProvAnnounceDeal, + indexProvRemoveAllCmd, }, } @@ -303,3 +304,29 @@ var indexProvAnnounceDeal = &cli.Command{ return nil }, } + +var indexProvRemoveAllCmd = &cli.Command{ + Name: "remove-all", + Usage: "Announce all removal ad for all contextIDs", + Flags: []cli.Flag{}, + Action: func(cctx *cli.Context) error { + ctx := lcli.ReqContext(cctx) + + // get boost api + napi, closer, err := bcli.GetBoostAPI(cctx) + if err != nil { + return err + } + defer closer() + + // announce markets and boost deals + cids, err := napi.BoostIndexerRemoveAll(ctx) + if err != nil { + return err + } + for _, c := range cids { + fmt.Println("Published the removal ad with CID", c.String()) + } + return nil + }, +} diff --git a/cmd/migrate-curio/ipni.go b/cmd/migrate-curio/ipni.go new file mode 100644 index 000000000..7cedd3940 --- /dev/null +++ b/cmd/migrate-curio/ipni.go @@ -0,0 +1,49 @@ +package main + +import ( + "fmt" + + "github.com/filecoin-project/boost/node/repo" + lotus_repo "github.com/filecoin-project/lotus/node/repo" + "github.com/mitchellh/go-homedir" + "github.com/urfave/cli/v2" +) + +var removeIPNICmd = &cli.Command{ + Name: "remove-ipni", + Description: "Sends remove ad for all the currently published Ads to IPNI", + Usage: "migrate-curio remove-ipni", + Before: before, + Action: func(cctx *cli.Context) error { + ctx := cctx.Context + + repoDir, err := homedir.Expand(cctx.String(FlagBoostRepo)) + if err != nil { + return err + } + + r, err := lotus_repo.NewFS(repoDir) + if err != nil { + return err + } + ok, err := r.Exists() + if err != nil { + return err + } + if !ok { + return fmt.Errorf("repo at '%s' is not initialized", cctx.String(FlagBoostRepo)) + } + + lr, err := r.Lock(repo.Boost) + if err != nil { + return err + } + + mds, err := lr.Datastore(ctx, "/metadata") + if err != nil { + return err + } + + return migrate(cctx, repoDir) + }, +} diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index 859bfc871..5dfd09678 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -20,6 +20,7 @@ * [BoostIndexerAnnounceLatestHttp](#boostindexerannouncelatesthttp) * [BoostIndexerAnnounceLegacyDeal](#boostindexerannouncelegacydeal) * [BoostIndexerListMultihashes](#boostindexerlistmultihashes) + * [BoostIndexerRemoveAll](#boostindexerremoveall) * [BoostLegacyDealByProposalCid](#boostlegacydealbyproposalcid) * [BoostOfflineDealWithData](#boostofflinedealwithdata) * [I](#i) @@ -374,7 +375,7 @@ Response: ``` ### BoostIndexerAnnounceAllDeals -There are not yet any comments for this method. + Perms: admin @@ -513,6 +514,20 @@ Response: ] ``` +### BoostIndexerRemoveAll +There are not yet any comments for this method. + +Perms: admin + +Inputs: `null` + +Response: +```json +[ + null +] +``` + ### BoostLegacyDealByProposalCid diff --git a/indexprovider/wrapper.go b/indexprovider/wrapper.go index 0976ad1a2..0a828933e 100644 --- a/indexprovider/wrapper.go +++ b/indexprovider/wrapper.go @@ -18,11 +18,11 @@ import ( "github.com/filecoin-project/lotus/chain/actors/builtin/miner" chainTypes "github.com/filecoin-project/lotus/chain/types" "github.com/google/uuid" - cbor "github.com/ipfs/go-ipld-cbor" - "go.uber.org/fx" - "github.com/ipfs/go-datastore" + cbor "github.com/ipfs/go-ipld-cbor" "github.com/ipld/go-ipld-prime" + "github.com/ipni/go-libipni/ingest/schema" + "go.uber.org/fx" "github.com/filecoin-project/boost/db" bdtypes "github.com/filecoin-project/boost/extern/boostd-data/svc/types" @@ -126,8 +126,6 @@ func (w *Wrapper) Start(_ context.Context) { }() log.Info("starting index provider") - - go w.checkForUpdates(runCtx) } func (w *Wrapper) checkForUpdates(ctx context.Context) { @@ -867,3 +865,52 @@ func (w *Wrapper) AnnounceBoostDirectDealRemoved(ctx context.Context, dealUUID u } return annCid, err } + +func (w *Wrapper) AnnounceRemoveAll(ctx context.Context) ([]cid.Cid, error) { + var allAds []*schema.Advertisement + _, ad, err := w.prov.GetLatestAdv(ctx) + if err != nil { + return nil, err + } + allAds = append(allAds, ad) + + prev, err := cid.Parse(ad.PreviousID.String()) + if err != nil { + return nil, err + } + + for prev != cid.Undef { + ad, err := w.prov.GetAdv(ctx, prev) + if err != nil { + return nil, err + } + + prev, err = cid.Parse(ad.PreviousID.String()) + if err != nil { + return nil, err + } + } + + var entryAds []*schema.Advertisement + + for _, ad := range allAds { + if !ad.IsRm { + entryAds = append(entryAds, ad) + } + } + + var newAds []cid.Cid + + for _, ad := range entryAds { + a, err := w.prov.NotifyRemove(ctx, w.h.ID(), ad.ContextID) + if err != nil { + if !errors.Is(err, provider.ErrContextIDNotFound) { + return nil, fmt.Errorf("failed to publish the removal ad: %w", err) + } + } + newAds = append(newAds, a) + } + + return newAds, nil + +} diff --git a/node/impl/boost.go b/node/impl/boost.go index 645fb0e49..c27be73ff 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -231,3 +231,7 @@ func (sm *BoostAPI) PdCleanup(ctx context.Context) error { func (sm *BoostAPI) MarketGetAsk(ctx context.Context) (*legacytypes.SignedStorageAsk, error) { return sm.StorageProvider.GetAsk(), nil } + +func (sm *BoostAPI) BoostIndexerRemoveAll(ctx context.Context) ([]cid.Cid, error) { + return sm.IndexProvider.AnnounceRemoveAll(ctx) +}