Skip to content

Commit

Permalink
Code cleanup
Browse files Browse the repository at this point in the history
  • Loading branch information
@wmitsuda
wmitsuda committed Dec 1, 2024
1 parent e74dd50 commit 1de7110
Show file tree
Hide file tree
Showing 6 changed files with 10 additions and 68 deletions.
13 changes: 2 additions & 11 deletions cmd/integration/commands/idx_compare.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import (
"github.com/spf13/cobra"
)

func ReadEliasFanoOrOpt(v []byte, baseTxNum uint64) *eliasfano32.EliasFano {
func readEliasFanoOrOpt(v []byte, baseTxNum uint64) *eliasfano32.EliasFano {
if v[0]&0b10000000 == 0 {
ef, _ := eliasfano32.ReadEliasFano(v)
return ef
Expand All @@ -27,20 +27,17 @@ func ReadEliasFanoOrOpt(v []byte, baseTxNum uint64) *eliasfano32.EliasFano {
count := (len(v) - 1) / 4
max := uint64(binary.BigEndian.Uint32(v[len(v)-4:])) + baseTxNum
ef := eliasfano32.NewEliasFano(uint64(count), max)
// log.Print("Count: ", count)
// log.Print("Max: ", max)
for i := 1; i <= len(v)-4; i += 4 {
n := uint64(binary.BigEndian.Uint32(v[i:i+4])) + baseTxNum
ef.AddOffset(n)
// log.Print("n: ", n)
}
ef.Build()
return ef
}

func compareOpt4(vOrig, vOpt []byte, baseTxNum uint64) bool {
efOrig, _ := eliasfano32.ReadEliasFano(vOrig)
efOpt := ReadEliasFanoOrOpt(vOpt, baseTxNum)
efOpt := readEliasFanoOrOpt(vOpt, baseTxNum)

if efOpt.Count() > efOrig.Count() {
log.Print("Optimized eliasfano is longer")
Expand Down Expand Up @@ -77,9 +74,7 @@ var idxCompare = &cobra.Command{
Short: "After an idx_optimize execution, deep compare original and optimized .ef files",
Run: func(cmd *cobra.Command, args []string) {
ctx, _ := common.RootContext()
// logger := debug.SetupCobra(cmd, "integration")

// accessorDir := filepath.Join(datadirCli, "snapshots", "accessor")
idxPath := filepath.Join(datadirCli, "snapshots", "idx")
idxDir := os.DirFS(idxPath)

Expand Down Expand Up @@ -151,11 +146,9 @@ var idxCompare = &cobra.Command{

kOrig, _ := readerOrig.Next(nil)
kOpt, _ := readerOpt.Next(nil)
// log.Printf("key %d", kkk)
if !bytes.Equal(kOrig, kOpt) {
log.Fatalf("key mismatch!")
}
// log.Printf("key: %v", hexutility.Encode(kOrig))

if !readerOrig.HasNext() {
log.Fatal("orig reader doesn't have next!")
Expand All @@ -167,7 +160,6 @@ var idxCompare = &cobra.Command{
// orig/opt value comparison
vOrig, _ := readerOrig.Next(nil)
vOpt, nextKeyOffset := readerOpt.Next(nil)
// log.Printf("value %d", nextKeyOffset)
if !compareOpt4(vOrig, vOpt, baseTxNum) {
log.Fatalf("value mismatch!")
}
Expand All @@ -177,7 +169,6 @@ var idxCompare = &cobra.Command{
if !found {
log.Fatalf("key %v not found in efi", hexutility.Encode(kOpt))
}
// log.Printf("ord key: %d", offset)
if offset != prevKeyOffset {
log.Fatalf("offset mismatch: %d != %d", offset, prevKeyOffset)
}
Expand Down
6 changes: 0 additions & 6 deletions cmd/integration/commands/idx_optimize.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,12 +105,6 @@ var idxOptimize = &cobra.Command{
cOpt++
baseTxNum := efInfo.startStep * config3.DefaultStepSize

// efi, err := recsplit.OpenIndex(datadirCli + "/snapshots/accessor/" + file.Name() + "i")
// if err != nil {
// log.Fatalf("Failed to open index: %v", err)
// }
// defer efi.Close()

tmpDir := datadirCli + "/temp"

idxInput, err := seg.NewDecompressor(datadirCli + "/snapshots/idx/" + file.Name())
Expand Down
52 changes: 6 additions & 46 deletions cmd/integration/commands/idx_stat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,7 +263,7 @@ var idxStat = &cobra.Command{
defer output.Close()
bufWriter := bufio.NewWriter(output)
csvWriter := csv.NewWriter(bufWriter)
csvWriter.Write([]string{
headers := []string{
"filename",
"type",
"start step",
Expand All @@ -273,16 +273,11 @@ var idxStat = &cobra.Command{
"record count",
"cumulative key size",
"cumulative value size",
"cumulative value optimized size",
"cumulative value optimized size v2",
"cumulative value optimized size v3",
"cumulative value optimized size v4",
"cumulative value optimized size v5",
"cumulative value optimized size v6",
"cumulative value optimized size v7",
"cumulative value optimized size v8",
"cumulative value optimized size v9",
})
}
for i := range SIM_COUNT {
headers = append(headers, "cumulative value optimized size v"+strconv.Itoa(i+1))
}
csvWriter.Write(headers)

log.Println("Reading idx files:")
grandTotalRecords := 0
Expand All @@ -308,12 +303,6 @@ var idxStat = &cobra.Command{
}
baseTxNum := efInfo.startStep * config3.DefaultStepSize

// efi, err := recsplit.OpenIndex(datadirCli + "/snapshots/accessor/" + file.Name() + "i")
// if err != nil {
// log.Fatalf("Failed to open index: %v", err)
// }
// defer efi.Close()

idx, err := seg.NewDecompressor(datadirCli + "/snapshots/idx/" + file.Name())
if err != nil {
log.Fatalf("Failed to open decompressor: %v", err)
Expand Down Expand Up @@ -424,19 +413,6 @@ var idxStat = &cobra.Command{
"recs", t.records,
"k", t.totalBytesK,
"v", t.totalBytesV,
// "%value", 100*(float64(t.totalBytesV)/float64(totalBytesV)),
// "opt-v", t.totalOptBytesV,
// "%value", 100*(float64(t.totalOptBytesV)/float64(totalOptBytesV)),
// "effec?", effec,
// "opt-v2", t.totalOptBytesV2,
// "%value", 100*(float64(t.totalOptBytesV2)/float64(totalOptBytesV2)),
// "effec2?", effec2,
// "opt-v3", t.totalOptBytesV3,
// "%value", 100*(float64(t.totalOptBytesV3)/float64(totalOptBytesV3)),
// "effec3?", effec3,
// "opt-v4", t.totalOptBytesV4,
// "%value", 100*(float64(t.totalOptBytesV4)/float64(totalOptBytesV4)),
// "effec4?", effec4,
)

data := []string{
Expand Down Expand Up @@ -496,30 +472,14 @@ var idxStat = &cobra.Command{
"k", gt.totalBytesK,
"v", gt.totalBytesV,
"%value", 100*(float64(gt.totalBytesV)/float64(grandTotalBytesV)),
// "%opt-value", 100*(float64(gt.totalOptBytesV)/float64(grandTotalOptBytesV)),
// "%opt-value2", 100*(float64(gt.totalOptBytesV2)/float64(grandTotalOptBytesV2)),
// "%opt-value3", 100*(float64(gt.totalOptBytesV3)/float64(grandTotalOptBytesV3)),
// "%opt-value4", 100*(float64(gt.totalOptBytesV4)/float64(grandTotalOptBytesV4)),
)
}

logger.Info("GRAND TOTALS",
"totalBytes", grandTotalBytesK+grandTotalBytesV,
// "totalBytesOpt", grandTotalBytesK+grandTotalOptBytesV,
// "totalBytesOpt2", grandTotalBytesK+grandTotalOptBytesV2,
// "totalBytesOpt3", grandTotalBytesK+grandTotalOptBytesV3,
// "totalBytesOpt4", grandTotalBytesK+grandTotalOptBytesV4,
"totalBytesFile", grandTotalFiles,
"%keys", 100*float64(grandTotalBytesK)/float64(grandTotalBytesK+grandTotalBytesV),
"%values", 100*float64(grandTotalBytesV)/float64(grandTotalBytesK+grandTotalBytesV),
// "%opt-keys", 100*float64(grandTotalBytesK)/float64(grandTotalBytesK+grandTotalOptBytesV),
// "%opt-values", 100*float64(grandTotalOptBytesV)/float64(grandTotalBytesK+grandTotalOptBytesV),
// "%opt-keys2", 100*float64(grandTotalBytesK)/float64(grandTotalBytesK+grandTotalOptBytesV2),
// "%opt-values2", 100*float64(grandTotalOptBytesV2)/float64(grandTotalBytesK+grandTotalOptBytesV2),
// "%opt-keys3", 100*float64(grandTotalBytesK)/float64(grandTotalBytesK+grandTotalOptBytesV3),
// "%opt-values3", 100*float64(grandTotalOptBytesV3)/float64(grandTotalBytesK+grandTotalOptBytesV3),
// "%opt-keys4", 100*float64(grandTotalBytesK)/float64(grandTotalBytesK+grandTotalOptBytesV4),
// "%opt-values4", 100*float64(grandTotalOptBytesV4)/float64(grandTotalBytesK+grandTotalOptBytesV4),
)

csvWriter.Flush()
Expand Down
1 change: 1 addition & 0 deletions erigon-lib/state/aggregator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,7 @@ func NewAggregator(ctx context.Context, dirs datadir.Dirs, aggregationStep uint6
return a, nil
}

// TODO: exported for idx_optimize.go
func GetStateIndicesSalt(baseDir string) (salt *uint32, err error) {
return getStateIndicesSalt(baseDir)
}
Expand Down
1 change: 1 addition & 0 deletions erigon-lib/state/domain.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1376,6 +1376,7 @@ func (d *Domain) BuildMissedAccessors(ctx context.Context, g *errgroup.Group, ps
}
}

// TODO: exported for idx_optimize.go
func BuildAccessor(ctx context.Context, d *seg.Decompressor, compressed seg.FileCompression, idxPath string, values bool, cfg recsplit.RecSplitArgs, ps *background.ProgressSet, logger log.Logger) error {
return buildAccessor(ctx, d, compressed, idxPath, values, cfg, ps, logger)
}
Expand Down
5 changes: 0 additions & 5 deletions erigon-lib/state/inverted_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -588,11 +588,6 @@ func (iit *InvertedIndexRoTx) seekInFiles(key []byte, txNum uint64) (found bool,
}

g := iit.statelessGetter(i)
// TODO: early read and cache magic key in file struct; not handling it in this prototype
g.Reset(0)
// magicKey, _ := g.Next(nil)
///////////////////////////

g.Reset(offset)
k, _ := g.Next(nil)
if !bytes.Equal(k, key) {
Expand Down

0 comments on commit 1de7110

Please sign in to comment.