Skip to content

Commit

Permalink
skip invalid history records and fix history file during start up
Browse files Browse the repository at this point in the history 
fail when ther is more than one decoding error
be less verbouse
  • Loading branch information
@curusarn
curusarn committed Feb 16, 2020
1 parent 96b5e9d commit 27db751
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 47 deletions.
43 changes: 30 additions & 13 deletions cmd/cli/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ var commit string
// special constant recognized by RESH wrappers
const exitCodeExecute = 111

var debug bool

func main() {
output, exitCode := runReshCli()
fmt.Print(output)
Expand All @@ -59,6 +61,7 @@ func runReshCli() (string, int) {
log.Fatal("Error reading config:", err)
}
if config.Debug {
debug = true
log.SetFlags(log.LstdFlags | log.Lmicroseconds)
}

Expand Down Expand Up @@ -225,20 +228,26 @@ func filterTerms(terms []string) []string {
}

func newQueryFromString(queryInput string, pwd string) query {
log.Println("QUERY input = <" + queryInput + ">")
if debug {
log.Println("QUERY input = <" + queryInput + ">")
}
terms := strings.Fields(queryInput)
var logStr string
for _, term := range terms {
logStr += " <" + term + ">"
}
log.Println("QUERY raw terms =" + logStr)
if debug {
log.Println("QUERY raw terms =" + logStr)
}
terms = filterTerms(terms)
logStr = ""
for _, term := range terms {
logStr += " <" + term + ">"
}
log.Println("QUERY filtered terms =" + logStr)
log.Println("QUERY pwd =" + pwd)
if debug {
log.Println("QUERY filtered terms =" + logStr)
log.Println("QUERY pwd =" + pwd)
}
return query{terms: terms, pwd: pwd}
}

Expand Down Expand Up @@ -445,9 +454,11 @@ func (m manager) SelectPaste(g *gocui.Gui, v *gocui.View) error {
}

func (m manager) UpdateData(input string) {
log.Println("EDIT start")
log.Println("len(fullRecords) =", len(m.s.fullRecords))
log.Println("len(data) =", len(m.s.data))
if debug {
log.Println("EDIT start")
log.Println("len(fullRecords) =", len(m.s.fullRecords))
log.Println("len(data) =", len(m.s.data))
}
query := newQueryFromString(input, m.pwd)
var data []item
itemSet := make(map[string]bool)
Expand All @@ -468,7 +479,9 @@ func (m manager) UpdateData(input string) {
data = append(data, itm)
// log.Println("DATA =", itm.display)
}
log.Println("len(tmpdata) =", len(data))
if debug {
log.Println("len(tmpdata) =", len(data))
}
sort.SliceStable(data, func(p, q int) bool {
return data[p].hits > data[q].hits
})
Expand All @@ -480,9 +493,11 @@ func (m manager) UpdateData(input string) {
m.s.data = append(m.s.data, itm)
}
m.s.highlightedItem = 0
log.Println("len(fullRecords) =", len(m.s.fullRecords))
log.Println("len(data) =", len(m.s.data))
log.Println("EDIT end")
if debug {
log.Println("len(fullRecords) =", len(m.s.fullRecords))
log.Println("len(data) =", len(m.s.data))
log.Println("EDIT end")
}
}

func (m manager) Edit(v *gocui.View, key gocui.Key, ch rune, mod gocui.Modifier) {
Expand Down Expand Up @@ -563,8 +578,10 @@ func (m manager) Layout(g *gocui.Gui) error {
// v.SetHighlight(m.s.highlightedItem, true)
// }
}
log.Println("len(data) =", len(m.s.data))
log.Println("highlightedItem =", m.s.highlightedItem)
if debug {
log.Println("len(data) =", len(m.s.data))
log.Println("highlightedItem =", m.s.highlightedItem)
}
return nil
}

Expand Down
32 changes: 27 additions & 5 deletions pkg/histfile/histfile.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,13 +46,11 @@ func New(input chan records.Record, sessionsToDrop chan string,
go hf.loadHistory(bashHistoryPath, zshHistoryPath, maxInitHistSize, minInitHistSizeKB)
go hf.writer(input, signals, shutdownDone)
go hf.sessionGC(sessionsToDrop)
go hf.loadFullRecords()
return &hf
}

// load records from resh history, reverse, enrich and save
func (h *Histfile) loadFullRecords() {
recs := records.LoadFromFile(h.historyPath, math.MaxInt32)
func (h *Histfile) loadFullRecords(recs []records.Record) {
for i := len(recs) - 1; i >= 0; i-- {
rec := recs[i]
h.fullRecords.AddRecord(rec)
Expand Down Expand Up @@ -83,10 +81,11 @@ func (h *Histfile) loadHistory(bashHistoryPath, zshHistoryPath string, maxInitHi
maxInitHistSize = math.MaxInt32
}
log.Println("histfile: Loading resh history from file ...")
reshCmdLines := histlist.New()
history := records.LoadFromFile(h.historyPath, math.MaxInt32)
go h.loadFullRecords(history)
// NOTE: keeping this weird interface for now because we might use it in the future
// when we only load bash or zsh history
records.LoadCmdLinesFromFile(&reshCmdLines, h.historyPath, maxInitHistSize)
reshCmdLines := loadCmdLines(history)
log.Println("histfile: resh history loaded - cmdLine count:", len(reshCmdLines.List))
if useNativeHistories == false {
h.bashCmdLines = reshCmdLines
Expand Down Expand Up @@ -230,3 +229,26 @@ func (h *Histfile) DumpRecords() histcli.Histcli {
// don't forget locks in the future
return h.fullRecords
}

func loadCmdLines(recs []records.Record) histlist.Histlist {
hl := histlist.New()
// go from bottom and deduplicate
var cmdLines []string
cmdLinesSet := map[string]bool{}
for i := len(recs) - 1; i >= 0; i-- {
cmdLine := recs[i].CmdLine
if cmdLinesSet[cmdLine] {
continue
}
cmdLinesSet[cmdLine] = true
cmdLines = append([]string{cmdLine}, cmdLines...)
// if len(cmdLines) > limit {
// break
// }
}
// add everything to histlist
for _, cmdLine := range cmdLines {
hl.AddCmdLine(cmdLine)
}
return hl
}
103 changes: 74 additions & 29 deletions pkg/records/records.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"bufio"
"encoding/json"
"errors"
"io"
"log"
"math"
"os"
Expand Down Expand Up @@ -173,16 +174,16 @@ func Enriched(r Record) EnrichedRecord {
record.Command, record.FirstWord, err = GetCommandAndFirstWord(r.CmdLine)
if err != nil {
record.Errors = append(record.Errors, "GetCommandAndFirstWord error:"+err.Error())
rec, _ := record.ToString()
log.Println("Invalid command:", rec)
// rec, _ := record.ToString()
// log.Println("Invalid command:", rec)
record.Invalid = true
return record
}
err = r.Validate()
if err != nil {
record.Errors = append(record.Errors, "Validate error:"+err.Error())
rec, _ := record.ToString()
log.Println("Invalid command:", rec)
// rec, _ := record.ToString()
// log.Println("Invalid command:", rec)
record.Invalid = true
}
return record
Expand Down Expand Up @@ -306,7 +307,7 @@ func Stripped(r EnrichedRecord) EnrichedRecord {
func GetCommandAndFirstWord(cmdLine string) (string, string, error) {
args, err := shellwords.Parse(cmdLine)
if err != nil {
log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )")
// log.Println("shellwords Error:", err, " (cmdLine: <", cmdLine, "> )")
return "", "", err
}
if len(args) == 0 {
Expand Down Expand Up @@ -456,31 +457,10 @@ func (r *EnrichedRecord) DistanceTo(r2 EnrichedRecord, p DistParams) float64 {
return dist
}

// LoadCmdLinesFromFile loads cmdlines from file
func LoadCmdLinesFromFile(hl *histlist.Histlist, fname string, limit int) {
recs := LoadFromFile(fname, limit*3) // assume that at least 1/3 of commands is unique
// go from bottom and deduplicate
var cmdLines []string
cmdLinesSet := map[string]bool{}
for i := len(recs) - 1; i >= 0; i-- {
cmdLine := recs[i].CmdLine
if cmdLinesSet[cmdLine] {
continue
}
cmdLinesSet[cmdLine] = true
cmdLines = append([]string{cmdLine}, cmdLines...)
if len(cmdLines) > limit {
break
}
}
// add everything to histlist
for _, cmdLine := range cmdLines {
hl.AddCmdLine(cmdLine)
}
}

// LoadFromFile loads records from 'fname' file
func LoadFromFile(fname string, limit int) []Record {
const allowedErrors = 1
var encounteredErrors int
// NOTE: limit does nothing atm
var recs []Record
file, err := os.Open(fname)
Expand All @@ -492,24 +472,89 @@ func LoadFromFile(fname string, limit int) []Record {
defer file.Close()

scanner := bufio.NewScanner(file)
var i int
var firstErrLine int
for scanner.Scan() {
i++
record := Record{}
fallbackRecord := FallbackRecord{}
line := scanner.Text()
err = json.Unmarshal([]byte(line), &record)
if err != nil {
err = json.Unmarshal([]byte(line), &fallbackRecord)
if err != nil {
if encounteredErrors == 0 {
firstErrLine = i
}
encounteredErrors++
log.Println("Line:", line)
log.Fatal("Decoding error:", err)
log.Println("Decoding error:", err)
if encounteredErrors > allowedErrors {
log.Fatalf("Fatal: Encountered more than %d decoding errors (%d)", allowedErrors, encounteredErrors)
}
}
record = Convert(&fallbackRecord)
}
recs = append(recs, record)
}
if encounteredErrors > 0 {
// fix errors in the history file
log.Printf("There were %d decoding errors, the first error happend on line %d/%d", encounteredErrors, firstErrLine, i)
log.Println("Backing up current history file ...")
err := copyFile(fname, fname+".bak")
if err != nil {
log.Fatalln("Failed to backup history file with decode errors")
}
log.Println("Writing out a history file without errors ...")
err = writeHistory(fname, recs)
if err != nil {
log.Fatalln("Fatal: Failed write out new history")
}
}
return recs
}

func copyFile(source, dest string) error {
from, err := os.Open(source)
if err != nil {
// log.Println("Open() resh history file error:", err)
return err
}
defer from.Close()

// to, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE, 0666)
to, err := os.Create(dest)
if err != nil {
// log.Println("Create() resh history backup error:", err)
return err
}
defer to.Close()

_, err = io.Copy(to, from)
if err != nil {
// log.Println("Copy() resh history to backup error:", err)
return err
}
return nil
}

func writeHistory(fname string, history []Record) error {
file, err := os.Create(fname)
if err != nil {
// log.Println("Create() resh history error:", err)
return err
}
defer file.Close()
for _, rec := range history {
jsn, err := json.Marshal(rec)
if err != nil {
log.Fatalln("Encode error!")
}
file.Write(append(jsn, []byte("\n")...))
}
return nil
}

// LoadCmdLinesFromZshFile loads cmdlines from zsh history file
func LoadCmdLinesFromZshFile(fname string) histlist.Histlist {
hl := histlist.New()
Expand Down

0 comments on commit 27db751

Please sign in to comment.