From 13069af8a585e1ba7b4223cbfc6e1e2a31a3b4da Mon Sep 17 00:00:00 2001 From: Alexei Shevchenko Date: Tue, 26 Mar 2024 22:34:19 +0300 Subject: [PATCH] compression grouiming, dependencies up (#74) --- .golangci.yml | 10 +- .goreleaser.yml | 4 +- cmd/decompose/main.go | 3 +- docker/README.md | 10 +- go.mod | 7 +- go.sum | 14 +- internal/builder/builder.go | 2 +- internal/builder/builder_test.go | 2 +- internal/builder/common.go | 10 +- internal/builder/dot.go | 204 +++--------- internal/builder/testdata/graphviz-dot.golden | 42 +-- internal/graph/compress.go | 159 ++++++--- internal/graph/compress_test.go | 307 +++++++++++++++++- 13 files changed, 497 insertions(+), 277 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index f951076..712a2ad 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,9 +1,8 @@ run: allow-parallel-runners: true - go: '1.21' + go: '1.22' output: - format: 'colored-line-number' print-issued-lines: true print-linter-name: true sort-results: true @@ -27,6 +26,7 @@ linters: - depguard - maligned - varcheck + - intrange - ifshort - ireturn - gofumpt @@ -37,16 +37,13 @@ linters-settings: errcheck: check-type-assertions: true govet: - check-shadowing: true enable-all: true funlen: lines: 80 ignore-comments: true cyclop: - max-complexity: 15 + max-complexity: 13 skip-tests: true - gocognit: - min-complexity: 40 gocritic: enabled-tags: - performance @@ -75,5 +72,6 @@ issues: - goerr113 - gocritic - errcheck + - maintidx - funlen - dupl diff --git a/.goreleaser.yml b/.goreleaser.yml index 2329e05..2a9a2b3 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -36,11 +36,11 @@ archives: - decompose-bin name_template: >- {{ .ProjectName }}_ + {{- .Tag }}_ {{- .Os }}_ {{- if eq .Arch "amd64" }}x86_64 {{- else if eq .Arch "386" }}i386 - {{- else }}{{ .Arch }}{{ end }}_ - {{- .Tag }} + {{- else }}{{ .Arch }}{{ end }} format_overrides: - goos: windows format: zip diff --git a/cmd/decompose/main.go b/cmd/decompose/main.go index 95ad8dd..e754591 100644 --- a/cmd/decompose/main.go +++ b/cmd/decompose/main.go @@ -32,6 +32,7 @@ const ( autoPrefix = "auto:" defaultProto = "all" defaultOutput = "-" + defaultDiff = 3 ) // build-time values. @@ -291,7 +292,7 @@ func prepareConfig() ( } if fCompress { - cmp := graph.NewCompressor(bildr) + cmp := graph.NewCompressor(bildr, defaultDiff, true) bildr, nwr = cmp, cmp } diff --git a/docker/README.md b/docker/README.md index 82c31ae..0623985 100644 --- a/docker/README.md +++ b/docker/README.md @@ -4,12 +4,20 @@ Reverse-engineering tool for docker environments. # how to run +# scan containers ``` docker run \ -v /var/run/docker.sock:/var/run/docker.sock \ -v /:/rootfs:ro \ -e IN_DOCKER_PROC_ROOT=/rootfs \ - s0rg/decompose:latest -format stat + s0rg/decompose:latest > mystream.json ``` +# process results +``` +docker run \ + s0rg/decompose:latest -load mystream.json -format sdsl > workspace.dsl +``` + + [more options and documentaion](https://github.com/s0rg/decompose) diff --git a/go.mod b/go.mod index 64968e5..7b69851 100644 --- a/go.mod +++ b/go.mod @@ -3,12 +3,12 @@ module github.com/s0rg/decompose go 1.22 require ( - github.com/docker/docker v25.0.5+incompatible + github.com/docker/docker v26.0.0+incompatible github.com/emicklei/dot v1.6.1 - github.com/expr-lang/expr v1.16.1 + github.com/expr-lang/expr v1.16.2 github.com/prometheus/procfs v0.13.0 github.com/s0rg/set v1.2.0 - github.com/s0rg/trie v1.3.1 + github.com/s0rg/trie v1.3.3 gopkg.in/yaml.v3 v3.0.1 ) @@ -23,6 +23,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/kr/pretty v0.3.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/term v0.5.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect diff --git a/go.sum b/go.sum index 69c9db5..14e7038 100644 --- a/go.sum +++ b/go.sum @@ -11,16 +11,16 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/distribution/reference v0.5.0 h1:/FUIFXtfc/x2gpa5/VGfiGLuOIdYa1t65IKK2OFGvA0= github.com/distribution/reference v0.5.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v25.0.5+incompatible h1:UmQydMduGkrD5nQde1mecF/YnSbTOaPeFIeP5C4W+DE= -github.com/docker/docker v25.0.5+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v26.0.0+incompatible h1:Ng2qi+gdKADUa/VM+6b6YaY2nlZhk/lVJiKR/2bMudU= +github.com/docker/docker v26.0.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/emicklei/dot v1.6.1 h1:ujpDlBkkwgWUY+qPId5IwapRW/xEoligRSYjioR6DFI= github.com/emicklei/dot v1.6.1/go.mod h1:DeV7GvQtIw4h2u73RKBkkFdvVAz0D9fzeJrgPW6gy/s= -github.com/expr-lang/expr v1.16.1 h1:Na8CUcMdyGbnNpShY7kzcHCU7WqxuL+hnxgHZ4vaz/A= -github.com/expr-lang/expr v1.16.1/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= +github.com/expr-lang/expr v1.16.2 h1:JvMnzUs3LeVHBvGFcXYmXo+Q6DPDmzrlcSBO6Wy3w4s= +github.com/expr-lang/expr v1.16.2/go.mod h1:uCkhfG+x7fcZ5A5sXHKuQ07jGZRl6J0FCAaf2k4PtVQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -46,6 +46,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= @@ -64,8 +66,8 @@ github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBO github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/s0rg/set v1.2.0 h1:53b207YMktNQJXYei/oHuTR5oOO2e9+eieZOncYsh9g= github.com/s0rg/set v1.2.0/go.mod h1:xz3nDbjF4nyMLvAHvmE7rigXpNrKKTsi6iANznIB1/4= -github.com/s0rg/trie v1.3.1 h1:O9THfLSiXqBN08gJoHegwdkkaGIIs3GnXlYQswfXWTk= -github.com/s0rg/trie v1.3.1/go.mod h1:BGS9ZEqxUvxDT+4qai+YZnzvUDvTpJrx8zBtP7LBjS8= +github.com/s0rg/trie v1.3.3 h1:eBzjWs7hU5RSNvWWQVLKKhurfXxjf50WbAFVfis81Uw= +github.com/s0rg/trie v1.3.3/go.mod h1:BGS9ZEqxUvxDT+4qai+YZnzvUDvTpJrx8zBtP7LBjS8= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 20b67ad..4bcb52a 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -51,7 +51,7 @@ func Create(kind string) (b graph.NamedBuilderWriter, ok bool) { func SupportCluster(n string) (yes bool) { switch n { - case KindDOT, KindStructurizr, KindSTAT, KindPlantUML: + case KindStructurizr, KindSTAT, KindPlantUML: return true } diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index adc425a..9758c2a 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -24,13 +24,13 @@ func TestSupportCluster(t *testing.T) { t.Parallel() does := []string{ - builder.KindDOT, builder.KindSTAT, builder.KindStructurizr, builder.KindPlantUML, } doesnt := []string{ + builder.KindDOT, builder.KindJSON, builder.KindTREE, builder.KindYAML, diff --git a/internal/builder/common.go b/internal/builder/common.go index a2a9b9e..f79b0ab 100644 --- a/internal/builder/common.go +++ b/internal/builder/common.go @@ -8,15 +8,16 @@ import ( ) func joinConnections(conns []*node.Connection, sep string) (rv string) { - raw := make([]string, 0, len(conns)) + tmp := make([]string, 0, len(conns)) for _, c := range conns { - raw = append(raw, c.Port.Label()) + tmp = append(tmp, c.Port.Label()) } - slices.Sort(raw) + slices.Sort(tmp) + tmp = slices.Compact(tmp) - return strings.Join(raw, sep) + return strings.Join(tmp, sep) } func joinListeners(ports map[string][]*node.Port, sep string) (rv string) { @@ -29,6 +30,7 @@ func joinListeners(ports map[string][]*node.Port, sep string) (rv string) { } slices.Sort(tmp) + tmp = slices.Compact(tmp) return strings.Join(tmp, sep) } diff --git a/internal/builder/dot.go b/internal/builder/dot.go index 877a893..3b05433 100644 --- a/internal/builder/dot.go +++ b/internal/builder/dot.go @@ -1,48 +1,25 @@ package builder import ( - "hash/fnv" "io" - "strings" + "slices" "github.com/emicklei/dot" "github.com/s0rg/decompose/internal/node" ) -const ( - outPort = "out" - dotLF = "\n" -) - -// dark28 color scheme from https://www.graphviz.org/doc/info/colors.html -var colors = []string{ - "#1b9e77", - "#d95f02", - "#7570b3", - "#e7298a", - "#66a61e", - "#e6ab02", - "#a6761d", - "#666666", -} - type DOT struct { - g *dot.Graph - clusters map[string]*dot.Graph - edges map[string]map[string][]string + g *dot.Graph + edges map[string]map[string][]string } func NewDOT() *DOT { g := dot.NewGraph(dot.Directed) - g.Attr("splines", "spline") - g.Attr("concentrate", dot.Literal("true")) - return &DOT{ - g: g, - clusters: make(map[string]*dot.Graph), - edges: make(map[string]map[string][]string), + g: g, + edges: make(map[string]map[string][]string), } } @@ -51,174 +28,89 @@ func (d *DOT) Name() string { } func (d *DOT) AddNode(n *node.Node) error { - g := d.g - - if n.Cluster != "" { - sg, ok := d.clusters[n.Cluster] - if !ok { - sg = g.Subgraph(n.Cluster, dot.ClusterOption{}) - d.clusters[n.Cluster] = sg - sg.Node(n.Cluster + "_" + outPort).Label(outPort) - } - - g = sg - } - label, color := renderNode(n) - rb := g.Node(n.ID).Attr( + d.g.Node(n.ID).Attr( "color", color, - ).NewRecordBuilder() - - rb.FieldWithId(label, outPort) - - if n.Ports.Len() > 0 { - rb.Nesting(func() { - n.Ports.Iter(func(process string, _ []*node.Port) { - rb.FieldWithId(process, portID(n.ID, process)) - }) - }) - } - - // this cannot return error, thus error case cannot be tested :( - _ = rb.Build() + ).Label(label) return nil } -func (d *DOT) getSrc(id string) (rv dot.Node, out string, ok bool) { - if rv, ok = d.g.FindNodeById(id); ok { - return rv, outPort, ok +func (d *DOT) AddEdge(e *node.Edge) { + if _, ok := d.g.FindNodeById(e.SrcID); !ok { + return } - sg, ok := d.clusters[id] - if !ok { + if _, ok := d.g.FindNodeById(e.DstID); !ok { return } - out = id + "_" + outPort + d.addEdge(e.SrcID, e.DstID, e.Port.Label()) +} - rv, ok = sg.FindNodeById(out) +func (d *DOT) addEdge(src, dst, label string) { + dmap, ok := d.edges[src] + if !ok { + dmap = make(map[string][]string) + d.edges[src] = dmap + } - return rv, out, ok + dmap[dst] = append(dmap[dst], label) } -func (d *DOT) getDst(edge *node.Edge) (rv dot.Node, out string, ok bool) { - dstID := portID(edge.DstID, edge.DstName) +func (d *DOT) buildEdges() { + order := make([]string, 0, len(d.edges)) - if rv, ok = d.g.FindNodeById(dstID); ok { - return rv, dstID, ok + for k := range d.edges { + order = append(order, k) } - if rv, ok = d.g.FindNodeById(edge.DstID); ok { - return rv, edge.DstID, ok - } - - sg, ok := d.clusters[edge.DstID] - if !ok { - return - } + slices.Sort(order) - if rv, ok = sg.FindNodeById(dstID); ok { - return rv, dstID, ok - } + for _, srcID := range order { + src, _ := d.g.FindNodeById(srcID) - return sg.Node(out).Label(edge.Port.Label()), out, true -} + dmap := d.edges[srcID] + dorder := make([]string, 0, len(dmap)) -func (d *DOT) AddEdge(e *node.Edge) { - if e.SrcID == "" || e.DstID == "" { // fast exit, dot doesnt have default cluster - return - } + for k := range dmap { + dorder = append(dorder, k) + } - src, srcPort, ok := d.getSrc(e.SrcID) - if !ok { - return - } + slices.Sort(dorder) - dst, dstPort, ok := d.getDst(e) - if !ok { - return - } + for _, dstID := range dorder { + dst, _ := d.g.FindNodeById(dstID) + ports := dmap[dstID] - label := e.Port.Label() - color := labelColor(label) + if tmp, ok := d.edges[dstID]; ok { + if dports, ok := tmp[srcID]; ok { + ports = append(ports, dports...) - var edge dot.Edge + delete(tmp, srcID) + } + } - if srcPort != outPort { - edge = d.g.Edge(src, dst, label) - } else { - edge = d.g.EdgeWithPorts(src, dst, srcPort, dstPort, label) + d.g.Edge(src, dst, ports...) + } } - - edge.Attr("color", color).Attr("fontcolor", color) } func (d *DOT) Write(w io.Writer) error { + d.buildEdges() d.g.Write(w) return nil } -func labelColor(label string) (rv string) { - h := fnv.New64a() - - _, _ = io.WriteString(h, label) - - hash := int(h.Sum64()) - - if hash < 0 { - hash = -hash - } - - return colors[hash%len(colors)] -} - -func portID(id, name string) (rv string) { - return "port_" + id + "_" + name -} - func renderNode(n *node.Node) (label, color string) { - var sb strings.Builder + label, color = n.Name, "black" if n.IsExternal() { color = "gray" - - sb.WriteString("external: ") - } else { - color = "black" - } - - sb.WriteString(n.Name) - sb.WriteString(dotLF) - - if n.Image != "" { - sb.WriteString("image: ") - sb.WriteString(n.Image) - sb.WriteString(dotLF) - } - - if len(n.Networks) > 0 { - sb.WriteString("nets: ") - sb.WriteString(strings.Join(n.Networks, ", ")) - sb.WriteString(dotLF) - } - - if n.Meta != nil { - if lines, ok := n.FormatMeta(); ok { - sb.WriteString("meta: ") - sb.WriteString(dotLF) - sb.WriteString(strings.Join(lines, dotLF)) - sb.WriteString(dotLF) - } - - if len(n.Meta.Tags) > 0 { - sb.WriteString("tags: ") - sb.WriteString(strings.Join(n.Meta.Tags, ",")) - sb.WriteString(dotLF) - } + label = "external: " + n.Name } - return sb.String(), color + return label, color } diff --git a/internal/builder/testdata/graphviz-dot.golden b/internal/builder/testdata/graphviz-dot.golden index eb0fdd2..16090e1 100644 --- a/internal/builder/testdata/graphviz-dot.golden +++ b/internal/builder/testdata/graphviz-dot.golden @@ -1,38 +1,10 @@ digraph { - subgraph cluster_s1 { - label="c1"; - n12[label="1/tcp"]; - n2[label="out"]; - n3[color="black",label=" 1\nimage: node-image\nnets: test-net\nmeta: \ninfo 1\ndocs-url\nrepo-url\ntags: 1\n|{ }",shape="record"]; - n4[color="black",label=" 2\nimage: node-image\nnets: test-net\nmeta: \ninfo 2\ntags: 2\n|{ }",shape="record"]; - - } - subgraph cluster_s8 { - label="c2"; - n11[label="2/tcp"]; - n10[color="gray",label=" external: 2\n|{ }",shape="record"]; - n9[label="out"]; - - } - subgraph cluster_s5 { - label="c3"; - n6[label="out"]; - n7[color="black",label=" 3\nimage: node-image\nnets: test-net\nmeta: \ninfo 3\ntags: 3\n|{ }",shape="record"]; - - } - concentrate=true;splines="spline"; - n10:out->n3:node-1[color="#66a61e",fontcolor="#66a61e",label="1/tcp"]; - n10:out->n3:node-1[color="#e7298a",fontcolor="#e7298a",label="2/tcp"]; - n10:out->n3:node-1[color="#7570b3",fontcolor="#7570b3",label="3/tcp"]; - n2->n11[color="#e7298a",fontcolor="#e7298a",label="2/tcp"]; - n2->n11[color="#e7298a",fontcolor="#e7298a",label="2/tcp"]; - n9->n12[color="#66a61e",fontcolor="#66a61e",label="1/tcp"]; - n9->n12[color="#66a61e",fontcolor="#66a61e",label="1/tcp"]; - n3:out->n10:2[color="#e7298a",fontcolor="#e7298a",label="2/tcp"]; - n3:out->n10:2[color="#7570b3",fontcolor="#7570b3",label="3/tcp"]; - n3:out->n4:node-2[color="#66a61e",fontcolor="#66a61e",label="1/tcp"]; - n3:out->n4:node-2[color="#66a61e",fontcolor="#66a61e",label="1/tcp"]; - n4:out->n3:node-1[color="#e7298a",fontcolor="#e7298a",label="2/tcp"]; - n4:out->n3:node-1[color="#e7298a",fontcolor="#e7298a",label="2/tcp"]; + + n4[color="gray",label="external: 2"]; + n1[color="black",label="1"]; + n2[color="black",label="2"]; + n3[color="black",label="3"]; + n4->n1[label="1/tcp,2/tcp,3/tcp,2/tcp,3/tcp"]; + n1->n2[label="1/tcp,1/tcp,2/tcp,2/tcp"]; } diff --git a/internal/graph/compress.go b/internal/graph/compress.go index ca13289..2743e95 100644 --- a/internal/graph/compress.go +++ b/internal/graph/compress.go @@ -17,23 +17,32 @@ import ( const ( externalGroup = "external" defaultGroup = "core" - defaultDiff = 3 ) type Compressor struct { - b NamedBuilderWriter - nodes map[string]*node.Node - conns map[string]map[string][]*node.Port - edges int + b NamedBuilderWriter + nodes map[string]*node.Node // "raw" incoming nodes nodeID -> node + groups map[string]*node.Node // "compressed" nodes groupID -> node + index map[string]string // index holds nodeID -> groupID mapping + conns map[string]map[string][]*node.Port // "raw" connections nodeID -> nodeID -> []port + edges int + diff int + force bool } func NewCompressor( bldr NamedBuilderWriter, + diff int, + force bool, ) *Compressor { return &Compressor{ - b: bldr, - nodes: make(map[string]*node.Node), - conns: make(map[string]map[string][]*node.Port), + b: bldr, + diff: diff, + force: force, + index: make(map[string]string), + nodes: make(map[string]*node.Node), + groups: make(map[string]*node.Node), + conns: make(map[string]map[string][]*node.Port), } } @@ -69,12 +78,38 @@ func (c *Compressor) Name() string { } func (c *Compressor) Write(w io.Writer) (err error) { - index, err := c.buildGroups() - if err != nil { - return err + c.buildGroups() + edges, count := c.buildEdges() + + for _, node := range c.groups { + if err = c.b.AddNode(node); err != nil { + return fmt.Errorf("compressor add-node [%s]: %w", c.b.Name(), err) + } + } + + log.Printf("[compress] nodes %d -> %d %.02f%%", + len(c.nodes), + len(c.groups), + percentOf(len(c.nodes)-len(c.groups), len(c.nodes)), + ) + + for src, dmap := range edges { + for dst, ports := range dmap { + for _, port := range ports { + c.b.AddEdge(&node.Edge{ + SrcID: src, + DstID: dst, + Port: port, + }) + } + } } - c.buildEdges(index) + log.Printf("[compress] edges %d -> %d %.02f%%", + c.edges, + count, + percentOf(c.edges-count, c.edges), + ) if err = c.b.Write(w); err != nil { return fmt.Errorf("compressor write [%s]: %w", c.b.Name(), err) @@ -83,8 +118,7 @@ func (c *Compressor) Write(w io.Writer) (err error) { return nil } -func (c *Compressor) buildGroups() (index map[string]string, err error) { - index = make(map[string]string) +func (c *Compressor) buildGroups() { groups := make(map[string][]string) t := trie.New[string]() @@ -100,7 +134,7 @@ func (c *Compressor) buildGroups() (index map[string]string, err error) { t.Add(node.Name, node.ID) } - comm := t.Common("", defaultDiff) + comm := t.Common("", c.diff) for _, key := range comm { nodes := []string{} @@ -115,7 +149,7 @@ func (c *Compressor) buildGroups() (index map[string]string, err error) { } for _, nodeID := range nodes { - index[nodeID] = grp + c.index[nodeID] = grp seen.Del(nodeID) } @@ -131,7 +165,7 @@ func (c *Compressor) buildGroups() (index map[string]string, err error) { } groups[grp] = append(groups[grp], id) - index[id] = grp + c.index[id] = grp return true }) @@ -143,64 +177,89 @@ func (c *Compressor) buildGroups() (index map[string]string, err error) { batch[i] = c.nodes[nodeID] } - if err = c.b.AddNode(compressNodes(grp, batch)); err != nil { - err = fmt.Errorf("compressor add-node [%s]: %w", c.b.Name(), err) - - return nil, err - } + c.groups[grp] = compressNodes(grp, batch) } - - log.Printf("[compress] nodes %d -> %d %.02f%%", - len(c.nodes), - len(groups), - percentOf(len(c.nodes)-len(groups), len(c.nodes)), - ) - - return index, nil } -func (c *Compressor) buildEdges(index map[string]string) { - gconns := make(map[string]map[string][]*node.Port) +func (c *Compressor) buildEdges() ( + edges map[string]map[string][]*node.Port, + count int, +) { + edges = make(map[string]map[string][]*node.Port) + // initial compression: compress to groups for src, dmap := range c.conns { - srcg := index[src] + srcg := c.index[src] - gmap, ok := gconns[srcg] + gmap, ok := edges[srcg] if !ok { gmap = make(map[string][]*node.Port) - gconns[srcg] = gmap + edges[srcg] = gmap } for dst, ports := range dmap { - dstg := index[dst] + if src == dst { // skip nodes cycles + continue + } + + dstg := c.index[dst] + if srcg == dstg { + continue // skip groups cycles + } gmap[dstg] = append(gmap[dstg], ports...) } } - var count int + if c.force { + edges = c.forceCompress(edges) + } - for src, dmap := range gconns { + for _, dmap := range edges { for dst, ports := range dmap { ports = compressPorts(ports) + dmap[dst] = ports + count += len(ports) + } + } - for _, port := range ports { - c.b.AddEdge(&node.Edge{ - SrcID: src, - DstID: dst, - Port: port, - }) + return edges, count +} - count++ +// force compression: remove single-connected groups. +func (c *Compressor) forceCompress( + edges map[string]map[string][]*node.Port, +) ( + rv map[string]map[string][]*node.Port, +) { + dsts := make(map[string][]string) + drop := func(k, v string) { + delete(c.groups, v) + delete(edges[k], v) + delete(edges, v) + } + + for src, dmap := range edges { + if len(dmap) == 1 { + for key := range dmap { + drop(key, src) } + + continue + } + + for dst := range dmap { + dsts[dst] = append(dsts[dst], src) } } - log.Printf("[compress] edges %d -> %d %.02f%%", - c.edges, - count, - percentOf(c.edges-count, c.edges), - ) + for k, v := range dsts { + if len(v) == 1 { + drop(v[0], k) + } + } + + return edges } func cleanName(a string) string { diff --git a/internal/graph/compress_test.go b/internal/graph/compress_test.go index aaea2c6..0d2037e 100644 --- a/internal/graph/compress_test.go +++ b/internal/graph/compress_test.go @@ -2,8 +2,8 @@ package graph_test import ( "errors" + "fmt" "io" - "log" "strings" "testing" @@ -11,6 +11,8 @@ import ( "github.com/s0rg/decompose/internal/node" ) +const diff = 3 + type testNamedBuilder struct { AddError error WriteError error @@ -18,17 +20,13 @@ type testNamedBuilder struct { Edges int } -func (b *testNamedBuilder) AddNode(n *node.Node) error { - log.Printf("%+v", n) - +func (b *testNamedBuilder) AddNode(_ *node.Node) error { b.Nodes++ return b.AddError } -func (b *testNamedBuilder) AddEdge(e *node.Edge) { - log.Printf("%+v", e) - +func (b *testNamedBuilder) AddEdge(_ *node.Edge) { b.Edges++ } @@ -40,12 +38,16 @@ func (b *testNamedBuilder) Write(_ io.Writer) error { return b.WriteError } +func (b *testNamedBuilder) String() string { + return fmt.Sprintf("nodes: %d edges: %d", b.Nodes, b.Edges) +} + func TestCompressor(t *testing.T) { t.Parallel() tb := &testNamedBuilder{} - c := graph.NewCompressor(tb) + c := graph.NewCompressor(tb, diff, false) // ingress nginx := &node.Node{ @@ -191,12 +193,295 @@ func TestCompressor(t *testing.T) { } } +func TestCompressorForce(t *testing.T) { + t.Parallel() + + tb := &testNamedBuilder{} + + c := graph.NewCompressor(tb, 1, true) + + a1 := &node.Node{ + ID: "a1-id", + Name: "a1", + Ports: &node.Ports{}, + } + + a1.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(a1) + + a2 := &node.Node{ + ID: "a2-id", + Name: "a2", + Ports: &node.Ports{}, + } + + a2.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(a2) + + a3 := &node.Node{ + ID: "a3-id", + Name: "a3", + Ports: &node.Ports{}, + } + + a3.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(a3) + + b1 := &node.Node{ + ID: "b1-id", + Name: "b1", + Ports: &node.Ports{}, + } + + b1.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(b1) + + b2 := &node.Node{ + ID: "b2-id", + Name: "b2", + Ports: &node.Ports{}, + } + + b2.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(b2) + + b3 := &node.Node{ + ID: "b3-id", + Name: "b3", + Ports: &node.Ports{}, + } + + b3.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(b3) + + c1 := &node.Node{ + ID: "c1-id", + Name: "c1", + Ports: &node.Ports{}, + } + + c1.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(c1) + + c2 := &node.Node{ + ID: "c2-id", + Name: "c2", + Ports: &node.Ports{}, + } + + c2.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(c2) + + c3 := &node.Node{ + ID: "c3-id", + Name: "c3", + Ports: &node.Ports{}, + } + + c3.Ports.Add("app", &node.Port{Kind: "tcp", Value: 8080}) + c.AddNode(c3) + + d1 := &node.Node{ + ID: "d1-id", + Name: "d1", + Ports: &node.Ports{}, + } + + d1.Ports.Add("app", &node.Port{Kind: "tcp", Value: 5555}) + c.AddNode(d1) + + d2 := &node.Node{ + ID: "d2-id", + Name: "d2", + Ports: &node.Ports{}, + } + + d2.Ports.Add("app", &node.Port{Kind: "tcp", Value: 5555}) + c.AddNode(d2) + + e1 := &node.Node{ + ID: "e1-id", + Name: "e1", + Ports: &node.Ports{}, + } + + e1.Ports.Add("app", &node.Port{Kind: "tcp", Value: 6666}) + c.AddNode(e1) + + e2 := &node.Node{ + ID: "e2-id", + Name: "e2", + Ports: &node.Ports{}, + } + + e2.Ports.Add("app", &node.Port{Kind: "tcp", Value: 6666}) + c.AddNode(e2) + + f1 := &node.Node{ + ID: "f1-id", + Name: "f1", + Ports: &node.Ports{}, + } + + f1.Ports.Add("app", &node.Port{Kind: "tcp", Value: 7777}) + c.AddNode(f1) + + f2 := &node.Node{ + ID: "f2-id", + Name: "f2", + Ports: &node.Ports{}, + } + + f2.Ports.Add("app", &node.Port{Kind: "tcp", Value: 7777}) + c.AddNode(f2) + + c.AddEdge(&node.Edge{ + SrcID: "a1-id", + DstID: "b1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "a1-id", + DstID: "c1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "a2-id", + DstID: "b2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "a2-id", + DstID: "c2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "b1-id", + DstID: "a1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "b1-id", + DstID: "c1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "b2-id", + DstID: "a2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "b2-id", + DstID: "c2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "c1-id", + DstID: "a1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "c1-id", + DstID: "b1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "c2-id", + DstID: "a2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "c2-id", + DstID: "a2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "a1-id", + DstID: "d1-id", + Port: &node.Port{Kind: "tcp", Value: 5555}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "a2-id", + DstID: "d2-id", + Port: &node.Port{Kind: "tcp", Value: 5555}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "b1-id", + DstID: "e1-id", + Port: &node.Port{Kind: "tcp", Value: 6666}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "b2-id", + DstID: "e2-id", + Port: &node.Port{Kind: "tcp", Value: 6666}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "e1-id", + DstID: "b2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "e2-id", + DstID: "b1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "f1-id", + DstID: "c1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "f2-id", + DstID: "c2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "a1-id", + DstID: "a1-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + c.AddEdge(&node.Edge{ + SrcID: "a1-id", + DstID: "a2-id", + Port: &node.Port{Kind: "tcp", Value: 8080}, + }) + + if err := c.Write(nil); err != nil { + t.Fail() + } + + if tb.Nodes != 3 || tb.Edges != 6 { + t.Fail() + } +} + func TestCompressorName(t *testing.T) { t.Parallel() tb := &testNamedBuilder{} - c := graph.NewCompressor(tb) + c := graph.NewCompressor(tb, diff, false) if !strings.Contains(c.Name(), "compressed") { t.Fail() @@ -211,7 +496,7 @@ func TestCompressorAddError(t *testing.T) { AddError: myErr, } - c := graph.NewCompressor(tb) + c := graph.NewCompressor(tb, diff, false) if err := c.AddNode(&node.Node{Ports: &node.Ports{}}); err != nil { t.Fail() @@ -231,7 +516,7 @@ func TestCompressorWriteError(t *testing.T) { WriteError: myErr, } - c := graph.NewCompressor(tb) + c := graph.NewCompressor(tb, diff, false) if err := c.AddNode(&node.Node{Ports: &node.Ports{}}); err != nil { t.Fail()