PMM-5680 store logs

agentlocal/agent_local.go

@@ -16,10 +16,13 @@
 package agentlocal
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	_ "expvar" // register /debug/vars
+	"fmt"
 	"html/template"
+	"io"
 	"log"
 	"net"
 	"net/http"
@@ -47,6 +50,7 @@ import (
 	"google.golang.org/protobuf/types/known/durationpb"
 
 	"github.com/percona/pmm-agent/config"
+	"github.com/percona/pmm-agent/storelogs"
 )
 
 const (
@@ -61,23 +65,28 @@ type Server struct {
 	configFilepath string
 
 	l               *logrus.Entry
+	ringLogs        *storelogs.LogsStore
 	reload          chan struct{}
 	reloadCloseOnce sync.Once
 
 	agentlocalpb.UnimplementedAgentLocalServer
 }
 
 // NewServer creates new server.
-//
+//`
 // Caller should call Run.
-func NewServer(cfg *config.Config, supervisor supervisor, client client, configFilepath string) *Server {
+func NewServer(cfg *config.Config, supervisor supervisor, client client, configFilepath string, ringLog *storelogs.LogsStore) *Server {
+	logger := logrus.New()
+	logger.Out = io.MultiWriter(os.Stderr, ringLog)
+
 	return &Server{
 		cfg:            cfg,
 		supervisor:     supervisor,
 		client:         client,
 		configFilepath: configFilepath,
-		l:              logrus.WithField("component", "local-server"),
+		l:              logger.WithField("component", "local-server"),
 		reload:         make(chan struct{}),
+		ringLogs:       ringLog,
 	}
 }
 
@@ -289,6 +298,43 @@ func (s *Server) runJSONServer(ctx context.Context, grpcAddress string) {
 	mux.Handle("/debug/", http.DefaultServeMux)
 	mux.Handle("/debug", debugPageHandler)
 	mux.Handle("/", proxyMux)
+	mux.HandleFunc("/logs.zip", func(w http.ResponseWriter, r *http.Request) {
+		buf := &bytes.Buffer{}
+		writer := zip.NewWriter(buf)
+		b := &bytes.Buffer{}
+		for _, serverLog := range s.ringLogs.GetLogs() {
+			_, err := b.WriteString(serverLog)
+			if err != nil {
+				log.Fatal(err)
+			}
+		}
+		addData(writer, "server.txt", b.Bytes())
+
+		for id, logs := range s.supervisor.AgentsLogs() {
+			if err != nil {
+				log.Fatal(err)
+			}
+			b := &bytes.Buffer{}
+			for _, l := range logs {
+				_, err := b.WriteString(l + "\n")
+				if err != nil {
+					log.Fatal(err)
+				}
+			}
+			addData(writer, fmt.Sprintf("%s.txt", id), b.Bytes())
+		}
+		err = writer.Close()
+		if err != nil {
+			log.Fatal(err)
+		}
+		w.Header().Set("Content-Type", "application/zip")
+		w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", "logs"))
+		// io.Copy(w, buf)
+		_, err = w.Write(buf.Bytes())
+		if err != nil {
+			log.Fatal(err)
+		}
+	})
 
 	server := &http.Server{
 		Addr:     address,
@@ -314,6 +360,18 @@ func (s *Server) runJSONServer(ctx context.Context, grpcAddress string) {
 	_ = server.Close() // call Close() in all cases
 }
 
+// addData add data to zip file
+func addData(zipW *zip.Writer, name string, data []byte) {
+	f, err := zipW.Create(name)
+	if err != nil {
+		log.Fatal(err)
+	}
+	_, err = f.Write(data)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
 // check interfaces
 var (
 	_ agentlocalpb.AgentLocalServer = (*Server)(nil)

agentlocal/agent_local_test.go

@@ -28,6 +28,7 @@ import (
 	"google.golang.org/protobuf/types/known/durationpb"
 
 	"github.com/percona/pmm-agent/config"
+	"github.com/percona/pmm-agent/storelogs"
 )
 
 func TestServerStatus(t *testing.T) {
@@ -61,7 +62,8 @@ func TestServerStatus(t *testing.T) {
 		agentInfo, supervisor, client, cfg := setup(t)
 		defer supervisor.AssertExpectations(t)
 		defer client.AssertExpectations(t)
-		s := NewServer(cfg, supervisor, client, "/some/dir/pmm-agent.yaml")
+		ringLog := storelogs.New(500)
+		s := NewServer(cfg, supervisor, client, "/some/dir/pmm-agent.yaml", ringLog)
 
 		// without network info
 		actual, err := s.Status(context.Background(), &agentlocalpb.StatusRequest{GetNetworkInfo: false})
@@ -87,7 +89,8 @@ func TestServerStatus(t *testing.T) {
 		client.On("GetNetworkInformation").Return(latency, clockDrift, nil)
 		defer supervisor.AssertExpectations(t)
 		defer client.AssertExpectations(t)
-		s := NewServer(cfg, supervisor, client, "/some/dir/pmm-agent.yaml")
+		ringLog := storelogs.New(500)
+		s := NewServer(cfg, supervisor, client, "/some/dir/pmm-agent.yaml", ringLog)
 
 		// with network info
 		actual, err := s.Status(context.Background(), &agentlocalpb.StatusRequest{GetNetworkInfo: true})

agentlocal/deps.go

@@ -39,4 +39,5 @@ type client interface {
 // We use it instead of real type for testing and to avoid dependency cycle.
 type supervisor interface {
 	AgentsList() []*agentlocalpb.AgentInfo
+	AgentsLogs() map[string][]string
 }

agents/process/process_child.go

@@ -25,7 +25,7 @@ import (
 	"context"
 	"flag"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"time"
 
 	"github.com/percona/pmm/api/inventorypb"
@@ -37,9 +37,8 @@ import (
 func main() {
 	flag.Parse()
 	logger := logrus.New()
-	logger.SetOutput(io.Discard)
+	logger.SetOutput(ioutil.Discard)
 	l := logrus.NewEntry(logger)
-
 	p := process.New(&process.Params{Path: "sleep", Args: []string{"100500"}}, nil, l)
 	go p.Run(context.Background())
 

agents/process/process_test.go

@@ -17,6 +17,7 @@ package process
 
 import (
 	"context"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"runtime"
@@ -123,7 +124,7 @@ func TestProcess(t *testing.T) {
 	})
 
 	t.Run("Killed", func(t *testing.T) {
-		f, err := os.CreateTemp("", "pmm-agent-process-test-noterm")
+		f, err := ioutil.TempFile("", "pmm-agent-process-test-noterm")
 		require.NoError(t, err)
 		require.NoError(t, f.Close())
 		defer func() {
@@ -146,7 +147,7 @@ func TestProcess(t *testing.T) {
 			t.Skip("Pdeathsig is implemented only on Linux")
 		}
 
-		f, err := os.CreateTemp("", "pmm-agent-process-test-child")
+		f, err := ioutil.TempFile("", "pmm-agent-process-test-child")
 		require.NoError(t, err)
 		require.NoError(t, f.Close())
 		defer func() {

agents/supervisor/supervisor.go

@@ -19,6 +19,8 @@ package supervisor
 import (
 	"context"
 	"fmt"
+	"io"
+	"os"
 	"path/filepath"
 	"regexp"
 	"runtime/pprof"
@@ -43,6 +45,7 @@ import (
 	"github.com/percona/pmm-agent/agents/postgres/pgstatstatements"
 	"github.com/percona/pmm-agent/agents/process"
 	"github.com/percona/pmm-agent/config"
+	"github.com/percona/pmm-agent/storelogs"
 	"github.com/percona/pmm-agent/utils/templates"
 )
 
@@ -70,6 +73,7 @@ type agentProcessInfo struct {
 	done           <-chan struct{} // closes when Process.Changes() channel closes
 	requestedState *agentpb.SetStateRequest_AgentProcess
 	listenPort     uint16
+	logs           *storelogs.LogsStore // store logs
 }
 
 // builtinAgentInfo describes built-in Agent.
@@ -79,6 +83,7 @@ type builtinAgentInfo struct {
 	requestedState *agentpb.SetStateRequest_BuiltinAgent
 	describe       func(chan<- *prometheus.Desc)  // agent's func to describe Prometheus metrics
 	collect        func(chan<- prometheus.Metric) // agent's func to provide Prometheus metrics
+	logs           *storelogs.LogsStore           // store logs
 }
 
 // NewSupervisor creates new Supervisor object.
@@ -141,6 +146,25 @@ func (s *Supervisor) AgentsList() []*agentlocalpb.AgentInfo {
 	return res
 }
 
+// AgentsLogs returns logs for all Agents managed by this supervisor.
+func (s *Supervisor) AgentsLogs() map[string][]string {
+	s.rw.RLock()
+	defer s.rw.RUnlock()
+	s.arw.RLock()
+	defer s.arw.RUnlock()
+
+	var res map[string][]string
+
+	for _, agent := range s.agentProcesses {
+		res[agent.requestedState.Type.String()] = agent.logs.GetLogs()
+	}
+
+	for _, agent := range s.builtinAgents {
+		res[agent.requestedState.Type.String()] = agent.logs.GetLogs()
+	}
+	return res
+}
+
 // Changes returns channel with Agent's state changes.
 func (s *Supervisor) Changes() <-chan *agentpb.StateChangedRequest {
 	return s.changes
@@ -325,8 +349,9 @@ func filter(existing, new map[string]agentpb.AgentParams) (toStart, toRestart, t
 
 //nolint:golint
 const (
-	type_TEST_SLEEP inventorypb.AgentType = 998 // process
-	type_TEST_NOOP  inventorypb.AgentType = 999 // built-in
+	type_TEST_SLEEP  inventorypb.AgentType = 998 // process
+	type_TEST_NOOP   inventorypb.AgentType = 999 // built-in
+	COUNT_AGENT_LOGS                       = 10
 )
 
 // startProcess starts Agent's process.
@@ -339,13 +364,8 @@ func (s *Supervisor) startProcess(agentID string, agentProcess *agentpb.SetState
 
 	ctx, cancel := context.WithCancel(s.ctx)
 	agentType := strings.ToLower(agentProcess.Type.String())
-	l := logrus.WithFields(logrus.Fields{
-		"component": "agent-process",
-		"agentID":   agentID,
-		"type":      agentType,
-	})
+	ringLog, l := s.newLogger("agent-process", agentID, agentType)
 	l.Debugf("Starting: %s.", processParams)
-
 	process := process.New(processParams, agentProcess.RedactWords, l)
 	go pprof.Do(ctx, pprof.Labels("agentID", agentID, "type", agentType), process.Run)
 
@@ -368,20 +388,30 @@ func (s *Supervisor) startProcess(agentID string, agentProcess *agentpb.SetState
 		done:           done,
 		requestedState: proto.Clone(agentProcess).(*agentpb.SetStateRequest_AgentProcess),
 		listenPort:     port,
+		logs:           ringLog,
 	}
 	return nil
 }
 
+func (s *Supervisor) newLogger(component string, agentID string, agentType string) (*storelogs.LogsStore, *logrus.Entry) {
+	ringLog := storelogs.New(COUNT_AGENT_LOGS)
+	logger := logrus.New()
+	logger.SetFormatter(logrus.StandardLogger().Formatter)
+	logger.Out = io.MultiWriter(os.Stderr, ringLog)
+	l := logger.WithFields(logrus.Fields{
+		"component": component,
+		"agentID":   agentID,
+		"type":      agentType,
+	})
+	return ringLog, l
+}
+
 // startBuiltin starts built-in Agent.
 // Must be called with s.rw held for writing.
 func (s *Supervisor) startBuiltin(agentID string, builtinAgent *agentpb.SetStateRequest_BuiltinAgent) error {
 	ctx, cancel := context.WithCancel(s.ctx)
 	agentType := strings.ToLower(builtinAgent.Type.String())
-	l := logrus.WithFields(logrus.Fields{
-		"component": "agent-builtin",
-		"agentID":   agentID,
-		"type":      agentType,
-	})
+	ringLog, l := s.newLogger("agent-process", agentID, agentType)
 
 	done := make(chan struct{})
 	var agent agents.BuiltinAgent
@@ -486,6 +516,7 @@ func (s *Supervisor) startBuiltin(agentID string, builtinAgent *agentpb.SetState
 		requestedState: proto.Clone(builtinAgent).(*agentpb.SetStateRequest_BuiltinAgent),
 		describe:       agent.Describe,
 		collect:        agent.Collect,
+		logs:           ringLog,
 	}
 	return nil
 }

commands/run.go

@@ -28,9 +28,12 @@ import (
 	"github.com/percona/pmm-agent/client"
 	"github.com/percona/pmm-agent/config"
 	"github.com/percona/pmm-agent/connectionchecker"
+	"github.com/percona/pmm-agent/storelogs"
 	"github.com/percona/pmm-agent/versioner"
 )
 
+const COUNT_SERVER_LOGS = 500
+
 // Run implements `pmm-agent run` default command.
 func Run() {
 	l := logrus.WithField("component", "main")
@@ -40,6 +43,7 @@ func Run() {
 	// handle termination signals
 	signals := make(chan os.Signal, 1)
 	signal.Notify(signals, unix.SIGTERM, unix.SIGINT)
+	ringLog := storelogs.New(COUNT_SERVER_LOGS)
 	go func() {
 		s := <-signals
 		signal.Stop(signals)
@@ -55,7 +59,7 @@ func Run() {
 		config.ConfigureLogger(cfg)
 		l.Debugf("Loaded configuration: %+v", cfg)
 
-		run(ctx, cfg, configFilepath)
+		run(ctx, cfg, configFilepath, ringLog)
 
 		if ctx.Err() != nil {
 			return
@@ -65,7 +69,7 @@ func Run() {
 
 // run runs all pmm-agent components with given configuration until ctx is cancellled.
 // See documentation for NewXXX, Run, and Done
-func run(ctx context.Context, cfg *config.Config, configFilepath string) {
+func run(ctx context.Context, cfg *config.Config, configFilepath string, ringLog *storelogs.LogsStore) {
 	var cancel context.CancelFunc
 	ctx, cancel = context.WithCancel(ctx)
 
@@ -77,7 +81,8 @@ func run(ctx context.Context, cfg *config.Config, configFilepath string) {
 	connectionChecker := connectionchecker.New(&cfg.Paths)
 	v := versioner.New(&versioner.RealExecFunctions{})
 	client := client.New(cfg, supervisor, connectionChecker, v)
-	localServer := agentlocal.NewServer(cfg, supervisor, client, configFilepath)
+
+	localServer := agentlocal.NewServer(cfg, supervisor, client, configFilepath, ringLog)
 
 	go func() {
 		_ = client.Run(ctx)