PMM-4879 Adding support for defaults-file in new mysql service.

api-tests/management/mysql_test.go

@@ -596,6 +596,7 @@ func TestAddMySQL(t *testing.T) {
 		assert.Nil(t, addMySQLOK)
 	})
 
+	// according to defaultFiles parameter, passing empty username is allowed
 	t.Run("Empty username", func(t *testing.T) {
 		nodeName := pmmapitests.TestString(t, "node-name")
 		nodeID, pmmAgentID := RegisterGenericNode(t, node.RegisterNodeBody{
@@ -614,11 +615,16 @@ func TestAddMySQL(t *testing.T) {
 				Address:     "10.10.10.10",
 				Port:        3306,
 				PMMAgentID:  pmmAgentID,
+
+				SkipConnectionCheck: true,
 			},
 		}
 		addMySQLOK, err := client.Default.MySQL.AddMySQL(params)
-		pmmapitests.AssertAPIErrorf(t, err, 400, codes.InvalidArgument, "invalid field Username: value '' must not be an empty string")
-		assert.Nil(t, addMySQLOK)
+		require.NoError(t, err)
+		require.NotNil(t, addMySQLOK)
+		require.NotNil(t, addMySQLOK.Payload.Service)
+		serviceID := addMySQLOK.Payload.Service.ServiceID
+		defer pmmapitests.RemoveServices(t, serviceID)
 	})
 
 	t.Run("With MetricsModePush", func(t *testing.T) {

go.mod

@@ -4,6 +4,8 @@ go 1.18
 
 // Use for local development, but do not commit:
 // replace github.com/percona/pmm => ../pmm
+replace github.com/percona/pmm => github.com/pkadej/pmm v0.0.0-20220427173424-9aa771bbf8c1
+
 // replace github.com/percona-platform/saas => ../saas
 // replace github.com/percona-platform/dbaas-api => ../dbaas-api
 

go.sum

@@ -465,13 +465,13 @@ github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472 h1:Henk
 github.com/percona-platform/dbaas-api v0.0.0-20220110092915-5aacd784d472/go.mod h1:WZZ3Hi+lAWCaGWmsrfkkvRQPkIa8n1OZ0s8Su+vbgus=
 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16 h1:0fx16uGtl4MwrBwm9/VSoNEhjL0cXYxS0quEhLthGcc=
 github.com/percona-platform/saas v0.0.0-20220427162947-f9d246ad0f16/go.mod h1:gFUwaFp6Ugu5qsBwiOVJYbDlzgZ77tmXdXGO7tG5xVI=
-github.com/percona/pmm v0.0.0-20220427122756-c52cc545e1f3 h1:6IKHKNDNJADEN/YsvXcUiJ8M75/PW+bJZ3kMb+CL7kM=
-github.com/percona/pmm v0.0.0-20220427122756-c52cc545e1f3/go.mod h1:FFbpyZXm+Ilg8M/GewuhdjJhPaW1/v1zjRRerYhCfEA=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54 h1:aI1emmycDTGWKsBdxFPKZqohfBbK4y2ta9G4+RX7gVg=
 github.com/percona/promconfig v0.2.4-0.20211110115058-98687f586f54/go.mod h1:Y2uXi5QNk71+ceJHuI9poank+0S1kjxd3K105fXKVkg=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pkadej/pmm v0.0.0-20220427173424-9aa771bbf8c1 h1:O8OIMYgFTptOkFihK2xCUEoXBX5N0As7p47k+X6qk9s=
+github.com/pkadej/pmm v0.0.0-20220427173424-9aa771bbf8c1/go.mod h1:FFbpyZXm+Ilg8M/GewuhdjJhPaW1/v1zjRRerYhCfEA=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=

main.go

@@ -136,6 +136,7 @@ type gRPCServerDeps struct {
 	actions              *agents.ActionsService
 	agentsStateUpdater   *agents.StateUpdater
 	connectionCheck      *agents.ConnectionChecker
+	defaultsFileParser   *agents.DefaultsFileParser
 	grafanaClient        *grafana.Client
 	checksService        *checks.Service
 	dbaasClient          *dbaas.Client
@@ -188,7 +189,7 @@ func runGRPCServer(ctx context.Context, deps *gRPCServerDeps) {
 
 	nodeSvc := management.NewNodeService(deps.db)
 	serviceSvc := management.NewServiceService(deps.db, deps.agentsStateUpdater, deps.vmdb)
-	mysqlSvc := management.NewMySQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck, deps.versionCache)
+	mysqlSvc := management.NewMySQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck, deps.versionCache, deps.defaultsFileParser)
 	mongodbSvc := management.NewMongoDBService(deps.db, deps.agentsStateUpdater, deps.connectionCheck)
 	postgresqlSvc := management.NewPostgreSQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck)
 	proxysqlSvc := management.NewProxySQLService(deps.db, deps.agentsStateUpdater, deps.connectionCheck)
@@ -729,6 +730,7 @@ func main() {
 	schedulerService := scheduler.New(db, backupService)
 	versionCache := versioncache.New(db, versioner)
 	emailer := alertmanager.NewEmailer(logrus.WithField("component", "alertmanager-emailer").Logger)
+	defaultsFileParser := agents.NewDefaultsFileParser(agentsRegistry)
 
 	serverParams := &server.Params{
 		DB:                   db,
@@ -929,6 +931,7 @@ func main() {
 				versionCache:         versionCache,
 				supervisord:          supervisord,
 				config:               &cfg.Config,
+				defaultsFileParser:   defaultsFileParser,
 			})
 	}()
 

models/defaults_file.go

@@ -0,0 +1,26 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package models
+
+// ParseDefaultsFileResult contains result of parsing defaults file.
+type ParseDefaultsFileResult struct {
+	Username string
+	Password string
+	Host     string
+	Port     uint32
+	Socket   string
+}

services/agents/channel/channel.go

@@ -275,6 +275,8 @@ func (c *Channel) runReceiver() {
 			c.publish(msg.Id, msg.Status, p.GetVersions)
 		case *agentpb.AgentMessage_PbmSwitchPitr:
 			c.publish(msg.Id, msg.Status, p.PbmSwitchPitr)
+		case *agentpb.AgentMessage_ParseDefaultsFile:
+			c.publish(msg.Id, msg.Status, p.ParseDefaultsFile)
 
 		case nil:
 			c.cancel(msg.Id, errors.Errorf("unimplemented: failed to handle received message %s", msg))

services/agents/channel/channel_test.go

@@ -356,3 +356,50 @@ func TestUnexpectedResponsePayloadFromAgent(t *testing.T) {
 	close(stopServer)
 	<-stop
 }
+
+func TestChannelForDefaultsFileParser(t *testing.T) {
+	const count = 50
+	require.True(t, count > agentRequestsCap)
+
+	connect := func(ch *Channel) error {
+		testValue := "test"
+		testPort := uint32(123123)
+		for i := uint32(1); i <= count; i++ {
+			resp, err := ch.SendAndWaitResponse(&agentpb.ParseDefaultsFileRequest{})
+			assert.NotNil(t, resp)
+			parserResponse := resp.(*agentpb.ParseDefaultsFileResponse)
+			assert.Equal(t, parserResponse.Username, testValue)
+			assert.Equal(t, parserResponse.Password, testValue)
+			assert.Equal(t, parserResponse.Socket, testValue)
+			assert.Equal(t, parserResponse.Port, testPort)
+			assert.NoError(t, err)
+		}
+
+		assert.Nil(t, <-ch.Requests())
+		return nil
+	}
+
+	stream, _, teardown := setup(t, connect, io.EOF) // EOF = server exits from handler
+	defer teardown(t)
+
+	for i := uint32(1); i <= count; i++ {
+		msg, err := stream.Recv()
+		assert.NoError(t, err)
+		assert.Equal(t, i, msg.Id)
+		assert.NotNil(t, msg.GetParseDefaultsFile())
+
+		err = stream.Send(&agentpb.AgentMessage{
+			Id: i,
+			Payload: (&agentpb.ParseDefaultsFileResponse{
+				Username: "test",
+				Password: "test",
+				Port:     123123,
+				Socket:   "test",
+			}).AgentMessageResponsePayload(),
+		})
+		assert.NoError(t, err)
+	}
+
+	err := stream.CloseSend()
+	assert.NoError(t, err)
+}

services/agents/parse_defaults_file.go

@@ -0,0 +1,98 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package agents
+
+import (
+	"context"
+	"time"
+
+	"github.com/percona/pmm/api/agentpb"
+	"github.com/percona/pmm/api/inventorypb"
+	"github.com/pkg/errors"
+
+	"github.com/percona/pmm-managed/models"
+	"github.com/percona/pmm-managed/utils/logger"
+)
+
+// DefaultsFileParser requests from agent to parse defaultsFile.
+type DefaultsFileParser struct {
+	r *Registry
+}
+
+// NewDefaultsFileParser creates new ParseDefaultsFile request.
+func NewDefaultsFileParser(r *Registry) *DefaultsFileParser {
+	return &DefaultsFileParser{
+		r: r,
+	}
+}
+
+// ParseDefaultsFile sends request (with file path) to pmm-agent to parse defaults file.
+func (p *DefaultsFileParser) ParseDefaultsFile(ctx context.Context, pmmAgentID, filePath string, serviceType models.ServiceType) (*models.ParseDefaultsFileResult, error) {
+	l := logger.Get(ctx)
+
+	pmmAgent, err := p.r.get(pmmAgentID)
+	if err != nil {
+		return nil, err
+	}
+
+	start := time.Now()
+	defer func() {
+		if dur := time.Since(start); dur > 5*time.Second {
+			l.Warnf("ParseDefaultsFile took %s.", dur)
+		}
+	}()
+
+	request, err := createRequest(filePath, serviceType)
+	if err != nil {
+		l.Debugf("can't create ParseDefaultsFileRequest %s", err)
+		return nil, err
+	}
+
+	resp, err := pmmAgent.channel.SendAndWaitResponse(request)
+	if err != nil {
+		return nil, err
+	}
+
+	l.Infof("ParseDefaultsFile response from agent: %+v.", resp)
+	parserResponse, ok := resp.(*agentpb.ParseDefaultsFileResponse)
+	if !ok {
+		return nil, errors.New("wrong response from agent (not ParseDefaultsFileResponse model)")
+	}
+	if parserResponse.Error != "" {
+		return nil, errors.New(parserResponse.Error)
+	}
+
+	return &models.ParseDefaultsFileResult{
+		Username: parserResponse.Username,
+		Password: parserResponse.Password,
+		Host:     parserResponse.Host,
+		Port:     parserResponse.Port,
+		Socket:   parserResponse.Socket,
+	}, nil
+}
+
+func createRequest(configPath string, serviceType models.ServiceType) (*agentpb.ParseDefaultsFileRequest, error) {
+	if serviceType == models.MySQLServiceType {
+		request := &agentpb.ParseDefaultsFileRequest{
+			ServiceType: inventorypb.ServiceType_MYSQL_SERVICE,
+			ConfigPath:  configPath,
+		}
+		return request, nil
+	} else {
+		return nil, errors.Errorf("unhandled service type %s", serviceType)
+	}
+}

services/agents/parse_defaults_file_test.go

@@ -0,0 +1,41 @@
+// pmm-managed
+// Copyright (C) 2017 Percona LLC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+package agents
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/percona/pmm-managed/models"
+)
+
+func TestCreateRequest(t *testing.T) {
+	t.Parallel()
+	response, err := createRequest("/path/to/file", models.MySQLServiceType)
+
+	require.NoError(t, err)
+	require.NotNil(t, response, "ParseDefaultsFileRequest is nil")
+}
+
+func TestCreateRequestNotSupported(t *testing.T) {
+	t.Parallel()
+	response, err := createRequest("/path/to/file", models.PostgreSQLServiceType)
+
+	require.Error(t, err)
+	require.Nil(t, response, "ParseDefaultsFileRequest is not nil")
+}

services/management/deps.go

@@ -34,6 +34,8 @@ import (
 //go:generate mockery -name=grafanaClient -case=snake -inpkg -testonly
 //go:generate mockery -name=jobsService -case=snake -inpkg -testonly
 //go:generate mockery -name=connectionChecker -case=snake -inpkg -testonly
+//go:generate mockery -name=defaultsFileParser -case=snake -inpkg -testonly
+//go:generate mockery -name=versionCache -case=snake -inpkg -testonly
 
 // agentsRegistry is a subset of methods of agents.Registry used by this package.
 // We use it instead of real type for testing and to avoid dependency cycle.
@@ -93,3 +95,9 @@ type connectionChecker interface {
 type versionCache interface {
 	RequestSoftwareVersionsUpdate()
 }
+
+// defaultsFileParser is a subset of methods of agents.ParseDefaultsFile.
+// We use it instead of real type for testing and to avoid dependency cycle.
+type defaultsFileParser interface {
+	ParseDefaultsFile(ctx context.Context, pmmAgentID, filePath string, serviceType models.ServiceType) (*models.ParseDefaultsFileResult, error)
+}