Skip to content

Commit

Permalink
PMM-4879 mysql add defaults file (#928)
Browse files Browse the repository at this point in the history
* PMM-4879 Added defaults file.

* PMM-4879 Added AgentMessage and ServerMessage: ParseDefaultsFiile.

* PMM-4879 Updating agent.go with ParseDefaultsFile request/responses.

* PMM-4879 Updating agents ParseDefaultsFileRequest with ServiceType field.

* PMM-4879 Error field in ParseDefaultsFileResponse.

* PMM-4879 Regenerate files after go update to 1.18.

* PMM-4879 Adding new fields to agent response ParseDefaultsFile.

* PMM-4879 Removing username validation and adding socket to ParseDefaultsFileResponse

* PMM-4879 Added support for defaultsfile parameter (pmm-admin).

* PMM-4879 Added support for defaultsfile parameter (pmm-agent).

Co-authored-by: Alexey Mukas <[email protected]>
  • Loading branch information
pkadej and ritbl authored Jun 11, 2022
1 parent d98959b commit e47bf13
Show file tree
Hide file tree
Showing 22 changed files with 1,549 additions and 932 deletions.
30 changes: 28 additions & 2 deletions admin/commands/management/add_mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,7 @@ type addMySQLCommand struct {
ServiceName string
Username string
Password string
DefaultsFile string
AgentPassword string
CredentialsSource string
Environment string
Expand Down Expand Up @@ -127,9 +128,17 @@ func (cmd *addMySQLCommand) GetAddress() string {
}

func (cmd *addMySQLCommand) GetDefaultAddress() string {
if cmd.DefaultsFile != "" {
// address might be specified in defaults file
return ""
}
return "127.0.0.1:3306"
}

func (cmd *addMySQLCommand) GetDefaultUsername() string {
return "root"
}

func (cmd *addMySQLCommand) GetSocket() string {
return cmd.Socket
}
Expand Down Expand Up @@ -194,6 +203,8 @@ func (cmd *addMySQLCommand) Run() (commands.Result, error) {
return nil, err
}

username := defaultsFileUsernameCheck(cmd)

tablestatsGroupTableLimit := int32(cmd.DisableTablestatsLimit)
if cmd.DisableTablestats {
if tablestatsGroupTableLimit != 0 {
Expand All @@ -220,7 +231,7 @@ func (cmd *addMySQLCommand) Run() (commands.Result, error) {
Environment: cmd.Environment,
Cluster: cmd.Cluster,
ReplicationSet: cmd.ReplicationSet,
Username: cmd.Username,
Username: username,
Password: cmd.Password,
AgentPassword: cmd.AgentPassword,
CustomLabels: customLabels,
Expand Down Expand Up @@ -273,8 +284,9 @@ func init() {
AddMySQLC.Flag("node-id", "Node ID (default is autodetected)").StringVar(&AddMySQL.NodeID)
AddMySQLC.Flag("pmm-agent-id", "The pmm-agent identifier which runs this instance (default is autodetected)").StringVar(&AddMySQL.PMMAgentID)

AddMySQLC.Flag("username", "MySQL username").Default("root").StringVar(&AddMySQL.Username)
AddMySQLC.Flag("username", "MySQL username").StringVar(&AddMySQL.Username)
AddMySQLC.Flag("password", "MySQL password").StringVar(&AddMySQL.Password)
AddMySQLC.Flag("defaults-file", "Path to defaults file").StringVar(&AddMySQL.DefaultsFile)
AddMySQLC.Flag("agent-password", "Custom password for /metrics endpoint").StringVar(&AddMySQL.AgentPassword)
AddMySQLC.Flag("credentials-source", "Credentials provider").ExistingFileVar(&AddMySQL.CredentialsSource)

Expand Down Expand Up @@ -307,3 +319,17 @@ func init() {
AddMySQLC.Flag("disable-collectors", "Comma-separated list of collector names to exclude from exporter").StringVar(&AddMySQL.DisableCollectors)
addGlobalFlags(AddMySQLC)
}

func defaultsFileUsernameCheck(cmd *addMySQLCommand) string {
// defaults file specified, but passed username has higher priority
if cmd.Username != "" && cmd.DefaultsFile != "" {
return cmd.Username
}

// username not specified, but can be in defaults files
if cmd.Username == "" && cmd.DefaultsFile != "" {
return ""
}

return cmd.GetDefaultUsername()
}
5 changes: 5 additions & 0 deletions agent/client/channel/channel.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,11 @@ func (c *Channel) runReceiver() {
ID: msg.Id,
Payload: p.PbmSwitchPitr,
}
case *agentpb.ServerMessage_ParseDefaultsFile:
c.requests <- &ServerRequest{
ID: msg.Id,
Payload: p.ParseDefaultsFile,
}

// responses
case *agentpb.ServerMessage_Pong:
Expand Down
30 changes: 17 additions & 13 deletions agent/client/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,10 +55,11 @@ const (

// Client represents pmm-agent's connection to nginx/pmm-managed.
type Client struct {
cfg *config.Config
supervisor supervisor
connectionChecker connectionChecker
softwareVersioner softwareVersioner
cfg *config.Config
supervisor supervisor
connectionChecker connectionChecker
softwareVersioner softwareVersioner
defaultsFileParser defaultsFileParser

l *logrus.Entry
backoff *backoff.Backoff
Expand All @@ -78,16 +79,17 @@ type Client struct {
// New creates new client.
//
// Caller should call Run.
func New(cfg *config.Config, supervisor supervisor, connectionChecker connectionChecker, sv softwareVersioner) *Client {
func New(cfg *config.Config, supervisor supervisor, connectionChecker connectionChecker, sv softwareVersioner, dfp defaultsFileParser) *Client {
return &Client{
cfg: cfg,
supervisor: supervisor,
connectionChecker: connectionChecker,
softwareVersioner: sv,
l: logrus.WithField("component", "client"),
backoff: backoff.New(backoffMinDelay, backoffMaxDelay),
done: make(chan struct{}),
dialTimeout: dialTimeout,
cfg: cfg,
supervisor: supervisor,
connectionChecker: connectionChecker,
softwareVersioner: sv,
l: logrus.WithField("component", "client"),
backoff: backoff.New(backoffMinDelay, backoffMaxDelay),
done: make(chan struct{}),
dialTimeout: dialTimeout,
defaultsFileParser: dfp,
}
}

Expand Down Expand Up @@ -436,6 +438,8 @@ func (c *Client) processChannelRequests(ctx context.Context) {
resp.Error = err.Error()
}
responsePayload = &resp
case *agentpb.ParseDefaultsFileRequest:
responsePayload = c.defaultsFileParser.ParseDefaultsFile(p)
default:
c.l.Errorf("Unhandled server request: %v.", req)
}
Expand Down
14 changes: 7 additions & 7 deletions agent/client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,7 @@ func TestClient(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())

cfg := &config.Config{}
client := New(cfg, nil, nil, nil)
client := New(cfg, nil, nil, nil, nil)
cancel()
err := client.Run(ctx)
assert.EqualError(t, err, "missing PMM Server address: context canceled")
Expand All @@ -95,7 +95,7 @@ func TestClient(t *testing.T) {
Address: "127.0.0.1:1",
},
}
client := New(cfg, nil, nil, nil)
client := New(cfg, nil, nil, nil, nil)
cancel()
err := client.Run(ctx)
assert.EqualError(t, err, "missing Agent ID: context canceled")
Expand All @@ -112,7 +112,7 @@ func TestClient(t *testing.T) {
Address: "127.0.0.1:1",
},
}
client := New(cfg, nil, nil, nil)
client := New(cfg, nil, nil, nil, nil)
err := client.Run(ctx)
assert.EqualError(t, err, "failed to dial: context deadline exceeded")
})
Expand Down Expand Up @@ -158,7 +158,7 @@ func TestClient(t *testing.T) {
s.On("Changes").Return(make(<-chan *agentpb.StateChangedRequest))
s.On("QANRequests").Return(make(<-chan *agentpb.QANCollectRequest))

client := New(cfg, &s, nil, nil)
client := New(cfg, &s, nil, nil, nil)
err := client.Run(context.Background())
assert.NoError(t, err)
assert.Equal(t, serverMD, client.GetServerConnectMetadata())
Expand Down Expand Up @@ -186,7 +186,7 @@ func TestClient(t *testing.T) {
},
}

client := New(cfg, nil, nil, nil)
client := New(cfg, nil, nil, nil, nil)
client.dialTimeout = 100 * time.Millisecond
err := client.Run(ctx)
assert.EqualError(t, err, "failed to get server metadata: rpc error: code = Canceled desc = context canceled", "%+v", err)
Expand Down Expand Up @@ -214,7 +214,7 @@ func TestGetActionTimeout(t *testing.T) {
for _, tc := range testCases {
tc := tc
t.Run(prototext.Format(tc.req), func(t *testing.T) {
client := New(nil, nil, nil, nil)
client := New(nil, nil, nil, nil, nil)
actual := client.getActionTimeout(tc.req)
assert.Equal(t, tc.expected, actual)
})
Expand Down Expand Up @@ -273,7 +273,7 @@ func TestUnexpectedActionType(t *testing.T) {
s.On("Changes").Return(make(<-chan *agentpb.StateChangedRequest))
s.On("QANRequests").Return(make(<-chan *agentpb.QANCollectRequest))

client := New(cfg, s, nil, nil)
client := New(cfg, s, nil, nil, nil)
err := client.Run(context.Background())
assert.NoError(t, err)
assert.Equal(t, serverMD, client.GetServerConnectMetadata())
Expand Down
4 changes: 4 additions & 0 deletions agent/client/deps.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

//go:generate ../../bin/mockery -name=connectionChecker -case=snake -inpkg -testonly
//go:generate ../../bin/mockery -name=supervisor -case=snake -inpkg -testonly
//go:generate ../../bin/mockery -name=defaultsFileParser -case=snake -inpkg -testonly

// connectionChecker is a subset of methods of connectionchecker.ConnectionChecker used by this package.
// We use it instead of real type for testing and to avoid dependency cycle.
Expand All @@ -49,3 +50,6 @@ type supervisor interface {
// Collector added to use client as Prometheus collector
prometheus.Collector
}
type defaultsFileParser interface {
ParseDefaultsFile(req *agentpb.ParseDefaultsFileRequest) *agentpb.ParseDefaultsFileResponse
}
4 changes: 3 additions & 1 deletion agent/commands/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/percona/pmm/agent/client"
"github.com/percona/pmm/agent/config"
"github.com/percona/pmm/agent/connectionchecker"
"github.com/percona/pmm/agent/defaultsfile"
"github.com/percona/pmm/agent/versioner"
)

Expand Down Expand Up @@ -75,8 +76,9 @@ func run(ctx context.Context, cfg *config.Config, configFilepath string) {

supervisor := supervisor.NewSupervisor(ctx, &cfg.Paths, &cfg.Ports, &cfg.Server)
connectionChecker := connectionchecker.New(&cfg.Paths)
defaultsFileParser := defaultsfile.New()
v := versioner.New(&versioner.RealExecFunctions{})
client := client.New(cfg, supervisor, connectionChecker, v)
client := client.New(cfg, supervisor, connectionChecker, v, defaultsFileParser)
localServer := agentlocal.NewServer(cfg, supervisor, client, configFilepath)

go func() {
Expand Down
132 changes: 132 additions & 0 deletions agent/defaultsfile/defaults_file.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
// pmm-agent
// Copyright 2019 Percona LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package defaultsfile provides managing of defaults file.
package defaultsfile

import (
"fmt"
"os/user"
"path/filepath"
"strings"

"github.com/pkg/errors"
"gopkg.in/ini.v1"

"github.com/percona/pmm/api/agentpb"
"github.com/percona/pmm/api/inventorypb"
)

// Parser is a struct which is responsible for parsing defaults file.
type Parser struct{}

// New creates new DefaultsFileParser.
func New() *Parser {
return &Parser{}
}

type defaultsFile struct {
username string
password string
host string
port uint32
socket string
}

// ParseDefaultsFile parses given defaultsFile in request. It returns the database specs.
func (d *Parser) ParseDefaultsFile(req *agentpb.ParseDefaultsFileRequest) *agentpb.ParseDefaultsFileResponse {
var res agentpb.ParseDefaultsFileResponse
defaultsFile, err := parseDefaultsFile(req.ConfigPath, req.ServiceType)
if err != nil {
res.Error = err.Error()
return &res
}

res.Username = defaultsFile.username
res.Password = defaultsFile.password
res.Host = defaultsFile.host
res.Port = defaultsFile.port
res.Socket = defaultsFile.socket

return &res
}

func parseDefaultsFile(configPath string, serviceType inventorypb.ServiceType) (*defaultsFile, error) {
if len(configPath) == 0 {
return nil, errors.New("configPath for DefaultsFile is empty")
}

switch serviceType {
case inventorypb.ServiceType_MYSQL_SERVICE:
return parseMySQLDefaultsFile(configPath)
case inventorypb.ServiceType_EXTERNAL_SERVICE:
case inventorypb.ServiceType_HAPROXY_SERVICE:
case inventorypb.ServiceType_MONGODB_SERVICE:
case inventorypb.ServiceType_POSTGRESQL_SERVICE:
case inventorypb.ServiceType_PROXYSQL_SERVICE:
case inventorypb.ServiceType_SERVICE_TYPE_INVALID:
return nil, errors.Errorf("unimplemented service type %s", serviceType)
}

return nil, errors.Errorf("unimplemented service type %s", serviceType)
}

func parseMySQLDefaultsFile(configPath string) (*defaultsFile, error) {
configPath, err := expandPath(configPath)
if err != nil {
return nil, fmt.Errorf("fail to normalize path: %w", err)
}

cfg, err := ini.Load(configPath)
if err != nil {
return nil, fmt.Errorf("fail to read config file: %w", err)
}

cfgSection := cfg.Section("client")
port, _ := cfgSection.Key("port").Uint()

parsedData := &defaultsFile{
username: cfgSection.Key("user").String(),
password: cfgSection.Key("password").String(),
host: cfgSection.Key("host").String(),
port: uint32(port),
socket: cfgSection.Key("socket").String(),
}

err = validateDefaultsFileResults(parsedData)
if err != nil {
return nil, err
}

return parsedData, nil
}

func validateDefaultsFileResults(data *defaultsFile) error {
if data.username == "" && data.password == "" && data.host == "" && data.port == 0 && data.socket == "" {
return errors.New("no data found in defaults file")
}
return nil
}

func expandPath(path string) (string, error) {
if strings.HasPrefix(path, "~/") {
usr, err := user.Current()
if err != nil {
return "", fmt.Errorf("failed to expand path: %w", err)
}
return filepath.Join(usr.HomeDir, path[2:]), nil
}
return path, nil
}
Loading

0 comments on commit e47bf13

Please sign in to comment.