diff --git a/VERSION b/VERSION index d33c3a212..51de3305b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.12.0 \ No newline at end of file +0.13.0 \ No newline at end of file diff --git a/agent/agent.go b/agent/agent.go index f3678b9dd..a0e580dc4 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -150,7 +150,7 @@ func (a *orbAgent) Restart(fullReset bool, reason string) { a.logger.Info("restarting all backends", zap.String("reason", reason)) for name, be := range a.backends { a.logger.Info("removing policies", zap.String("backend", name)) - if err := a.policyManager.RemoveBackendPolicies(be); err != nil { + if err := a.policyManager.RemoveBackendPolicies(be, false); err != nil { a.logger.Error("failed to remove policies", zap.String("backend", name), zap.Error(err)) } a.logger.Info("resetting backend", zap.String("backend",name)) diff --git a/agent/backend/pktvisor/pktvisor.go b/agent/backend/pktvisor/pktvisor.go index 805809c15..b5616675d 100644 --- a/agent/backend/pktvisor/pktvisor.go +++ b/agent/backend/pktvisor/pktvisor.go @@ -477,13 +477,25 @@ func createReceiver(ctx context.Context, exporter component.MetricsExporter, log } func (p *pktvisorBackend) FullReset() error { - if err := p.Stop(); err != nil { - p.logger.Error("failed to stop backend on restart procedure", zap.Error(err)) - return err + + // State always will have a value, even if error is not null + // State it's been used to identify broken pktvisor + state, errMsg, err := p.GetState() + if err != nil { + p.logger.Warn("broken pktvisor, trying to start", zap.String("broken_reason", errMsg)) } - if err := p.Start(); err != nil { - p.logger.Error("failed to start backend on restart procedure", zap.Error(err)) - return err + + if state == backend.Running { + if err := p.Stop(); err != nil { + p.logger.Error("failed to stop backend on restart procedure", zap.Error(err)) + return err + } + } else { + if err := p.Start(); err != nil { + p.logger.Error("failed to start backend on restart procedure", zap.Error(err)) + return err + } } + return nil } diff --git a/agent/cloud_config/cloud_config.go b/agent/cloud_config/cloud_config.go index 1f5b8d079..962ab1db8 100644 --- a/agent/cloud_config/cloud_config.go +++ b/agent/cloud_config/cloud_config.go @@ -121,7 +121,8 @@ func (cc *cloudConfigManager) autoProvision(apiAddress string, token string) (co } type AgentReq struct { - Name string `json:"name"` + Name string `json:"name"` + AgentTags map[string]string `json:"agent_tags"` } aname := cc.config.OrbAgent.Cloud.Config.AgentName @@ -133,7 +134,7 @@ func (cc *cloudConfigManager) autoProvision(apiAddress string, token string) (co aname = hostname } - agentReq := AgentReq{Name: strings.Replace(aname, ".", "-", -1)} + agentReq := AgentReq{Name: strings.Replace(aname, ".", "-", -1), AgentTags: cc.config.OrbAgent.Tags} body, err := json.Marshal(agentReq) if err != nil { return config.MQTTConfig{}, err diff --git a/agent/policyMgr/manager.go b/agent/policyMgr/manager.go index 0dce6b750..6d742fedd 100644 --- a/agent/policyMgr/manager.go +++ b/agent/policyMgr/manager.go @@ -11,6 +11,7 @@ import ( "github.com/ns1labs/orb/agent/config" "github.com/ns1labs/orb/agent/policies" "github.com/ns1labs/orb/fleet" + "github.com/ns1labs/orb/pkg/errors" "go.uber.org/zap" ) @@ -20,7 +21,8 @@ type PolicyManager interface { GetPolicyState() ([]policies.PolicyData, error) GetRepo() policies.PolicyRepo ApplyBackendPolicies(be backend.Backend) error - RemoveBackendPolicies(be backend.Backend) error + RemoveBackendPolicies(be backend.Backend, permanently bool) error + RemovePolicy(policyID string, policyName string, beName string) error } var _ PolicyManager = (*policyManager)(nil) @@ -85,7 +87,7 @@ func (a *policyManager) ManagePolicy(payload fleet.AgentPolicyRPCPayload) { a.logger.Error("failed to retrieve policy", zap.String("policy_id", payload.ID), zap.Error(err)) return } - if currentPolicy.Version >= pd.Version { + if currentPolicy.Version >= pd.Version && currentPolicy.State == policies.Running { a.logger.Info("a better version of this policy has already been applied, skipping", zap.String("policy_id", pd.ID), zap.String("policy_name", pd.Name), zap.String("attempted_version", fmt.Sprint(pd.Version)), zap.String("current_version", fmt.Sprint(currentPolicy.Version))) return } else { @@ -114,30 +116,36 @@ func (a *policyManager) ManagePolicy(payload fleet.AgentPolicyRPCPayload) { a.repo.Update(pd) return case "remove": - var pd = policies.PolicyData{ - ID: payload.ID, - Name: payload.Name, - } - if !backend.HaveBackend(payload.Backend) { - a.logger.Warn("policy remove for a backend we do not have, ignoring", zap.String("policy_id", payload.ID), zap.String("policy_name", payload.Name)) - return - } - be := backend.GetBackend(payload.Backend) - // Remove policy via http request - err := be.RemovePolicy(pd) + err := a.RemovePolicy(payload.ID, payload.Name, payload.Backend) if err != nil { - a.logger.Warn("policy failed to remove", zap.String("policy_id", payload.ID), zap.String("policy_name", payload.Name), zap.Error(err)) - } - // Remove policy from orb-agent local repo - err = a.repo.Remove(pd.ID) - if err != nil { - a.logger.Warn("policy failed to remove local", zap.String("policy_id", pd.ID), zap.String("policy_name", pd.Name), zap.Error(err)) + a.logger.Error("policy failed to be removed", zap.String("policy_id", payload.ID), zap.String("policy_name", payload.Name), zap.Error(err)) } break default: a.logger.Error("unknown policy action, ignored", zap.String("action", payload.Action)) } +} +func (a *policyManager) RemovePolicy(policyID string, policyName string, beName string) error { + var pd = policies.PolicyData{ + ID: policyID, + Name: policyName, + } + if !backend.HaveBackend(beName) { + return errors.New("policy remove for a backend we do not have, ignoring") + } + be := backend.GetBackend(beName) + // Remove policy via http request + err := be.RemovePolicy(pd) + if err != nil { + return err + } + // Remove policy from orb-agent local repo + err = a.repo.Remove(pd.ID) + if err != nil { + return err + } + return nil } func (a *policyManager) RemovePolicyDataset(policyID string, datasetID string, be backend.Backend) { @@ -178,7 +186,7 @@ func (a *policyManager) applyPolicy(payload fleet.AgentPolicyRPCPayload, be back } } -func (a *policyManager) RemoveBackendPolicies(be backend.Backend) error { +func (a *policyManager) RemoveBackendPolicies(be backend.Backend, permanently bool) error { plcies, err := a.repo.GetAll() if err != nil { a.logger.Error("failed to retrieve list of policies", zap.Error(err)) @@ -191,8 +199,12 @@ func (a *policyManager) RemoveBackendPolicies(be backend.Backend) error { a.logger.Error("failed to remove policy from backend", zap.String("policy_id", plcy.ID), zap.String("policy_name", plcy.Name), zap.Error(err)) return err } - plcy.State = policies.Unknown - a.repo.Update(plcy) + if permanently { + a.repo.Remove(plcy.ID) + } else { + plcy.State = policies.Unknown + a.repo.Update(plcy) + } } return nil } diff --git a/agent/rpc_from.go b/agent/rpc_from.go index 638c23fdc..ab73f448b 100644 --- a/agent/rpc_from.go +++ b/agent/rpc_from.go @@ -18,21 +18,58 @@ func (a *orbAgent) handleGroupMembership(rpc fleet.GroupMembershipRPCPayload) { if rpc.FullList { a.unsubscribeGroupChannels() a.subscribeGroupChannels(rpc.Groups) + err := a.sendAgentPoliciesReq() + if err != nil { + a.logger.Error("failed to send agent policies request", zap.Error(err)) + } } else { // otherwise, just add these subscriptions to the existing list a.subscribeGroupChannels(rpc.Groups) } } -func (a *orbAgent) handleAgentPolicies(rpc []fleet.AgentPolicyRPCPayload) { +func (a *orbAgent) handleAgentPolicies(rpc []fleet.AgentPolicyRPCPayload, fullList bool) { + if fullList { + policies, err := a.policyManager.GetRepo().GetAll() + if err != nil { + a.logger.Error("failed to retrieve policies on handle subscriptions") + return + } + // Create a map with all the old policies + policyRemove := map[string]bool{} + for _, p := range policies { + policyRemove[p.ID] = true + } + for _, payload := range rpc { + if ok := policyRemove[payload.ID]; ok { + policyRemove[payload.ID] = false + } + } + // Remove only the policy which should be removed + for k, v := range policyRemove { + if v == true { + policy, err := a.policyManager.GetRepo().Get(k) + if err != nil { + a.logger.Warn("failed to retrieve policy", zap.String("policy_id", k), zap.Error(err)) + continue + } + err = a.policyManager.RemovePolicy(policy.ID, policy.Name, policy.Backend) + if err != nil { + a.logger.Warn("failed to remove a policy, ignoring", zap.String("policy_id", policy.ID), zap.String("policy_name", policy.Name), zap.Error(err)) + continue + } + } + } + } for _, payload := range rpc { - a.policyManager.ManagePolicy(payload) + if payload.Action != "sanitize" { + a.policyManager.ManagePolicy(payload) + } } // heart beat with new policy status after application a.sendSingleHeartbeat(time.Now(), fleet.Online) - } func (a *orbAgent) handleGroupRPCFromCore(client mqtt.Client, message mqtt.Message) { @@ -61,7 +98,7 @@ func (a *orbAgent) handleGroupRPCFromCore(client mqtt.Client, message mqtt.Messa a.logger.Error("error decoding agent policy message from core", zap.Error(fleet.ErrSchemaMalformed)) return } - a.handleAgentPolicies(r.Payload) + a.handleAgentPolicies(r.Payload, r.FullList) case fleet.GroupRemovedRPCFunc: var r fleet.GroupRemovedRPC if err := json.Unmarshal(message.Payload(), &r); err != nil { @@ -134,7 +171,7 @@ func (a *orbAgent) handleRPCFromCore(client mqtt.Client, message mqtt.Message) { a.logger.Error("error decoding agent policy message from core", zap.Error(fleet.ErrSchemaMalformed)) return } - a.handleAgentPolicies(r.Payload) + a.handleAgentPolicies(r.Payload, r.FullList) case fleet.AgentStopRPCFunc: var r fleet.AgentStopRPC if err := json.Unmarshal(message.Payload(), &r); err != nil { diff --git a/cmd/agent/main.go b/cmd/agent/main.go index df110c5f7..8e233e64f 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -81,11 +81,11 @@ func Run(cmd *cobra.Command, args []string) { } // handle signals - sigs := make(chan os.Signal, 1) done := make(chan bool, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) go func() { + sigs := make(chan os.Signal, 1) + signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, syscall.SIGKILL) <-sigs a.Stop() done <- true diff --git a/cmd/fleet/main.go b/cmd/fleet/main.go index db768f485..34bc054b8 100644 --- a/cmd/fleet/main.go +++ b/cmd/fleet/main.go @@ -122,7 +122,10 @@ func main() { agentGroupRepo := postgres.NewAgentGroupRepository(db, logger) commsSvc := fleet.NewFleetCommsService(logger, policiesGRPCClient, agentRepo, agentGroupRepo, pubSub) - svc := newFleetService(authGRPCClient, db, logger, esClient, sdkCfg, agentRepo, agentGroupRepo, commsSvc) + + aDone := make(chan bool) + + svc := newFleetService(authGRPCClient, db, logger, esClient, sdkCfg, agentRepo, agentGroupRepo, commsSvc, aDone) defer commsSvc.Stop() errs := make(chan error, 2) @@ -141,6 +144,8 @@ func main() { c := make(chan os.Signal) signal.Notify(c, syscall.SIGINT) errs <- fmt.Errorf("%s", <-c) + //aTicker.Stop() + aDone <- true }() err = <-errs @@ -194,7 +199,7 @@ func initJaeger(svcName, url string, logger *zap.Logger) (opentracing.Tracer, io return tracer, closer } -func newFleetService(auth mainflux.AuthServiceClient, db *sqlx.DB, logger *zap.Logger, esClient *r.Client, sdkCfg config.MFSDKConfig, agentRepo fleet.AgentRepository, agentGroupRepo fleet.AgentGroupRepository, agentComms fleet.AgentCommsService) fleet.Service { +func newFleetService(auth mainflux.AuthServiceClient, db *sqlx.DB, logger *zap.Logger, esClient *r.Client, sdkCfg config.MFSDKConfig, agentRepo fleet.AgentRepository, agentGroupRepo fleet.AgentGroupRepository, agentComms fleet.AgentCommsService, aDone chan bool) fleet.Service { config := mfsdk.Config{ BaseURL: sdkCfg.BaseURL, @@ -205,7 +210,7 @@ func newFleetService(auth mainflux.AuthServiceClient, db *sqlx.DB, logger *zap.L pktvisor.Register(auth, agentRepo) - svc := fleet.NewFleetService(logger, auth, agentRepo, agentGroupRepo, agentComms, mfsdk) + svc := fleet.NewFleetService(logger, auth, agentRepo, agentGroupRepo, agentComms, mfsdk, aDone) svc = redisprod.NewEventStoreMiddleware(svc, esClient) svc = fleethttp.NewLoggingMiddleware(svc, logger) svc = fleethttp.MetricsMiddleware( diff --git a/cmd/policies/main.go b/cmd/policies/main.go index 94dcfbbec..750863347 100644 --- a/cmd/policies/main.go +++ b/cmd/policies/main.go @@ -120,6 +120,7 @@ func main() { go startHTTPServer(tracer, svc, svcCfg, logger, errs) go startGRPCServer(svc, tracer, policiesGRPCCfg, logger, errs) go subscribeToFleetES(svc, esClient, esCfg, logger) + go subscribeToSinksES(svc, esClient, esCfg, logger) go func() { c := make(chan os.Signal) @@ -275,7 +276,15 @@ func startGRPCServer(svc policies.Service, tracer opentracing.Tracer, cfg config func subscribeToFleetES(svc policies.Service, client *r.Client, cfg config.EsConfig, logger *zap.Logger) { eventStore := rediscon.NewEventStore(svc, client, cfg.Consumer, logger) logger.Info("Subscribed to Redis Event Store for agent groups") - if err := eventStore.Subscribe(context.Background()); err != nil { + if err := eventStore.SubscribeToFleet(context.Background()); err != nil { + logger.Error("Bootstrap service failed to subscribe to event sourcing", zap.Error(err)) + } +} + +func subscribeToSinksES(svc policies.Service, client *r.Client, cfg config.EsConfig, logger *zap.Logger) { + eventStore := rediscon.NewEventStore(svc, client, cfg.Consumer, logger) + logger.Info("Subscribed to Redis Event Store for sinks") + if err := eventStore.SubscribeToSink(context.Background()); err != nil { logger.Error("Bootstrap service failed to subscribe to event sourcing", zap.Error(err)) } } diff --git a/fleet/agent_group_service.go b/fleet/agent_group_service.go index 16722403d..3845efa57 100644 --- a/fleet/agent_group_service.go +++ b/fleet/agent_group_service.go @@ -13,6 +13,8 @@ import ( mfsdk "github.com/mainflux/mainflux/pkg/sdk/go" "github.com/ns1labs/orb/pkg/errors" "go.uber.org/zap" + "reflect" + "strings" ) var ( @@ -125,6 +127,35 @@ func (svc fleetService) EditAgentGroup(ctx context.Context, token string, group // append both lists and remove duplicates // need to unsubscribe the agents who are no longer matching with the group list := removeDuplicates(listSub, listUnsub) + + // connect all agents to the group channel (check the already connected and connect the new ones) + if !reflect.DeepEqual(listSub, listUnsub) { + for _, a := range listUnsub { + err = svc.mfsdk.DisconnectThing(a.MFThingID, ag.MFChannelID, token) + if err != nil { + svc.logger.Error("failed to disconnect thing", zap.String("agent_name", a.Name.String()), zap.String("agent_id", a.MFThingID), zap.Error(err)) + } + } + + for _, a := range listSub { + idList := make([]string, 1) + idList[0] = a.MFThingID + ids := mfsdk.ConnectionIDs{ + ChannelIDs: []string{ag.MFChannelID}, + ThingIDs: idList, + } + err = svc.mfsdk.Connect(ids, token) + if err != nil { + if strings.Contains(err.Error(), "409") { + svc.logger.Warn("agent already connected, skipping...") + } else { + return AgentGroup{}, err + } + } + } + + } + for _, agent := range list { err := svc.agentComms.NotifyAgentGroupMemberships(agent) if err != nil { diff --git a/fleet/agent_group_service_test.go b/fleet/agent_group_service_test.go index 06086e795..9196b8735 100644 --- a/fleet/agent_group_service_test.go +++ b/fleet/agent_group_service_test.go @@ -89,7 +89,8 @@ func newService(auth mainflux.AuthServiceClient, url string) fleet.Service { mfsdk := mfsdk.NewSDK(config) pktvisor.Register(auth, agentRepo) - return fleet.NewFleetService(logger, auth, agentRepo, agentGroupRepo, agentComms, mfsdk) + aDone := make(chan bool) + return fleet.NewFleetService(logger, auth, agentRepo, agentGroupRepo, agentComms, mfsdk, aDone) } func TestCreateAgentGroup(t *testing.T) { @@ -107,10 +108,10 @@ func TestCreateAgentGroup(t *testing.T) { require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) agent := fleet.Agent{ - Name: agNameID, - MFOwnerID: ownerID.String(), - MFChannelID: "", - AgentTags: map[string]string{ + Name: agNameID, + MFOwnerID: ownerID.String(), + MFChannelID: "", + AgentTags: map[string]string{ "region": "eu", "node_type": "dns", }, @@ -317,9 +318,9 @@ func TestUpdateAgentGroup(t *testing.T) { require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) agent := fleet.Agent{ - Name: agNameID, - MFOwnerID: ag.MFOwnerID, - MFChannelID: ag.MFChannelID, + Name: agNameID, + MFOwnerID: ag.MFOwnerID, + MFChannelID: ag.MFChannelID, } _, err = fleetService.CreateAgent(context.Background(), token, agent) diff --git a/fleet/agent_service.go b/fleet/agent_service.go index 5593df26b..84c9e9022 100644 --- a/fleet/agent_service.go +++ b/fleet/agent_service.go @@ -15,6 +15,7 @@ import ( "github.com/ns1labs/orb/fleet/backend" "github.com/ns1labs/orb/pkg/errors" "go.uber.org/zap" + "strings" ) var ( @@ -29,11 +30,11 @@ var ( ) func (svc fleetService) addAgentToAgentGroupChannels(token string, a Agent) error { - // first we get the agent group to connect the new agent to the correct group channel groupList, err := svc.agentGroupRepository.RetrieveAllByAgent(context.Background(), a) if err != nil { return err } + if len(groupList) == 0 { return nil } @@ -47,7 +48,11 @@ func (svc fleetService) addAgentToAgentGroupChannels(token string, a Agent) erro } err = svc.mfsdk.Connect(ids, token) if err != nil { - return err + if strings.Contains(err.Error(), "409") { + svc.logger.Warn("agent already connected, skipping...") + } else { + return err + } } } @@ -155,6 +160,7 @@ func (svc fleetService) CreateAgent(ctx context.Context, token string, a Agent) // TODO should we roll back? svc.logger.Error("failed to add agent to a existing group channel", zap.String("agent_id", a.MFThingID), zap.Error(err)) } + return a, nil } diff --git a/fleet/agent_state_check.go b/fleet/agent_state_check.go new file mode 100644 index 000000000..f3decc0a3 --- /dev/null +++ b/fleet/agent_state_check.go @@ -0,0 +1,39 @@ +package fleet + +import ( + "context" + "fmt" + "go.uber.org/zap" + "time" +) + +const ( + HeartbeatFreq = 60 * time.Second + DefaultTimeout = 300 * time.Second +) + +func (svc *fleetService) checkState(t time.Time) { + svc.logger.Info("checking for stale agents") + count, err := svc.agentRepo.SetStaleStatus(context.Background(), DefaultTimeout) + if err != nil { + svc.logger.Error("failed to change agents status to stale", zap.Error(err)) + } + if count > 0 { + svc.logger.Info(fmt.Sprintf("%d agents with more than %v without heartbeats had their state changed to stale", count, DefaultTimeout)) + } +} + + +func (svc *fleetService) checkAgents() { + svc.checkState(time.Now()) + for { + select { + case <-svc.aDone: + svc.logger.Info("stopping stale agent routine") + svc.aTicker.Stop() + return + case t := <-svc.aTicker.C: + svc.checkState(t) + } + } +} diff --git a/fleet/agents.go b/fleet/agents.go index 0af77bc74..c030bfa2d 100644 --- a/fleet/agents.go +++ b/fleet/agents.go @@ -122,6 +122,8 @@ type AgentRepository interface { RetrieveAgentMetadataByOwner(ctx context.Context, ownerID string) ([]types.Metadata, error) // RetrieveOwnerByChannelID retrieves a ownerID by a provided channelID RetrieveOwnerByChannelID(ctx context.Context, channelID string) (Agent, error) + // SetStaleStatus change status to stale according provided duration without heartbeats + SetStaleStatus(ctx context.Context, minutes time.Duration) (int64, error) } type AgentHeartbeatRepository interface { diff --git a/fleet/api/http/endpoint.go b/fleet/api/http/endpoint.go index bdd410265..644f57f65 100644 --- a/fleet/api/http/endpoint.go +++ b/fleet/api/http/endpoint.go @@ -176,6 +176,7 @@ func addAgentEndpoint(svc fleet.Service) endpoint.Endpoint { agent := fleet.Agent{ Name: nID, OrbTags: req.OrbTags, + AgentTags: req.AgentTags, } saved, err := svc.CreateAgent(c, req.token, agent) if err != nil { diff --git a/fleet/api/http/endpoint_test.go b/fleet/api/http/endpoint_test.go index 937785850..63168d392 100644 --- a/fleet/api/http/endpoint_test.go +++ b/fleet/api/http/endpoint_test.go @@ -138,7 +138,8 @@ func newService(auth mainflux.AuthServiceClient, url string) fleet.Service { mfsdk := mfsdk.NewSDK(config) pktvisor.Register(auth, agentRepo) - return fleet.NewFleetService(logger, auth, agentRepo, agentGroupRepo, agentComms, mfsdk) + aDone := make(chan bool) + return fleet.NewFleetService(logger, auth, agentRepo, agentGroupRepo, agentComms, mfsdk, aDone) } func newServer(svc fleet.Service) *httptest.Server { diff --git a/fleet/api/http/requests.go b/fleet/api/http/requests.go index bec0dab1b..8a162f647 100644 --- a/fleet/api/http/requests.go +++ b/fleet/api/http/requests.go @@ -79,9 +79,10 @@ func (req updateAgentGroupReq) validate() error { } type addAgentReq struct { - token string - Name string `json:"name,omitempty"` - OrbTags types.Tags `json:"orb_tags,omitempty"` + token string + Name string `json:"name,omitempty"` + OrbTags types.Tags `json:"orb_tags,omitempty"` + AgentTags types.Tags `json:"agent_tags,omitempty"` } func (req addAgentReq) validate() error { diff --git a/fleet/comms.go b/fleet/comms.go index 1996899de..642c4c549 100644 --- a/fleet/comms.go +++ b/fleet/comms.go @@ -90,10 +90,11 @@ func (svc fleetCommsService) NotifyGroupNewDataset(ctx context.Context, ag Agent DatasetID: datasetID, }} - data := RPC{ + data := AgentPolicyRPC{ SchemaVersion: CurrentRPCSchemaVersion, Func: AgentPolicyRPCFunc, Payload: payload, + FullList: false, } body, err := json.Marshal(data) @@ -157,11 +158,6 @@ func (svc fleetCommsService) NotifyAgentAllDatasets(a Agent) error { return err } - if len(groups) == 0 { - // no groups, nothing to do - return nil - } - groupIDs := make([]string, len(groups)) for i, group := range groups { groupIDs[i] = group.ID @@ -173,35 +169,44 @@ func (svc fleetCommsService) NotifyAgentAllDatasets(a Agent) error { return err } - p, err := svc.policyClient.RetrievePoliciesByGroups(ctx, &pb.PoliciesByGroupsReq{GroupIDs: groupIDs, OwnerID: a.MFOwnerID}) - if err != nil { - return err - } - - payload := make([]AgentPolicyRPCPayload, len(p.Policies)) - for i, policy := range p.Policies { - - var pdata interface{} - if err := json.Unmarshal(policy.Data, &pdata); err != nil { + var payload []AgentPolicyRPCPayload + if len(groups) > 0 { + p, err := svc.policyClient.RetrievePoliciesByGroups(ctx, &pb.PoliciesByGroupsReq{GroupIDs: groupIDs, OwnerID: a.MFOwnerID}) + if err != nil { return err } + payload = make([]AgentPolicyRPCPayload, len(p.Policies)) + for i, policy := range p.Policies { + + var pdata interface{} + if err := json.Unmarshal(policy.Data, &pdata); err != nil { + return err + } + + payload[i] = AgentPolicyRPCPayload{ + Action: "manage", + ID: policy.Id, + Name: policy.Name, + Backend: policy.Backend, + Version: policy.Version, + Data: pdata, + DatasetID: policy.DatasetId, + } - payload[i] = AgentPolicyRPCPayload{ - Action: "manage", - ID: policy.Id, - Name: policy.Name, - Backend: policy.Backend, - Version: policy.Version, - Data: pdata, - DatasetID: policy.DatasetId, } - + } else { + // Even with no policies, we should send the signal to agent for policy sanitization + payload = make([]AgentPolicyRPCPayload, 1) + payload[0] = AgentPolicyRPCPayload{ + Action: "sanitize", + } } - data := RPC{ + data := AgentPolicyRPC{ SchemaVersion: CurrentRPCSchemaVersion, Func: AgentPolicyRPCFunc, Payload: payload, + FullList: true, } body, err := json.Marshal(data) @@ -319,10 +324,11 @@ func (svc fleetCommsService) NotifyGroupPolicyUpdate(ctx context.Context, ag Age Data: pdata, }} - data := RPC{ + data := AgentPolicyRPC{ SchemaVersion: CurrentRPCSchemaVersion, Func: AgentPolicyRPCFunc, Payload: payload, + FullList: false, } body, err := json.Marshal(data) @@ -360,6 +366,7 @@ func (svc fleetCommsService) NotifyGroupPolicyRemoval(ag AgentGroup, policyID st SchemaVersion: CurrentRPCSchemaVersion, Func: AgentPolicyRPCFunc, Payload: payloads, + FullList: false, } body, err := json.Marshal(data) diff --git a/fleet/comms_rpc.go b/fleet/comms_rpc.go index 0525dbaa7..7b6f71f2f 100644 --- a/fleet/comms_rpc.go +++ b/fleet/comms_rpc.go @@ -39,6 +39,7 @@ type AgentPolicyRPC struct { SchemaVersion string `json:"schema_version"` Func string `json:"func"` Payload []AgentPolicyRPCPayload `json:"payload"` + FullList bool `json:"full_list"` } type AgentPolicyRPCPayload struct { diff --git a/fleet/comms_test.go b/fleet/comms_test.go index cf33882c8..85ca7819e 100644 --- a/fleet/comms_test.go +++ b/fleet/comms_test.go @@ -52,7 +52,8 @@ func newFleetService(auth mainflux.AuthServiceClient, url string, agentGroupRepo mfsdk := mfsdk.NewSDK(config) pktvisor.Register(auth, agentRepo) - return fleet.NewFleetService(logger, auth, agentRepo, agentGroupRepo, agentComms, mfsdk) + aDone := make(chan bool) + return fleet.NewFleetService(logger, auth, agentRepo, agentGroupRepo, agentComms, mfsdk, aDone) } func newPoliciesService(auth mainflux.AuthServiceClient) policies.Service { diff --git a/fleet/mocks/agent.go b/fleet/mocks/agent.go index b33d5992c..29339a0aa 100644 --- a/fleet/mocks/agent.go +++ b/fleet/mocks/agent.go @@ -5,6 +5,7 @@ import ( "github.com/ns1labs/orb/fleet" "github.com/ns1labs/orb/pkg/errors" "github.com/ns1labs/orb/pkg/types" + "time" ) var _ fleet.AgentRepository = (*agentRepositoryMock)(nil) @@ -14,6 +15,10 @@ type agentRepositoryMock struct { agentsMock map[string]fleet.Agent } +func (a agentRepositoryMock) SetStaleStatus(ctx context.Context, minutes time.Duration) (int64, error) { + return 0, nil +} + func (a agentRepositoryMock) RetrieveOwnerByChannelID(ctx context.Context, channelID string) (fleet.Agent, error) { for _, ag := range a.agentsMock{ if ag.MFChannelID == channelID{ diff --git a/fleet/postgres/agents.go b/fleet/postgres/agents.go index 5cba705c2..d502cff4b 100644 --- a/fleet/postgres/agents.go +++ b/fleet/postgres/agents.go @@ -481,6 +481,36 @@ func (r agentRepository) RetrieveOwnerByChannelID(ctx context.Context, channelID return toAgent(ownerScan) } +func (r agentRepository) SetStaleStatus(ctx context.Context, duration time.Duration) (int64, error) { + + q := `UPDATE agents SET state = :state WHERE state <> 'stale' AND ts_last_hb <= now() - :duration * interval '1 seconds';` + + params := map[string]interface{}{ + "duration": duration.Seconds(), + "state": fleet.Stale, + } + res, err := r.db.NamedExecContext(ctx, q, params) + if err != nil { + pqErr, ok := err.(*pq.Error) + if ok { + switch pqErr.Code.Name() { + case db.ErrInvalid, db.ErrTruncation: + return 0, errors.Wrap(errors.ErrMalformedEntity, err) + case db.ErrDuplicate: + return 0, errors.Wrap(errors.ErrConflict, err) + } + } + return 0, errors.Wrap(db.ErrUpdateDB, err) + } + + cnt, errdb := res.RowsAffected() + if errdb != nil { + return 0, errors.Wrap(errors.ErrUpdateEntity, errdb) + } + + return cnt, nil +} + type dbAgent struct { Name types.Identifier `db:"name"` MFOwnerID string `db:"mf_owner_id"` diff --git a/fleet/postgres/agents_test.go b/fleet/postgres/agents_test.go index 5946ef8a8..c5e64a23e 100644 --- a/fleet/postgres/agents_test.go +++ b/fleet/postgres/agents_test.go @@ -23,6 +23,7 @@ import ( "reflect" "strings" "testing" + "time" ) const maxNameSize = 1024 @@ -88,39 +89,39 @@ func TestAgentSave(t *testing.T) { }, "create new agent with empty OwnerID": { agent: fleet.Agent{ - Name: nameID, - MFOwnerID: "", - MFThingID: thID.String(), - MFChannelID: chID.String(), + Name: nameID, + MFOwnerID: "", + MFThingID: thID.String(), + MFChannelID: chID.String(), }, - err: errors.ErrMalformedEntity, + err: errors.ErrMalformedEntity, }, "create new agent with empty ThingID": { agent: fleet.Agent{ - Name: nameID, - MFOwnerID: oID.String(), - MFThingID: "", - MFChannelID: chID.String(), + Name: nameID, + MFOwnerID: oID.String(), + MFThingID: "", + MFChannelID: chID.String(), }, - err: errors.ErrMalformedEntity, + err: errors.ErrMalformedEntity, }, "create new agent with empty channelID": { agent: fleet.Agent{ - Name: nameID, - MFOwnerID: oID.String(), - MFThingID: thID.String(), - MFChannelID: "", + Name: nameID, + MFOwnerID: oID.String(), + MFThingID: thID.String(), + MFChannelID: "", }, - err: errors.ErrMalformedEntity, + err: errors.ErrMalformedEntity, }, "create new agent with empty invalid OwnerID": { agent: fleet.Agent{ - Name: nameID, - MFOwnerID: "123", - MFThingID: thID.String(), - MFChannelID: chID.String(), + Name: nameID, + MFOwnerID: "123", + MFThingID: thID.String(), + MFChannelID: chID.String(), }, - err: errors.ErrMalformedEntity, + err: errors.ErrMalformedEntity, }, } @@ -810,7 +811,6 @@ func TestMultiAgentRetrievalByAgentGroup(t *testing.T) { subTags := types.Tags{} json.Unmarshal([]byte(subTagsStr), &subTags) - group := fleet.AgentGroup{ Name: groupNameID, MFOwnerID: oID.String(), @@ -1052,7 +1052,6 @@ func TestMatchingAgentRetrieval(t *testing.T) { agentTagsStr := `{"region": "EU"}` mixTagsStr := `{"node_type": "dns", "region": "EU"}` - orbTags := types.Tags{} json.Unmarshal([]byte(orbTagsStr), &orbTags) @@ -1091,25 +1090,25 @@ func TestMatchingAgentRetrieval(t *testing.T) { }{ "retrieve matching agents with mix tags": { owner: oID.String(), - tag: mixTags, + tag: mixTags, matchingAgents: types.Metadata{ - "total": float64(n), + "total": float64(n), "online": float64(0), }, }, "retrieve matching agents with orb tags": { owner: oID.String(), - tag: orbTags, + tag: orbTags, matchingAgents: types.Metadata{ - "total": float64(n), + "total": float64(n), "online": float64(0), }, }, "retrieve matching agents with agent tags": { owner: oID.String(), - tag: agentTags, + tag: agentTags, matchingAgents: types.Metadata{ - "total": float64(n), + "total": float64(n), "online": float64(0), }, }, @@ -1119,7 +1118,7 @@ func TestMatchingAgentRetrieval(t *testing.T) { "wrong": "tag", }, matchingAgents: types.Metadata{ - "total": nil, + "total": nil, "online": nil, }, }, @@ -1127,11 +1126,11 @@ func TestMatchingAgentRetrieval(t *testing.T) { owner: oID.String(), tag: types.Tags{ "node_type": "dns", - "region": "EU", - "wrong": "tag", + "region": "EU", + "wrong": "tag", }, matchingAgents: types.Metadata{ - "total": nil, + "total": nil, "online": nil, }, }, @@ -1145,3 +1144,72 @@ func TestMatchingAgentRetrieval(t *testing.T) { }) } } + +func TestSetAgentStale(t *testing.T) { + dbMiddleware := postgres.NewDatabase(db) + agentRepo := postgres.NewAgentRepository(dbMiddleware, logger) + + thID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + chID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + oID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + nameID, err := types.NewIdentifier("myagent") + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + agent := fleet.Agent{ + Name: nameID, + MFThingID: thID.String(), + MFOwnerID: oID.String(), + MFChannelID: chID.String(), + OrbTags: types.Tags{"testkey": "testvalue"}, + AgentTags: types.Tags{"testkey": "testvalue"}, + AgentMetadata: types.Metadata{"testkey": "testvalue"}, + LastHB: time.Now().Add(fleet.DefaultTimeout), + } + + err = agentRepo.Save(context.Background(), agent) + require.Nil(t, err, fmt.Sprintf("unexpected error: %s\n", err)) + + cases := map[string]struct { + agent fleet.Agent + duration time.Duration + owner string + state fleet.State + }{ + "set agent state to stale when stops do send heartbeats": { + agent: agent, + duration: 1 * time.Second, + owner: oID.String(), + state: fleet.Stale, + }, + "keep agent state online when agent it's sending heartbeats": { + agent: agent, + duration: 3 * time.Second, + owner: oID.String(), + state: fleet.Online, + }, + } + + for desc, tc := range cases { + t.Run(desc, func(t *testing.T) { + + // simulating a heartbeat from agent + tc.agent.State = fleet.Online + err = agentRepo.UpdateHeartbeatByIDWithChannel(context.Background(), tc.agent) + require.Nil(t, err, fmt.Sprintf("unexpected error: %s\n", err)) + time.Sleep(2 * time.Second) + + _, err := agentRepo.SetStaleStatus(context.Background(), tc.duration) + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + agent, err := agentRepo.RetrieveByID(context.Background(), tc.owner, tc.agent.MFThingID) + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + require.Equal(t, tc.state, agent.State, fmt.Sprintf("%s: expected %s got %s", desc, tc.state.String(), agent.State.String())) + }) + } + +} diff --git a/fleet/postgres/init.go b/fleet/postgres/init.go index 71b6fab94..9707dd8be 100644 --- a/fleet/postgres/init.go +++ b/fleet/postgres/init.go @@ -100,6 +100,22 @@ func migrateDB(db *sqlx.DB) error { "DROP TABLE agent_groups", "DROP VIEW agent_group_membership", }, + }, { + Id: "fleet_2", + Up: []string{ + `CREATE or REPLACE VIEW agent_group_membership(agent_groups_id, agent_groups_name, agent_mf_thing_id, agent_mf_channel_id, group_mf_channel_id, mf_owner_id, agent_state) as + SELECT agent_groups.id, + agent_groups.name, + agents.mf_thing_id, + agents.mf_channel_id, + agent_groups.mf_channel_id, + agent_groups.mf_owner_id, + agents.state + FROM agents, + agent_groups + WHERE agent_groups.mf_owner_id = agents.mf_owner_id + AND (agent_groups.tags <@ coalesce(agents.agent_tags || agents.orb_tags, agents.agent_tags, agents.orb_tags))`, + }, }, }, } diff --git a/fleet/service.go b/fleet/service.go index bc353e839..0cf7b2be5 100644 --- a/fleet/service.go +++ b/fleet/service.go @@ -48,6 +48,9 @@ type fleetService struct { agentGroupRepository AgentGroupRepository // Agent Comms agentComms AgentCommsService + + aTicker *time.Ticker + aDone chan bool } func (svc fleetService) identify(token string) (string, error) { @@ -92,15 +95,21 @@ func (svc fleetService) thing(token, id string, name string, md map[string]inter return thing, nil } -func NewFleetService(logger *zap.Logger, auth mainflux.AuthServiceClient, agentRepo AgentRepository, agentGroupRepository AgentGroupRepository, agentComms AgentCommsService, mfsdk mfsdk.SDK) Service { +func NewFleetService(logger *zap.Logger, auth mainflux.AuthServiceClient, agentRepo AgentRepository, agentGroupRepository AgentGroupRepository, agentComms AgentCommsService, mfsdk mfsdk.SDK, aDone chan bool) Service { + + aTicker := time.NewTicker(HeartbeatFreq) - return &fleetService{ + service := fleetService{ logger: logger, auth: auth, agentRepo: agentRepo, agentGroupRepository: agentGroupRepository, agentComms: agentComms, mfsdk: mfsdk, + aTicker: aTicker, + aDone: aDone, } - + + go service.checkAgents() + return service } diff --git a/policies/api/http/logging.go b/policies/api/http/logging.go index 8bab6583c..ab29eb1ef 100644 --- a/policies/api/http/logging.go +++ b/policies/api/http/logging.go @@ -18,6 +18,20 @@ type loggingMiddleware struct { svc policies.Service } +func (l loggingMiddleware) InactivateDatasetByIDInternal(ctx context.Context, ownerID string, datasetID string) (err error) { + defer func(begin time.Time) { + if err != nil { + l.logger.Warn("method call: inactivate_dataset_by_id_internal", + zap.Error(err), + zap.Duration("duration", time.Since(begin))) + } else { + l.logger.Info("method call: inactivate_dataset_by_id_internal", + zap.Duration("duration", time.Since(begin))) + } + }(time.Now()) + return l.svc.InactivateDatasetByIDInternal(ctx, ownerID, datasetID) +} + func (l loggingMiddleware) ViewDatasetByIDInternal(ctx context.Context, ownerID string, datasetID string) (_ policies.Dataset, err error) { defer func(begin time.Time) { if err != nil { @@ -256,6 +270,20 @@ func (l loggingMiddleware) ListDatasets(ctx context.Context, token string, pm po return l.svc.ListDatasets(ctx, token, pm) } +func (l loggingMiddleware) DeleteSinkFromAllDatasetsInternal(ctx context.Context, sinkID string, ownerID string) (ds []policies.Dataset, err error) { + defer func(begin time.Time) { + if err != nil { + l.logger.Warn("method call: delete_sink_from_all_datasets", + zap.Error(err), + zap.Duration("duration", time.Since(begin))) + } else { + l.logger.Info("method call: delete_sink_from_all_datasets", + zap.Duration("duration", time.Since(begin))) + } + }(time.Now()) + return l.svc.DeleteSinkFromAllDatasetsInternal(ctx, sinkID, ownerID) +} + func NewLoggingMiddleware(svc policies.Service, logger *zap.Logger) policies.Service { return &loggingMiddleware{logger, svc} } diff --git a/policies/api/http/metrics.go b/policies/api/http/metrics.go index cb53192ca..9cfe42506 100644 --- a/policies/api/http/metrics.go +++ b/policies/api/http/metrics.go @@ -22,6 +22,23 @@ type metricsMiddleware struct { svc policies.Service } +func (m metricsMiddleware) InactivateDatasetByIDInternal(ctx context.Context, ownerID string, datasetID string) error { + defer func(begin time.Time) { + labels := []string{ + "method", "inactivateDatasetByIDInternal", + "owner_id", ownerID, + "policy_id", "", + "dataset_id", datasetID, + } + + m.counter.With(labels...).Add(1) + m.latency.With(labels...).Observe(float64(time.Since(begin).Microseconds())) + + }(time.Now()) + + return m.svc.InactivateDatasetByIDInternal(ctx, ownerID, datasetID) +} + func (m metricsMiddleware) ViewDatasetByIDInternal(ctx context.Context, ownerID string, datasetID string) (policies.Dataset, error) { defer func(begin time.Time) { labels := []string{ @@ -376,6 +393,23 @@ func (m metricsMiddleware) ListDatasets(ctx context.Context, token string, pm po return m.svc.ListDatasets(ctx, token, pm) } +func (m metricsMiddleware) DeleteSinkFromAllDatasetsInternal(ctx context.Context, sinkID string, ownerID string) ([]policies.Dataset, error) { + defer func(begin time.Time) { + labels := []string{ + "method", "deleteSinkFromAllDatasetsInternal", + "owner_id", ownerID, + "policy_id", "", + "dataset_id", "", + } + + m.counter.With(labels...).Add(1) + m.latency.With(labels...).Observe(float64(time.Since(begin).Microseconds())) + + }(time.Now()) + + return m.svc.DeleteSinkFromAllDatasetsInternal(ctx, sinkID, ownerID) +} + func (m metricsMiddleware) identify(token string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() diff --git a/policies/api/http/responses.go b/policies/api/http/responses.go index 22bd8108a..5687a0c0c 100644 --- a/policies/api/http/responses.go +++ b/policies/api/http/responses.go @@ -20,7 +20,7 @@ type policyRes struct { Policy types.Metadata `json:"policy,omitempty"` Format string `json:"format,omitempty"` PolicyData string `json:"policy_data,omitempty"` - Version int32 `json:"version,omitempty"` + Version int32 `json:"version"` LastModified time.Time `json:"ts_last_modified"` created bool } diff --git a/policies/mocks/policies.go b/policies/mocks/policies.go index 0896ae687..f561d26ea 100644 --- a/policies/mocks/policies.go +++ b/policies/mocks/policies.go @@ -21,6 +21,51 @@ type mockPoliciesRepository struct { gdb map[string][]policies.PolicyInDataset } +func (m *mockPoliciesRepository) RetrieveAllDatasetsInternal(ctx context.Context, owner string) ([]policies.Dataset, error) { + var datasetList []policies.Dataset + id := uint64(0) + for _, d := range m.ddb { + if d.MFOwnerID == owner { + datasetList = append(datasetList, d) + } + id++ + } + + return datasetList, nil +} + +func (m *mockPoliciesRepository) InactivateDatasetByID(ctx context.Context, sinkID string, ownerID string) error { + for _, ds := range m.ddb{ + if ds.MFOwnerID == ownerID{ + for _, sID := range ds.SinkIDs { + if sID == sinkID{ + ds.Valid = false + } + } + } + } + return nil +} + +func (m *mockPoliciesRepository) DeleteSinkFromAllDatasets(ctx context.Context, sinkID string, ownerID string) ([]policies.Dataset, error) { + var datasets []policies.Dataset + + for _, ds := range m.ddb{ + if ds.MFOwnerID == ownerID{ + for i, sID := range ds.SinkIDs { + if sID == sinkID{ + ds.SinkIDs[i] = ds.SinkIDs[len(ds.SinkIDs)-1] + ds.SinkIDs[len(ds.SinkIDs)-1] = "" + ds.SinkIDs = ds.SinkIDs[:len(ds.SinkIDs)-1] + + datasets = append(datasets, ds) + } + } + } + } + return datasets, nil +} + func (m *mockPoliciesRepository) DeleteDataset(ctx context.Context, ownerID string, dsID string) error { if _, ok := m.ddb[dsID]; ok { if m.ddb[dsID].MFOwnerID != ownerID { diff --git a/policies/policies.go b/policies/policies.go index a27abe826..cb6fd6025 100644 --- a/policies/policies.go +++ b/policies/policies.go @@ -102,6 +102,12 @@ type Service interface { // ListDatasets retrieve a list of Dataset by owner ListDatasets(ctx context.Context, token string, pm PageMetadata) (PageDataset, error) + + // InactivateDatasetByIDInternal inactivate a dataset + InactivateDatasetByIDInternal(ctx context.Context, ownerID string, datasetID string) error + + // DeleteSinkFromAllDatasetsInternal removes a sink from a dataset + DeleteSinkFromAllDatasetsInternal(ctx context.Context, sinkID string, ownerID string) ([]Dataset, error) } type Repository interface { @@ -148,4 +154,10 @@ type Repository interface { // RetrieveAllDatasetsByOwner retrieves the subset of Datasets owned by the specified user RetrieveAllDatasetsByOwner(ctx context.Context, ownerID string, pm PageMetadata) (PageDataset, error) + + // InactivateDatasetByID inactivate a dataset + InactivateDatasetByID(ctx context.Context, sinkID string, ownerID string) error + + // DeleteSinkFromAllDatasets removes a sink from a dataset + DeleteSinkFromAllDatasets(ctx context.Context, sinkID string, ownerID string) ([]Dataset, error) } diff --git a/policies/policy_service.go b/policies/policy_service.go index 08bb1cd7d..09cc9257c 100644 --- a/policies/policy_service.go +++ b/policies/policy_service.go @@ -328,3 +328,29 @@ func (s policiesService) ListDatasets(ctx context.Context, token string, pm Page } return s.repo.RetrieveAllDatasetsByOwner(ctx, ownerID, pm) } + +func (s policiesService) DeleteSinkFromAllDatasetsInternal(ctx context.Context, sinkID string, ownerID string) ([]Dataset, error) { + if sinkID == "" || ownerID == ""{ + return []Dataset{}, ErrMalformedEntity + } + + datasets, err := s.repo.DeleteSinkFromAllDatasets(ctx, sinkID, ownerID) + if err != nil { + return []Dataset{}, err + } + + return datasets, nil +} + +func (s policiesService) InactivateDatasetByIDInternal(ctx context.Context, ownerID string, datasetID string) error { + if datasetID == "" || ownerID == ""{ + return ErrMalformedEntity + } + + err := s.repo.InactivateDatasetByID(ctx, datasetID, ownerID) + if err != nil { + return errors.Wrap(ErrInactivateDataset, err) + } + + return nil +} \ No newline at end of file diff --git a/policies/policy_service_test.go b/policies/policy_service_test.go index 0c86126de..8ed8c4107 100644 --- a/policies/policy_service_test.go +++ b/policies/policy_service_test.go @@ -785,11 +785,11 @@ func TestListPoliciesByGroupIDInternal(t *testing.T) { err error }{ "retrieve a list of policies by groupID": { - ownerID: oID, - groupID: []string{agentGroupID.String()}, + ownerID: oID, + groupID: []string{agentGroupID.String()}, policies: listPlTest, - size: uint64(total), - err: nil, + size: uint64(total), + err: nil, }, "retrieve a list of policies by non-existent groupID": { ownerID: oID, @@ -841,9 +841,9 @@ func TestRetrievePolicyByIDInternal(t *testing.T) { err: nil, }, "view policy with empty ownerID": { - policyID: policy.ID, - ownerID: "", - err: policies.ErrMalformedEntity, + policyID: policy.ID, + ownerID: "", + err: policies.ErrMalformedEntity, }, "view non-existing policy": { policyID: wrongPlID.String(), @@ -949,18 +949,18 @@ func TestRetrieveDatasetByIDInternal(t *testing.T) { cases := map[string]struct { datasetID string - ownerID string - err error + ownerID string + err error }{ "view a existing dataset": { datasetID: dataset.ID, - ownerID: oID, - err: nil, + ownerID: oID, + err: nil, }, "view non-existing policy": { datasetID: wrongPlID.String(), - ownerID: oID, - err: policies.ErrNotFound, + ownerID: oID, + err: policies.ErrNotFound, }, } for desc, tc := range cases { @@ -1103,6 +1103,157 @@ func createDataset(t *testing.T, svc policies.Service, name string) policies.Dat return res } +func TestDeleteSinkFromDataset(t *testing.T) { + users := flmocks.NewAuthService(map[string]string{token: email}) + svc := newService(users) + + agentGroupID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + policy := createPolicy(t, svc, "policy") + + var total = 10 + + datasetsTest := make([]policies.Dataset, total) + + sinkIDs := make([]string, 0) + for i := 0; i < total; i++ { + ID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + for i := 0; i < 2; i++ { + sinkID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + sinkIDs = append(sinkIDs, sinkID.String()) + } + + validName, err := types.NewIdentifier(fmt.Sprintf("dataset-%d", i)) + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + dataset := policies.Dataset{ + ID: ID.String(), + Name: validName, + PolicyID: policy.ID, + AgentGroupID: agentGroupID.String(), + SinkIDs: sinkIDs, + } + + ds, err := svc.AddDataset(context.Background(), token, dataset) + if err != nil { + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + } + + datasetsTest[i] = ds + } + + cases := map[string]struct { + ownerID string + sinkID string + err error + }{ + "delete sinkID of a set of datasets": { + sinkID: sinkIDs[0], + ownerID: datasetsTest[0].MFOwnerID, + err: nil, + }, + "delete sinkID of a set of datasets with empty sinkID": { + sinkID: "", + ownerID: datasetsTest[0].MFOwnerID, + err: policies.ErrMalformedEntity, + }, + "delete sinkID of a set of datasets with empty owner": { + sinkID: sinkIDs[0], + ownerID: "", + err: policies.ErrMalformedEntity, + }, + } + + for desc, tc := range cases { + t.Run(desc, func(t *testing.T) { + _, err := svc.DeleteSinkFromAllDatasetsInternal(context.Background(), tc.sinkID, tc.ownerID) + assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s", desc, tc.err, err)) + }) + } +} + +func TestInactivateDatasetByID(t *testing.T) { + users := flmocks.NewAuthService(map[string]string{token: email}) + svc := newService(users) + + agentGroupID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + policy := createPolicy(t, svc, "policy") + + var total = 10 + + datasetsTest := make([]policies.Dataset, total) + datasetsID := make([]string, 0, total) + + sinkIDs := make([]string, 0) + for i := 0; i < total; i++ { + ID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + datasetsID = append(datasetsID, ID.String()) + + for i := 0; i < 2; i++ { + sinkID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + sinkIDs = append(sinkIDs, sinkID.String()) + } + + validName, err := types.NewIdentifier(fmt.Sprintf("dataset-%d", i)) + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + dataset := policies.Dataset{ + ID: ID.String(), + Name: validName, + PolicyID: policy.ID, + AgentGroupID: agentGroupID.String(), + SinkIDs: sinkIDs, + } + + ds, err := svc.AddDataset(context.Background(), token, dataset) + if err != nil { + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + } + + datasetsTest[i] = ds + } + + cases := map[string]struct { + ownerID string + datasetIDs []string + err error + }{ + "inactivate a set of datasets by ID": { + datasetIDs: datasetsID, + ownerID: datasetsTest[0].MFOwnerID, + err: nil, + }, + "inactivate datasets with empty ownerID": { + datasetIDs: datasetsID, + ownerID: "", + err: policies.ErrMalformedEntity, + }, + "inactivate datasets with empty ID": { + datasetIDs: []string{""}, + ownerID: datasetsTest[0].MFOwnerID, + err: policies.ErrMalformedEntity, + }, + } + + for desc, tc := range cases { + t.Run(desc, func(t *testing.T) { + for _, id := range tc.datasetIDs { + err := svc.InactivateDatasetByIDInternal(context.Background(), tc.ownerID, id) + assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %s got %s", desc, tc.err, err)) + } + }) + } +} + func testSortPolicies(t *testing.T, pm policies.PageMetadata, ags []policies.Policy) { t.Helper() switch pm.Order { @@ -1151,4 +1302,4 @@ func identify(token string, auth mainflux.AuthServiceClient) (string, error) { } return res.GetId(), nil -} \ No newline at end of file +} diff --git a/policies/postgres/datasets_test.go b/policies/postgres/datasets_test.go index 3e2497317..68c9ca266 100644 --- a/policies/postgres/datasets_test.go +++ b/policies/postgres/datasets_test.go @@ -87,7 +87,7 @@ func TestDatasetSave(t *testing.T) { PolicyID: policyID.String(), SinkIDs: sinkIDs, }, - err: errors.ErrMalformedEntity, + err: errors.ErrMalformedEntity, }, } @@ -161,7 +161,7 @@ func TestDatasetUpdate(t *testing.T) { Created: time.Time{}, ID: wrongID.String(), }, - err: policies.ErrNotFound, + err: policies.ErrNotFound, }, } @@ -535,9 +535,9 @@ func TestInactivateDatasetByPolicyID(t *testing.T) { dataset.ID = dsID cases := map[string]struct { - ownerID string + ownerID string policyID string - err error + err error }{ "inactivate a existing dataset by policy ID": { ownerID: dataset.MFOwnerID, @@ -624,6 +624,198 @@ func TestMultiDatasetRetrievalPolicyID(t *testing.T) { } } +func TestInactivateDatasetByID(t *testing.T) { + dbMiddleware := postgres.NewDatabase(db) + repo := postgres.NewPoliciesRepository(dbMiddleware, logger) + + oID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + oID2, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + groupID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + sinkIDs := make([]string, 2) + for i := 0; i < 2; i++ { + sinkID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + sinkIDs[i] = sinkID.String() + } + + nameID, err := types.NewIdentifier("mydataset") + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + nameID2, err := types.NewIdentifier("mydataset-2") + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + dataset := policies.Dataset{ + Name: nameID, + MFOwnerID: oID.String(), + Valid: true, + AgentGroupID: groupID.String(), + PolicyID: policyID.String(), + SinkIDs: sinkIDs, + Metadata: types.Metadata{"testkey": "testvalue"}, + Created: time.Time{}, + } + + dataset2 := dataset + dataset2.Name = nameID2 + dataset2.MFOwnerID = oID2.String() + + dsID, err := repo.SaveDataset(context.Background(), dataset) + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + dataset.ID = dsID + + dsID2, err := repo.SaveDataset(context.Background(), dataset2) + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + dataset2.ID = dsID2 + + cases := map[string]struct { + ownerID string + id string + dataset policies.Dataset + valid bool + err error + }{ + "inactivate a existing dataset by ID": { + ownerID: dataset.MFOwnerID, + id: dataset.ID, + dataset: dataset, + valid: false, + err: nil, + }, + "inactivate dataset with an invalid ownerID": { + id: dataset.ID, + ownerID: "", + dataset: dataset2, + valid: true, + err: policies.ErrMalformedEntity, + }, + } + + for desc, tc := range cases { + t.Run(desc, func(t *testing.T) { + err := repo.InactivateDatasetByID(context.Background(), tc.id, tc.ownerID) + assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected '%s' got '%s'", desc, tc.err, err)) + + check, _ := repo.RetrieveDatasetByID(context.Background(), tc.dataset.ID, tc.dataset.MFOwnerID) + assert.Equal(t, tc.valid, check.Valid, fmt.Sprintf("%s: expected '%t' got '%t'", desc, tc.valid, check.Valid)) + }) + } +} + +func TestDeleteSinkFromDataset(t *testing.T) { + dbMiddleware := postgres.NewDatabase(db) + repo := postgres.NewPoliciesRepository(dbMiddleware, logger) + + oID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + oID2, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + wrongSinkID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + groupID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + sinkIDs := make([]string, 2) + for i := 0; i < 2; i++ { + sinkID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + sinkIDs[i] = sinkID.String() + } + + nameID, err := types.NewIdentifier("mydataset") + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + nameID2, err := types.NewIdentifier("mydataset") + require.Nil(t, err, fmt.Sprintf("got unexpected error: %s", err)) + + dataset := policies.Dataset{ + Name: nameID, + MFOwnerID: oID.String(), + Valid: true, + AgentGroupID: groupID.String(), + PolicyID: policyID.String(), + SinkIDs: sinkIDs, + Metadata: types.Metadata{"testkey": "testvalue"}, + Created: time.Time{}, + } + + dataset2 := dataset + dataset2.Name = nameID2 + dataset2.MFOwnerID = oID2.String() + + dsID, err := repo.SaveDataset(context.Background(), dataset) + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + dataset.ID = dsID + + dsID2, err := repo.SaveDataset(context.Background(), dataset2) + require.Nil(t, err, fmt.Sprintf("Unexpected error: %s", err)) + + dataset2.ID = dsID2 + + cases := map[string]struct { + owner string + sinkID string + contains bool + dataset policies.Dataset + err error + }{ + "delete a sink from existing dataset": { + owner: dataset.MFOwnerID, + sinkID: dataset.SinkIDs[0], + contains: false, + dataset: dataset, + err: nil, + }, + "delete a non-existing sink from a dataset": { + owner: dataset.MFOwnerID, + sinkID: wrongSinkID.String(), + contains: false, + dataset: dataset, + err: nil, + }, + "delete a sink from a dataset with an invalid ownerID": { + sinkID: dataset2.SinkIDs[0], + owner: "", + contains: true, + dataset: dataset2, + err: errors.ErrMalformedEntity, + }, + } + + for desc, tc := range cases { + t.Run(desc, func(t *testing.T) { + dataset, err := repo.DeleteSinkFromAllDatasets(context.Background(), tc.sinkID, tc.owner) + assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected '%s' got '%s'", desc, tc.err, err)) + + for _, d := range dataset { + switch tc.contains { + case false: + assert.NotContains(t, d.SinkIDs, tc.sinkID, fmt.Sprintf("%s: expected '%s' to not contains '%s'", desc, d.SinkIDs, tc.sinkID)) + case true: + assert.Contains(t, d.SinkIDs, tc.sinkID, fmt.Sprintf("%s: expected '%s' to contains '%s'", desc, d.SinkIDs, tc.sinkID)) + } + } + }) + } +} + func testSortDataset(t *testing.T, pm policies.PageMetadata, ags []policies.Dataset) { t.Helper() switch pm.Order { diff --git a/policies/postgres/policies.go b/policies/postgres/policies.go index 73301f45e..ca25a6efb 100644 --- a/policies/postgres/policies.go +++ b/policies/postgres/policies.go @@ -504,6 +504,68 @@ func (r policiesRepository) RetrieveAllDatasetsByOwner(ctx context.Context, owne return pageDataset, nil } +func (r policiesRepository) InactivateDatasetByID(ctx context.Context, id string, ownerID string) error { + q := `UPDATE datasets SET valid = false WHERE mf_owner_id = :mf_owner_id AND :id = id` + + params := map[string]interface{}{ + "mf_owner_id": ownerID, + "id": id, + } + + _, err := r.db.NamedExecContext(ctx, q, params) + if err != nil { + pqErr, ok := err.(*pq.Error) + if ok { + switch pqErr.Code.Name() { + case db.ErrInvalid, db.ErrTruncation: + return errors.Wrap(policies.ErrMalformedEntity, err) + } + } + return errors.Wrap(policies.ErrUpdateEntity, err) + } + + return nil +} + +func (r policiesRepository) DeleteSinkFromAllDatasets(ctx context.Context, sinkID string, ownerID string) ([]policies.Dataset, error) { + q := `UPDATE datasets SET sink_ids = array_remove(sink_ids, :sink_ids) WHERE mf_owner_id = :mf_owner_id RETURNING *` + + if ownerID == "" { + return []policies.Dataset{}, errors.ErrMalformedEntity + } + + params := map[string]interface{}{ + "mf_owner_id": ownerID, + "sink_ids": sinkID, + } + + res, err := r.db.NamedQueryContext(ctx, q, params) + if err != nil { + pqErr, ok := err.(*pq.Error) + if ok { + switch pqErr.Code.Name() { + case db.ErrInvalid, db.ErrTruncation: + return []policies.Dataset{}, errors.Wrap(policies.ErrMalformedEntity, err) + } + } + return []policies.Dataset{}, errors.Wrap(errors.ErrSelectEntity, err) + } + + defer res.Close() + + var datasets []policies.Dataset + for res.Next() { + dbDataset := dbDataset{MFOwnerID: ownerID} + if err := res.StructScan(&dbDataset); err != nil { + return []policies.Dataset{}, errors.Wrap(errors.ErrSelectEntity, err) + } + dataset := toDataset(dbDataset) + datasets = append(datasets, dataset) + } + + return datasets, nil +} + type dbPolicy struct { ID string `db:"id"` Name types.Identifier `db:"name"` diff --git a/policies/redis/consumer/events.go b/policies/redis/consumer/events.go index 837ebaafc..0d327e8ab 100644 --- a/policies/redis/consumer/events.go +++ b/policies/redis/consumer/events.go @@ -15,3 +15,9 @@ type removeAgentGroupEvent struct { token string timestamp time.Time } + +type removeSinkEvent struct { + sinkID string + ownerID string + timestamp time.Time +} diff --git a/policies/redis/consumer/streams.go b/policies/redis/consumer/streams.go index 99af129bc..118ce3bc1 100644 --- a/policies/redis/consumer/streams.go +++ b/policies/redis/consumer/streams.go @@ -16,17 +16,21 @@ import ( ) const ( - stream = "orb.fleet" - group = "orb.policies" + stream = "orb.fleet" + streamSink = "orb.sinks" + group = "orb.policies" agentGroupPrefix = "agent_group." agentGroupRemove = agentGroupPrefix + "remove" + sinkPrefix = "sinks." + sinkRemove = sinkPrefix + "remove" exists = "BUSYGROUP Consumer Group name already exists" ) type Subscriber interface { - Subscribe(context context.Context) error + SubscribeToFleet(context context.Context) error + SubscribeToSink(context context.Context) error } type eventStore struct { @@ -46,7 +50,7 @@ func NewEventStore(policiesService policies.Service, client *redis.Client, escon } } -func (es eventStore) Subscribe(context context.Context) error { +func (es eventStore) SubscribeToFleet(context context.Context) error { err := es.client.XGroupCreateMkStream(context, stream, group, "$").Err() if err != nil && err.Error() != exists { return err @@ -81,6 +85,42 @@ func (es eventStore) Subscribe(context context.Context) error { } } +func (es eventStore) SubscribeToSink(context context.Context) error { + err := es.client.XGroupCreateMkStream(context, streamSink, group, "$").Err() + if err != nil && err.Error() != exists { + return err + } + + for { + streams, err := es.client.XReadGroup(context, &redis.XReadGroupArgs{ + Group: group, + Consumer: es.esconsumer, + Streams: []string{streamSink, ">"}, + Count: 100, + }).Result() + if err != nil || len(streams) == 0 { + continue + } + + for _, msg := range streams[0].Messages { + event := msg.Values + + var err error + switch event["operation"] { + case sinkRemove: + rte := decodeSinkRemove(event) + err = es.handleSinkRemove(context, rte.sinkID, rte.ownerID) + } + + if err != nil { + es.logger.Error("Failed to handle event", zap.String("operation", event["operation"].(string)), zap.Error(err)) + break + } + es.client.XAck(context, streamSink, group, msg.ID) + } + } +} + func decodeAgentGroupRemove(event map[string]interface{}) removeAgentGroupEvent { return removeAgentGroupEvent{ groupID: read(event, "group_id", ""), @@ -88,6 +128,13 @@ func decodeAgentGroupRemove(event map[string]interface{}) removeAgentGroupEvent } } +func decodeSinkRemove(event map[string]interface{}) removeSinkEvent { + return removeSinkEvent{ + sinkID: read(event, "sink_id", ""), + ownerID: read(event, "owner_id", ""), + } +} + // Inactivate a Dataset after AgentGroup deletion func (es eventStore) handleAgentGroupRemove(ctx context.Context, groupID string, token string) error { @@ -98,6 +145,25 @@ func (es eventStore) handleAgentGroupRemove(ctx context.Context, groupID string, return nil } +func (es eventStore) handleSinkRemove(ctx context.Context, sinkID string, ownerID string) error { + + datasets, err := es.policiesService.DeleteSinkFromAllDatasetsInternal(ctx, sinkID, ownerID) + if err != nil { + return err + } + + for _, ds := range datasets { + if len(ds.SinkIDs) == 0 { + err = es.policiesService.InactivateDatasetByIDInternal(ctx, ownerID, ds.ID) + if err != nil { + return err + } + } + } + + return nil +} + func read(event map[string]interface{}, key, def string) string { val, ok := event[key].(string) if !ok { diff --git a/policies/redis/producer/streams.go b/policies/redis/producer/streams.go index cd6836776..e3d3efc1d 100644 --- a/policies/redis/producer/streams.go +++ b/policies/redis/producer/streams.go @@ -229,6 +229,42 @@ func (e eventStore) ValidatePolicy(ctx context.Context, token string, p policies return e.svc.ValidatePolicy(ctx, token, p, format, policyData) } +func (e eventStore) DeleteSinkFromAllDatasetsInternal(ctx context.Context, sinkID string, token string) ([]policies.Dataset, error) { + return e.svc.DeleteSinkFromAllDatasetsInternal(ctx, sinkID, token) +} + +func (e eventStore) InactivateDatasetByIDInternal(ctx context.Context, ownerID string, datasetID string) error { + ds, err := e.svc.ViewDatasetByIDInternal(ctx, ownerID, datasetID) + if err != nil { + return err + } + + if err := e.svc.InactivateDatasetByIDInternal(ctx, ownerID, datasetID); err != nil { + return err + } + + event := removeDatasetEvent{ + id: datasetID, + ownerID: ds.MFOwnerID, + agentGroupID: ds.AgentGroupID, + policyID: ds.PolicyID, + datasetID: ds.ID, + } + record := &redis.XAddArgs{ + Stream: streamID, + MaxLenApprox: streamLen, + Values: event.Encode(), + } + + err = e.client.XAdd(ctx, record).Err() + if err != nil { + e.logger.Error("error sending event to event store", zap.Error(err)) + return err + } + + return nil +} + // NewEventStoreMiddleware returns wrapper around policies service that sends // events to event store. func NewEventStoreMiddleware(svc policies.Service, client *redis.Client, logger *zap.Logger) policies.Service { diff --git a/python-test/docs/agent_groups/edit_agent_group_description_removing_description.md b/python-test/docs/agent_groups/edit_agent_group_description_removing_description.md new file mode 100644 index 000000000..a6a149df5 --- /dev/null +++ b/python-test/docs/agent_groups/edit_agent_group_description_removing_description.md @@ -0,0 +1,17 @@ +## Scenario: Edit Agent Group description removing description +## Steps: +1 - Create an agent group + +- REST API Method: POST +- endpoint: /agent_groups +- header: {authorization:token} + +2- Edit this group description using None + +- REST API Method: PUT +- endpoint: /agent_groups/group_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 200 (ok) and changes must be applied diff --git a/python-test/docs/agent_groups/edit_agent_group_name,_description_and_tags.md b/python-test/docs/agent_groups/edit_agent_group_name,_description_and_tags.md new file mode 100644 index 000000000..2bca63469 --- /dev/null +++ b/python-test/docs/agent_groups/edit_agent_group_name,_description_and_tags.md @@ -0,0 +1,17 @@ +## Scenario: Edit Agent Group name, description and tags +## Steps: +1 - Create an agent group + +- REST API Method: POST +- endpoint: /agent_groups +- header: {authorization:token} + +2- Edit this group name, description and tags + +- REST API Method: PUT +- endpoint: /agent_groups/group_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 200 (ok) and changes must be applied diff --git a/python-test/docs/agent_groups/edit_agent_group_name_removing_name.md b/python-test/docs/agent_groups/edit_agent_group_name_removing_name.md new file mode 100644 index 000000000..b67eb0f71 --- /dev/null +++ b/python-test/docs/agent_groups/edit_agent_group_name_removing_name.md @@ -0,0 +1,17 @@ +## Scenario: Edit Agent Group name removing name +## Steps: +1 - Create an agent group + +- REST API Method: POST +- endpoint: /agent_groups +- header: {authorization:token} + +2- Edit this group name using None + +- REST API Method: PUT +- endpoint: /agent_groups/group_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 400 (error) and changes must not be applied diff --git a/python-test/docs/agent_groups/edit_agent_group_removing_tags.md b/python-test/docs/agent_groups/edit_agent_group_removing_tags.md new file mode 100644 index 000000000..d4e1d1d9b --- /dev/null +++ b/python-test/docs/agent_groups/edit_agent_group_removing_tags.md @@ -0,0 +1,17 @@ +## Scenario: Edit Agent Group removing tags +## Steps: +1 - Create an agent group + +- REST API Method: POST +- endpoint: /agent_groups +- header: {authorization:token} + +2- Edit this group tags using None + +- REST API Method: PUT +- endpoint: /agent_groups/group_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 400 (error) and changes must not be applied \ No newline at end of file diff --git a/python-test/docs/agent_groups/edit_agent_group_tags_to_subscribe_agent.md b/python-test/docs/agent_groups/edit_agent_group_tags_to_subscribe_agent.md new file mode 100644 index 000000000..c40f98416 --- /dev/null +++ b/python-test/docs/agent_groups/edit_agent_group_tags_to_subscribe_agent.md @@ -0,0 +1,13 @@ +## Scenario: Edit Agent Group tags to subscribe agent + +Steps: +- +1. Provision an agent with tags +2. Create a group with different tags +3. Edit groups' tags changing the value to match with agent + + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is subscribed to the group \ No newline at end of file diff --git a/python-test/docs/agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md b/python-test/docs/agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md new file mode 100644 index 000000000..c915b7636 --- /dev/null +++ b/python-test/docs/agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md @@ -0,0 +1,11 @@ +## Scenario: Edit Agent Group tags to unsubscribe agent +Steps: +- +1. Provision an agent with tags +2. Create a group with same tags +3. Edit groups' tags changing the value + +Expected result: +- +- Agent heartbeat must show 0 group matching +- Agent logs must show that agent is unsubscribed to the group \ No newline at end of file diff --git a/python-test/docs/development_guide.md b/python-test/docs/development_guide.md index b1d037e45..c2efb1b29 100644 --- a/python-test/docs/development_guide.md +++ b/python-test/docs/development_guide.md @@ -1,37 +1,46 @@ ## **INTEGRATION** -| Integration Scenario | Automated via API | Automated via UI | Smoke | Sanity | -|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-----------------:|:----------------:|:-----:|:------:| -| [Check if sink is active while scraping metrics](integration/sink_active_while_scraping_metrics.md) | ✅ | | 👍 | 👍 | -| [Check if sink with invalid credentials becomes active](integration/sink_error_invalid_credentials.md) | ✅ | | 👍 | 👍 | -| [Check if after 30 minutes without data sink becomes idle](integration/sink_idle_30_minutes.md) | | | | | -| [Provision agent before group (check if agent subscribes to the group)](integration/provision_agent_before_group.md) | ✅ | | 👍 | 👍 | -| [Provision agent after group (check if agent subscribes to the group)](integration/provision_agent_after_group.md) | ✅ | | 👍 | 👍 | -| [Provision agent with tag matching existing group linked to a valid dataset](integration/multiple_agents_subscribed_to_a_group.md) | ✅ | | 👍 | 👍 | -| [Apply multiple policies to a group](integration/apply_multiple_policies.md) | ✅ | | 👍 | 👍 | -| [Apply multiple policies to a group and remove one policy](integration/remove_one_of_multiple_policies.md) | ✅ | | 👍 | 👍 | -| [Apply multiple policies to a group and remove all of them](integration/remove_all_policies.md) | | | | | -| [Apply multiple policies to a group and remove one dataset](integration/remove_one_of_multiple_datasets.md) | ✅ | | 👍 | 👍 | -| [Apply multiple policies to a group and remove all datasets](integration/remove_all_datasets.md) | | | | | -| [Apply the same policy twice to the agent](integration/apply_policy_twice.md) | ✅ | | 👍 | 👍 | -| [Delete sink linked to a dataset, create another one and edit dataset using new sink](integration/change_sink_on_dataset.md) | | | | | -| [Remove one of multiples datasets that apply the same policy to the agent](integration/remove_one_dataset_of_multiples_with_same_policy.md) | | | | | -| [Remove group (invalid dataset, agent logs)](integration/remove_group.md) | ✅ | | 👍 | 👍 | -| [Remove sink (invalid dataset, agent logs)](integration/remove_sink.md) | | | 👍 | 👍 | -| [Remove policy (invalid dataset, agent logs, heartbeat)](integration/remove_policy.md) | ✅ | | 👍 | 👍 | -| [Remove dataset (check agent logs, heartbeat)](integration/remove_dataset.md) | ✅ | | 👍 | 👍 | -| [Remove agent container (logs, agent groups matches)](integration/remove_agent_container.md) | | | 👍 | 👍 | -| [Remove agent container force (logs, agent groups matches)](integration/remove_agent_container_force.md) | | | 👍 | 👍 | -| [Remove agent (logs, agent groups matches)](integration/remove_agent.md) | | | 👍 | 👍 | -| [Subscribe an agent to multiple groups created before agent provisioning](integration/subscribe_an_agent_to_multiple_groups_created_before_agent_provisioning.md) | ✅ | | 👍 | 👍 | -| [Subscribe an agent to multiple groups created after agent provisioning](integration/subscribe_an_agent_to_multiple_groups_created_after_agent_provisioning.md) | ✅ | | 👍 | 👍 | -| [Agent subscription to group after editing agent's tags (editing before agent provision)](integration/agent_subscription_to_group_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | -| [Agent subscription to group after editing agent's tags (editing after agent provision and after groups creation)](integration/agent_subscription_to_group_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | -| [Agent subscription to group after editing agent's tags (editing after agent provision and before second group creation)](integration/agent_subscription_to_group_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | -| [Agent subscription to group with policies after editing agent's tags](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | -| [Agent subscription to multiple groups with policies after editing agent's tags](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | -| [Edit agent name and apply policies to then](integration/edit_agent_name_and_apply_policies_to_then.md) | ✅ | | 👍 | 👍 | -| [Insert tags in agents created without tags and apply policies to group matching new tags.md](integration/insert_tags_in_agents_created_without_tags_and_apply_policies_to_group_matching_new_tags.md) | ✅ | | 👍 | 👍 | +| Integration Scenario | Automated via API | Automated via UI | Smoke | Sanity | +|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-----------------:|:----------------:|:-----:|:------:| +| [Check if sink is active while scraping metrics](integration/sink_active_while_scraping_metrics.md) | ✅ | | 👍 | 👍 | +| [Check if sink with invalid credentials becomes active](integration/sink_error_invalid_credentials.md) | ✅ | | 👍 | 👍 | +| [Check if after 30 minutes without data sink becomes idle](integration/sink_idle_30_minutes.md) | | | | | +| [Provision agent before group (check if agent subscribes to the group)](integration/provision_agent_before_group.md) | ✅ | | 👍 | 👍 | +| [Provision agent after group (check if agent subscribes to the group)](integration/provision_agent_after_group.md) | ✅ | | 👍 | 👍 | +| [Provision agent with tag matching existing group linked to a valid dataset](integration/multiple_agents_subscribed_to_a_group.md) | ✅ | | 👍 | 👍 | +| [Apply multiple simple policies to a group](integration/apply_multiple_policies.md) | ✅ | | 👍 | 👍 | +| [Apply multiple advanced policies to a group](integration/apply_multiple_policies.md) | ✅ | | 👍 | 👍 | +| [Apply multiple policies to a group and remove one policy](integration/remove_one_of_multiple_policies.md) | ✅ | | 👍 | 👍 | +| [Apply multiple policies to a group and remove all of them](integration/remove_all_policies.md) | | | | | +| [Apply multiple policies to a group and remove one dataset](integration/remove_one_of_multiple_datasets.md) | ✅ | | 👍 | 👍 | +| [Apply multiple policies to a group and remove all datasets](integration/remove_all_datasets.md) | | | | | +| [Apply the same policy twice to the agent](integration/apply_policy_twice.md) | ✅ | | 👍 | 👍 | +| [Delete sink linked to a dataset, create another one and edit dataset using new sink](integration/change_sink_on_dataset.md) | | | | | +| [Remove one of multiples datasets that apply the same policy to the agent](integration/remove_one_dataset_of_multiples_with_same_policy.md) | | | | | +| [Remove group (invalid dataset, agent logs)](integration/remove_group.md) | ✅ | | 👍 | 👍 | +| [Remove sink (invalid dataset, agent logs)](integration/remove_sink.md) | | | 👍 | 👍 | +| [Remove policy (invalid dataset, agent logs, heartbeat)](integration/remove_policy.md) | ✅ | | 👍 | 👍 | +| [Remove dataset (check agent logs, heartbeat)](integration/remove_dataset.md) | ✅ | | 👍 | 👍 | +| [Remove agent container (logs, agent groups matches)](integration/remove_agent_container.md) | | | 👍 | 👍 | +| [Remove agent container force (logs, agent groups matches)](integration/remove_agent_container_force.md) | | | 👍 | 👍 | +| [Remove agent (logs, agent groups matches)](integration/remove_agent.md) | | | 👍 | 👍 | +| [Subscribe an agent to multiple groups created before agent provisioning](integration/subscribe_an_agent_to_multiple_groups_created_before_agent_provisioning.md) | ✅ | | 👍 | 👍 | +| [Subscribe an agent to multiple groups created after agent provisioning](integration/subscribe_an_agent_to_multiple_groups_created_after_agent_provisioning.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to group after editing agent's tags (editing before agent provision)](integration/agent_subscription_to_group_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to group after editing agent's tags (editing after agent provision and after groups creation)](integration/agent_subscription_to_group_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to group after editing agent's tags (editing after agent provision and before second group creation)](integration/agent_subscription_to_group_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to group with policies after editing agent's tags](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to multiple groups with policies after editing agent's tags](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md) | ✅ | | 👍 | 👍 | +| [Edit agent name and apply policies to then](integration/edit_agent_name_and_apply_policies_to_then.md) | ✅ | | 👍 | 👍 | +| [Insert tags in agents created without tags and apply policies to group matching new tags.md](integration/insert_tags_in_agents_created_without_tags_and_apply_policies_to_group_matching_new_tags.md) | ✅ | | 👍 | 👍 | +| [Agent unsubscription to group with policies after editing agent group's tags (editing tags after agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to group with policies after editing agent group's tags (editing tags after agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md) | ✅ | | 👍 | 👍 | +| [Agent unsubscription to group with policies after editing agent group's tags (editing tags before agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to group with policies after editing agent group's tags (editing tags before agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md) | ✅ | | 👍 | 👍 | +| [Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md) | ✅ | | 👍 | 👍 | +| [Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md) | ✅ | | 👍 | 👍 | +| [Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md) | ✅ | | 👍 | 👍 | --------------------------------- ## **LOGIN** @@ -93,21 +102,27 @@ | [Create agent group with invalid name (regex)](agent_groups/create_agent_group_with_invalid_name_(regex).md) | | | | 👍 | | [Create agent group with duplicate name](agent_groups/create_agent_group_with_duplicate_name.md) | | | | 👍 | | [Create agent group with description](agent_groups/create_agent_group_with_description.md) | ✅ | | 👍 | 👍 | -| [Create agent group without description](agent_groups/create_agent_group_without_description.md) | | | | 👍 | +| [Create agent group without description](agent_groups/create_agent_group_without_description.md) | ✅ | | | 👍 | | [Create agent group without tag](agent_groups/create_agent_group_without_tag.md) | ✅ | | | 👍 | | [Create agent group with one tag](agent_groups/create_agent_group_with_one_tag.md) | ✅ | | 👍 | 👍 | | [Create agent group with multiple tags](agent_groups/create_agent_group_with_multiple_tags.md) | ✅ | | | 👍 | | [Test agent groups filters](agent_groups/test_agent_groups_filters.md) | | | | | -| [Visualize matching agents](agent_groups/visualize_matching_agents.md) | | | | 👍 | +| [Visualize matching agents](agent_groups/visualize_matching_agents.md) | ✅ | | | 👍 | | [Check agent groups details](agent_groups/check_agent_groups_details.md) | | | | 👍 | | [Edit an agent group through the details modal](agent_groups/edit_an_agent_group_through_the_details_modal.md) | | | | 👍 | | [Check if is possible cancel operations with no change](agent_groups/check_if_is_possible_cancel_operations_with_no_change.md) | | | | | -| [Edit agent group name](agent_groups/edit_agent_group_name.md) | | | 👍 | 👍 | -| [Edit agent group description](agent_groups/edit_agent_group_description.md) | | | | 👍 | -| [Edit agent group tag](agent_groups/edit_agent_group_tag.md) | | | 👍 | 👍 | +| [Edit agent group name](agent_groups/edit_agent_group_name.md) | ✅ | | 👍 | 👍 | +| [Edit agent group description](agent_groups/edit_agent_group_description.md) | ✅ | | | 👍 | +| [Edit agent group tag](agent_groups/edit_agent_group_tag.md) | ✅ | | 👍 | 👍 | | [Remove agent group using correct name](agent_groups/remove_agent_group_using_correct_name.md) | ✅ | | 👍 | 👍 | | [Remove agent group using incorrect name](agent_groups/remove_agent_group_using_incorrect_name.md) | | | | 👍 | - + | [Edit Agent Group name removing name](agent_groups/edit_agent_group_name_removing_name.md) | ✅ | | 👍 | 👍 | + | [Edit Agent Group description removing description](agent_groups/edit_agent_group_description_removing_description.md) | ✅ | | 👍 | 👍 | + | [Edit Agent Group description](agent_groups/edit_agent_group_description.md) | ✅ | | 👍 | 👍 | + | [Edit Agent Group tags to unsubscribe agent](agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md) | ✅ | | 👍 | 👍 | + | [Edit Agent Group tags to unsubscribe agent](agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md) | ✅ | | 👍 | 👍 | + | [Edit Agent Group removing tags](agent_groups/edit_agent_group_removing_tags.md) | ✅ | | 👍 | 👍 | + | [Edit Agent Group name, description and tags](agent_groups/edit_agent_group_name,_description_and_tags.md) | ✅ | | 👍 | 👍 | --------------------------------- ## **SINK** @@ -142,27 +157,36 @@ ## **POLICIES** -| Policies Scenario | Automated via API | Automated via UI | Smoke | Sanity | -|:--------------------------------------------------------------------------------------------------------------------------:|:-----------------:|:----------------:|:-----:|:------:| -| [Check if total policies on policies' page is correct](policies/check_if_total_policies_on_policies'_page_is_correct.md) | | | | | -| [Create policy with invalid name (regex)](policies/create_policy_with_invalid_name_(regex).md) | | | | 👍 | -| [Create policy with no agent provisioned](policies/create_policy_with_no_agent_provisioned.md) | | | | 👍 | -| [Create policy with duplicate name](policies/create_policy_with_duplicate_name.md) | | | | 👍 | -| [Create policy with description](policies/create_policy_with_description.md) | ✅ | | 👍 | 👍 | -| [Create policy without description](policies/create_policy_without_description.md) | | | | 👍 | -| [Create policy with dhcp handler](policies/create_policy_with_dhcp_handler.md) | ✅ | | 👍 | 👍 | -| [Create policy with dns handler](policies/create_policy_with_dns_handler.md) | ✅ | | 👍 | 👍 | -| [Create policy with net handler](policies/create_policy_with_net_handler.md) | ✅ | | 👍 | 👍 | -| [Create policy with multiple handlers](policies/create_policy_with_multiple_handlers.md) | | | | 👍 | -| [Test policy filters](policies/test_policy_filters.md) | | | | -| [Check policies details](policies/check_policies_details.md) | | | | 👍 | -| [Edit a policy through the details modal](policies/edit_a_policy_through_the_details_modal.md) | | | | 👍 | -| [Edit policy name](policies/edit_policy_name.md) | | | | 👍 | -| [Edit policy description](policies/edit_policy_description.md) | | | | 👍 | -| [Edit policy handler](policies/edit_policy_handler.md) | | | 👍 | 👍 | -| [Check if is possible cancel operations with no change](policies/check_if_is_possible_cancel_operations_with_no_change.md) | | | | | -| [Remove policy using correct name](policies/remove_policy_using_correct_name.md) | ✅ | | 👍 | 👍 | -| [Remove policy using incorrect name](policies/remove_policy_using_incorrect_name.md) | | | | 👍 | +| Policies Scenario | Automated via API | Automated via UI | Smoke | Sanity | +|:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:-----------------:|:----------------:|:-----:|:------:| +| [Check if total policies on policies' page is correct](policies/check_if_total_policies_on_policies'_page_is_correct.md) | | | | | +| [Create policy with invalid name (regex)](policies/create_policy_with_invalid_name_(regex).md) | | | | 👍 | +| [Create policy with no agent provisioned](policies/create_policy_with_no_agent_provisioned.md) | | | | 👍 | +| [Create policy with duplicate name](policies/create_policy_with_duplicate_name.md) | | | | 👍 | +| [Create policy with description](policies/create_policy_with_description.md) | ✅ | | 👍 | 👍 | +| [Create policy without description](policies/create_policy_without_description.md) | ✅ | | | 👍 | +| [Create a policy with dns handler, description, host specification, bpf filter, pcap source, only qname suffix and only rcode](policies/create_policy_with_dns_handler.md) | ✅ | | 👍 | 👍 | +| [Create a policy with dns handler, host specification, bpf filter, pcap source, only qname suffix and only rcode](policies/create_policy_with_dns_handler.md) | ✅ | | 👍 | 👍 | +| [Create a policy with dns handler, bpf filter, pcap source, only qname suffix and only rcode](policies/create_policy_with_dns_handler.md) | ✅ | | 👍 | 👍 | +| [Create a policy with dns handler, pcap source, only qname suffix and only rcode](policies/create_policy_with_dns_handler.md) | ✅ | | 👍 | 👍 | +| [Create a policy with dns handler, only qname suffix](policies/create_policy_with_dns_handler.md) | ✅ | | 👍 | 👍 | +| [Create a policy with net handler, description, host specification, bpf filter and pcap source](policies/create_policy_with_net_handler.md) | ✅ | | 👍 | 👍 | +| [Create a policy with dhcp handler, description, host specification, bpf filter and pcap source](policies/create_policy_with_dhcp_handler.md) | ✅ | | 👍 | 👍 | +| [Create policy with multiple handlers](policies/create_policy_with_multiple_handlers.md) | | | | 👍 | +| [Test policy filters](policies/test_policy_filters.md) | | | | +| [Check policies details](policies/check_policies_details.md) | | | | 👍 | +| [Edit a policy through the details modal](policies/edit_a_policy_through_the_details_modal.md) | | | | 👍 | +| [Edit policy name](policies/edit_policy_name.md) | ✅ | | 👍 | 👍 | +| [Edit policy host_specification](policies/edit_policy_host_specification.md) | ✅ | | 👍 | 👍 | +| [Edit policy bpf_filter_expression](policies/edit_policy_bpf_filter_expression.md) | ✅ | | 👍 | 👍 | +| [Edit policy pcap_source](policies/edit_policy_pcap_source.md) | ✅ | | 👍 | 👍 | +| [Edit policy only_qname_suffix](policies/edit_policy_only_qname_suffix.md) | ✅ | | 👍 | 👍 | +| [Edit policy only_rcode](policies/edit_policy_only_rcode.md) | ✅ | | 👍 | 👍 | +| [Edit policy description](policies/edit_policy_description.md) | ✅ | | | 👍 | +| [Edit policy handler](policies/edit_policy_handler.md) | ✅ | | 👍 | 👍 | +| [Check if is possible cancel operations with no change](policies/check_if_is_possible_cancel_operations_with_no_change.md) | | | | | +| [Remove policy using correct name](policies/remove_policy_using_correct_name.md) | ✅ | | 👍 | 👍 | +| [Remove policy using incorrect name](policies/remove_policy_using_incorrect_name.md) | | | | 👍 | --------------------------------- diff --git a/python-test/docs/index.md b/python-test/docs/index.md index 828e8a67a..0249602a7 100644 --- a/python-test/docs/index.md +++ b/python-test/docs/index.md @@ -1,4 +1,4 @@ -## Login +## Login Scenarios: - [Request registration of a registered account using registered password username and company](login/request_registration_of_a_registered_account_using_registered_password_username_and_company.md) - [Request registration of a registered account using registered password and username](login/request_registration_of_a_registered_account_using_registered_password_and_username.md) @@ -18,8 +18,7 @@ - [Request password with registered email address](login/request_password_with_registered_email_address.md) - [Request password with unregistered email address](login/request_password_with_unregistered_email_address.md) - -## Agents +## Agents Scenarios: - [Check if total agent on agents' page is correct](agents/check_if_total_agent_on_agents'_page_is_correct.md) - [Create agent without tags](agents/create_agent_without_tags.md) @@ -37,13 +36,10 @@ - [Check if is possible cancel operations with no change](agents/check_if_is_possible_cancel_operations_with_no_change.md) - [Remove agent using correct name](agents/remove_agent_using_correct_name.md) - [Remove agent using incorrect name](agents/remove_agent_using_incorrect_name.md) -- [Run two orb agents on the same port](agents/run_two_orb_agents_on_the_same_port.md) -- [Run two orb agents on different ports](agents/run_two_orb_agents_on_different_ports.md) - [Edit agent name and tag](agents/edit_agent_name_and_tags.md) -## Agent Groups - +## Agent Groups Scenarios: - [Check if total agent groups on agent groups' page is correct](agent_groups/check_if_total_agent_groups_on_agent_groups'_page_is_correct.md) - [Create agent group with invalid name (regex)](agent_groups/create_agent_group_with_invalid_name_(regex).md) - [Create agent group with duplicate name](agent_groups/create_agent_group_with_duplicate_name.md) @@ -57,15 +53,21 @@ - [Check agent groups details](agent_groups/check_agent_groups_details.md) - [Edit an agent group through the details modal](agent_groups/edit_an_agent_group_through_the_details_modal.md) - [Check if is possible cancel operations with no change](agent_groups/check_if_is_possible_cancel_operations_with_no_change.md) +- [Remove agent group using correct name](agent_groups/remove_agent_group_using_correct_name.md) +- [Remove agent group using incorrect name](agent_groups/remove_agent_group_using_incorrect_name.md) +- [Run two orb agents on the same port](agents/run_two_orb_agents_on_the_same_port.md) +- [Run two orb agents on different ports](agents/run_two_orb_agents_on_different_ports.md) +- [Edit Agent Group name removing name](agent_groups/edit_agent_group_name_removing_name.md) - [Edit agent group name](agent_groups/edit_agent_group_name.md) - [Edit agent group description](agent_groups/edit_agent_group_description.md) +- [Edit Agent Group description removing description](agent_groups/edit_agent_group_description_removing_description.md) - [Edit agent group tag](agent_groups/edit_agent_group_tag.md) -- [Remove agent group using correct name](agent_groups/remove_agent_group_using_correct_name.md) -- [Remove agent group using incorrect name](agent_groups/remove_agent_group_using_incorrect_name.md) - - -## Sinks +- [Edit Agent Group tags to subscribe agent](agent_groups/edit_agent_group_tags_to_subscribe_agent.md) +- [Edit Agent Group tags to unsubscribe agent](agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md) +- [Edit Agent Group removing tags](agent_groups/edit_agent_group_removing_tags.md) +- [Edit Agent Group name, description and tags](agent_groups/edit_agent_group_name,_description_and_tags.md) +## Sink Scenarios: - [Check if total sinks on sinks' page is correct](sinks/check_if_total_sinks_on_sinks'_page_is_correct.md) - [Create sink with invalid name (regex)](sinks/create_sink_with_invalid_name_(regex).md) - [Create sink with duplicate name](sinks/create_sink_with_duplicate_name.md) @@ -88,8 +90,7 @@ - [Remove sink using correct name](sinks/remove_sink_using_correct_name.md) - [Remove sink using incorrect name](sinks/remove_sink_using_incorrect_name.md) -## Policies - +## Policies Scenarios: - [Check if total policies on policies' page is correct](policies/check_if_total_policies_on_policies'_page_is_correct.md) - [Create policy with invalid name (regex)](policies/create_policy_with_invalid_name_(regex).md) - [Create policy with no agent provisioned](policies/create_policy_with_no_agent_provisioned.md) @@ -104,14 +105,18 @@ - [Check policies details](policies/check_policies_details.md) - [Edit a policy through the details modal](policies/edit_a_policy_through_the_details_modal.md) - [Edit policy name](policies/edit_policy_name.md) +- [Edit policy host_specification](policies/edit_policy_host_specification.md) +- [Edit policy bpf_filter_expression](policies/edit_policy_bpf_filter_expression.md) +- [Edit policy pcap_source](policies/edit_policy_pcap_source.md) +- [Edit policy only_qname_suffix](policies/edit_policy_only_qname_suffix.md) +- [Edit policy only_rcode](policies/edit_policy_only_rcode.md) - [Edit policy description](policies/edit_policy_description.md) - [Edit policy handler](policies/edit_policy_handler.md) - [Check if is possible cancel operations with no change](policies/check_if_is_possible_cancel_operations_with_no_change.md) - [Remove policy using correct name](policies/remove_policy_using_correct_name.md) - [Remove policy using incorrect name](policies/remove_policy_using_incorrect_name.md) -## Datasets - +## Datasets Scenarios: - [Check if total datasets on datasets' page is correct](datasets/check_if_total_datasets_on_datasets'_page_is_correct.md) - [Create dataset with invalid name (regex)](datasets/create_dataset_with_invalid_name_(regex).md) - [Create dataset](datasets/create_dataset.md) @@ -124,14 +129,13 @@ - [Remove dataset using correct name](datasets/remove_dataset_using_correct_name.md) - [Remove dataset using incorrect name](datasets/remove_dataset_using_incorrect_name.md) -## Integration tests - +## Integration Scenarios: - [Check if sink is active while scraping metrics](integration/sink_active_while_scraping_metrics.md) - [Check if sink with invalid credentials becomes active](integration/sink_error_invalid_credentials.md) - [Check if after 30 minutes without data sink becomes idle](integration/sink_idle_30_minutes.md) - [Provision agent before group (check if agent subscribes to the group)](integration/provision_agent_before_group.md) - [Provision agent after group (check if agent subscribes to the group)](integration/provision_agent_after_group.md) -- [Create agent with tag matching existing group linked to a valid dataset](integration/multiple_agents_subscribed_to_a_group.md) +- [Provision agent with tag matching existing group linked to a valid dataset](integration/multiple_agents_subscribed_to_a_group.md) - [Apply multiple policies to a group](integration/apply_multiple_policies.md) - [Apply multiple policies to a group and remove one policy](integration/remove_one_of_multiple_policies.md) - [Apply multiple policies to a group and remove all of them](integration/remove_all_policies.md) @@ -153,6 +157,14 @@ - [Agent subscription to group with policies after editing agent's tags](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md) - [Edit agent name and apply policies to then](integration/edit_agent_name_and_apply_policies_to_then.md) - [Insert tags in agents created without tags and apply policies to group matching new tags.md](integration/insert_tags_in_agents_created_without_tags_and_apply_policies_to_group_matching_new_tags.md) +- [Agent unsubscription to group with policies after editing agent group's tags (editing tags after agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent subscription to group with policies after editing agent group's tags (editing tags after agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent group's tags (editing tags before agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent subscription to group with policies after editing agent group's tags (editing tags before agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md) ## Pktvisor Agent diff --git a/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md b/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md new file mode 100644 index 000000000..008d471f6 --- /dev/null +++ b/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md @@ -0,0 +1,18 @@ +## Scenario: Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision) + +Steps: +- +1. Provision an agent with tags +2. Create a group with same tags as agent +3. Create a sink +4. Create 1 policy +5. Create a dataset linking the group, the sink and the policy +6. Edit groups' tags changing the value +7. Edit agent orb tags to match with new groups tags + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is unsubscribed from the group +- Agent logs must show that agent is resubscribed to the group + diff --git a/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md b/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md new file mode 100644 index 000000000..417e90160 --- /dev/null +++ b/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md @@ -0,0 +1,17 @@ +## Scenario: Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision) +Steps: +- +1. Create an agent with tags +2. Create a group with same tags as agent +3. Create a sink +4. Create 1 policy +5. Create a dataset linking the group, the sink and the policy +6. Edit groups' tags changing the value +7. Edit agent orb tags to match with new groups tags +8. Provision the agent + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is subscribed to the group + \ No newline at end of file diff --git a/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md b/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md new file mode 100644 index 000000000..2cb61a9cd --- /dev/null +++ b/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md @@ -0,0 +1,15 @@ +## Scenario: Agent subscription to group with policies after editing agent group's tags (editing tags after agent provision) +Steps: +- +1. Provision an agent with tags +2. Create a group with different tags +3. Create a sink +4. Create 1 policy +5. Create a dataset linking the group, the sink and the policy +6. Edit groups' tags changing the value to match with agent + + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is subscribed to the group \ No newline at end of file diff --git a/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md b/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md new file mode 100644 index 000000000..dc753c51d --- /dev/null +++ b/python-test/docs/integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md @@ -0,0 +1,16 @@ +## Scenario: Agent subscription to group with policies after editing agent group's tags (editing tags before agent provision) +Steps: +- +1. Create an agent with tags +2. Create a group with different tags +3. Create a sink +4. Create 1 policy +5. Create a dataset linking the group, the sink and the policy +6. Edit groups' tags changing the value to match with agent +7. Provision the agent + + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is subscribed to the group \ No newline at end of file diff --git a/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md b/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md new file mode 100644 index 000000000..6ccfc7004 --- /dev/null +++ b/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md @@ -0,0 +1,16 @@ +## Scenario: Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision) +Steps: +- +1. Provision an agent with tags +2. Create a group with another tag +3. Create a sink +4. Create 1 policy +5. Create a dataset linking the group, the sink and the policy +6. Edit agent tags using the same tag as the group +7. Edit groups' tags using a different one + + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is unsubscribed to the group \ No newline at end of file diff --git a/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md b/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md new file mode 100644 index 000000000..7df7e9a24 --- /dev/null +++ b/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md @@ -0,0 +1,16 @@ +## Scenario: Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision) +Steps: +- +1. Create an agent with tags +2. Create a group with another tag +3. Create a sink +4. Create 1 policy +5. Create a dataset linking the group, the sink and the policy +6. Edit agent tags using the same tag as the group +7. Edit groups' tags using a different one +8. Provision the agent + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is unsubscribed to the group \ No newline at end of file diff --git a/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md b/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md new file mode 100644 index 000000000..d3cd831d4 --- /dev/null +++ b/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md @@ -0,0 +1,15 @@ +## Scenario: Agent unsubscription to group with policies after editing agent group's tags (editing tags after agent provision) +Steps: +- +1. Provision an agent with tags +2. Create a group with same tags as the agent +3. Create a sink +4. Create 1 policy +5. Create a dataset linking the group, the sink and the policy +6. Edit groups' tags changing the value + + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is unsubscribed to the group \ No newline at end of file diff --git a/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md b/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md new file mode 100644 index 000000000..8eb71955d --- /dev/null +++ b/python-test/docs/integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md @@ -0,0 +1,16 @@ +## Scenario: Agent unsubscription to group with policies after editing agent group's tags (editing tags before agent provision) +Steps: +- +1. Create an agent with tags +2. Create a group with same tags as the agent +3. Create a sink +4. Create 1 policy +5. Create a dataset linking the group, the sink and the policy +6. Edit groups' tags changing the value +7. Provision the agent + + +Expected result: +- +- Agent heartbeat must show just one group matching +- Agent logs must show that agent is unsubscribed to the group \ No newline at end of file diff --git a/python-test/docs/integration/apply_multiple_policies.md b/python-test/docs/integration/apply_multiple_policies.md index f629a8af4..b6f7299a7 100644 --- a/python-test/docs/integration/apply_multiple_policies.md +++ b/python-test/docs/integration/apply_multiple_policies.md @@ -1,4 +1,24 @@ -## Scenario: apply multiple policies to agents subscribed to a group +## 1- Scenario: apply multiple advanced policies to agents subscribed to a group + +Steps: +- +1. Provision an agent with tags +2. Create a group with same tags as agent +3. Create a sink +4. Create multiple advanced policies (with filters, source pcap) +5. Create a dataset linking the group, the sink and one of the policies +6. Create another dataset linking the same group, sink and the other policy + +Expected result: +- +- All the policies must be applied to the agent (orb-agent API response) +- The container logs contain the message "policy applied successfully" referred to each policy +- The container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy +- Referred sink must have active state on response +- Datasets related to all existing policies have validity valid + + +## 2- Scenario: apply multiple simple policies to agents subscribed to a group Steps: - diff --git a/python-test/docs/policies/create_policy_with_dhcp_handler.md b/python-test/docs/policies/create_policy_with_dhcp_handler.md index 373ad59ff..2e46898f9 100644 --- a/python-test/docs/policies/create_policy_with_dhcp_handler.md +++ b/python-test/docs/policies/create_policy_with_dhcp_handler.md @@ -1,12 +1,10 @@ ## Scenario: Create policy with dhcp handler -## Steps: - -1 - Create a policy with dhcp handler +## 1 - Create a policy with dhcp handler, description, host specification, bpf filter and pcap source - REST API Method: POST - endpoint: /policies/agent/ - header: {authorization:token} -## Expected Result: +### Expected Result: - Request must have status code 201 (created) and the policy must be created \ No newline at end of file diff --git a/python-test/docs/policies/create_policy_with_dns_handler.md b/python-test/docs/policies/create_policy_with_dns_handler.md index c69134248..d6e029d1e 100644 --- a/python-test/docs/policies/create_policy_with_dns_handler.md +++ b/python-test/docs/policies/create_policy_with_dns_handler.md @@ -1,12 +1,54 @@ ## Scenario: Create policy with dns handler -## Steps: -1 - Create a policy with dns handler +## 1 - Create a policy with dns handler, description, host specification, bpf filter, pcap source, only qname suffix and only rcode - REST API Method: POST - endpoint: /policies/agent/ - header: {authorization:token} -## Expected Result: +### Expected Result: +- Request must have status code 201 (created) and the policy must be created + + +## 2 - Create a policy with dns handler, host specification, bpf filter, pcap source, only qname suffix and only rcode + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + + +### Expected Result: +- Request must have status code 201 (created) and the policy must be created + + +## 3 - Scenario: Create a policy with dns handler, bpf filter, pcap source, only qname suffix and only rcode + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + + +### Expected Result: +- Request must have status code 201 (created) and the policy must be created + +## 4 - Scenario: Create a policy with dns handler, pcap source, only qname suffix and only rcode + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + + +### Expected Result: +- Request must have status code 201 (created) and the policy must be created + + +## 5 - Scenario: Create a policy with dns handler, only qname suffix + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + + +### Expected Result: - Request must have status code 201 (created) and the policy must be created \ No newline at end of file diff --git a/python-test/docs/policies/create_policy_with_net_handler.md b/python-test/docs/policies/create_policy_with_net_handler.md index 32f7d62d9..042d27dac 100644 --- a/python-test/docs/policies/create_policy_with_net_handler.md +++ b/python-test/docs/policies/create_policy_with_net_handler.md @@ -1,12 +1,12 @@ ## Scenario: Create policy with net handler -## Steps: -1 - Create a policy with net handler +## 1 - Create a policy with net handler, description, host specification, bpf filter and pcap source - REST API Method: POST - endpoint: /policies/agent/ - header: {authorization:token} -## Expected Result: -- Request must have status code 201 (created) and the policy must be created \ No newline at end of file +### Expected Result: +- Request must have status code 201 (created) and the policy must be created + diff --git a/python-test/docs/policies/edit_policy_bpf_filter_expression.md b/python-test/docs/policies/edit_policy_bpf_filter_expression.md new file mode 100644 index 000000000..5d805a0e6 --- /dev/null +++ b/python-test/docs/policies/edit_policy_bpf_filter_expression.md @@ -0,0 +1,18 @@ +## Scenario: Edit policy bpf_filter_expression + +## Steps: +1 - Create a policy + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + +2- Edit this policy bpf_filter_expression + +- REST API Method: PUT +- endpoint: /policies/agent/policy_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 200 (ok) and changes must be applied \ No newline at end of file diff --git a/python-test/docs/policies/edit_policy_host_specification.md b/python-test/docs/policies/edit_policy_host_specification.md new file mode 100644 index 000000000..bd091f10d --- /dev/null +++ b/python-test/docs/policies/edit_policy_host_specification.md @@ -0,0 +1,18 @@ +## Scenario: Edit policy host_specification + +## Steps: +1 - Create a policy + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + +2- Edit this policy host_specification + +- REST API Method: PUT +- endpoint: /policies/agent/policy_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 200 (ok) and changes must be applied \ No newline at end of file diff --git a/python-test/docs/policies/edit_policy_only_qname_suffix.md b/python-test/docs/policies/edit_policy_only_qname_suffix.md new file mode 100644 index 000000000..425b2ec22 --- /dev/null +++ b/python-test/docs/policies/edit_policy_only_qname_suffix.md @@ -0,0 +1,17 @@ +## Scenario: Edit policy only_qname_suffix +## Steps: +1 - Create a policy + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + +2- Edit this policy only_qname_suffix + +- REST API Method: PUT +- endpoint: /policies/agent/policy_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 200 (ok) and changes must be applied \ No newline at end of file diff --git a/python-test/docs/policies/edit_policy_only_rcode.md b/python-test/docs/policies/edit_policy_only_rcode.md new file mode 100644 index 000000000..9069d88d4 --- /dev/null +++ b/python-test/docs/policies/edit_policy_only_rcode.md @@ -0,0 +1,17 @@ +## Scenario: Edit policy only_rcode +## Steps: +1 - Create a policy + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + +2- Edit this policy only_rcode + +- REST API Method: PUT +- endpoint: /policies/agent/policy_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 200 (ok) and changes must be applied \ No newline at end of file diff --git a/python-test/docs/policies/edit_policy_pcap_source.md b/python-test/docs/policies/edit_policy_pcap_source.md new file mode 100644 index 000000000..274fc720e --- /dev/null +++ b/python-test/docs/policies/edit_policy_pcap_source.md @@ -0,0 +1,17 @@ +## Scenario: Edit policy pcap_source +## Steps: +1 - Create a policy + +- REST API Method: POST +- endpoint: /policies/agent/ +- header: {authorization:token} + +2- Edit this policy pcap_source + +- REST API Method: PUT +- endpoint: /policies/agent/policy_id +- header: {authorization:token} + + +## Expected Result: +- Request must have status code 200 (ok) and changes must be applied \ No newline at end of file diff --git a/python-test/docs/sanity.md b/python-test/docs/sanity.md index b5ecf056a..72950d713 100644 --- a/python-test/docs/sanity.md +++ b/python-test/docs/sanity.md @@ -57,6 +57,16 @@ - [Edit agent group tag](agent_groups/edit_agent_group_tag.md) - [Remove agent group using correct name](agent_groups/remove_agent_group_using_correct_name.md) - [Remove agent group using incorrect name](agent_groups/remove_agent_group_using_incorrect_name.md) +- [Run two orb agents on the same port](agents/run_two_orb_agents_on_the_same_port.md) +- [Run two orb agents on different ports](agents/run_two_orb_agents_on_different_ports.md) +- [Edit Agent Group name removing name](agent_groups/edit_agent_group_name_removing_name.md) +- [Edit agent group name](agent_groups/edit_agent_group_name.md) +- [Edit agent group description](agent_groups/edit_agent_group_description.md) +- [Edit Agent Group description removing description](agent_groups/edit_agent_group_description_removing_description.md) +- [Edit Agent Group tags to subscribe agent](agent_groups/edit_agent_group_tags_to_subscribe_agent.md) +- [Edit Agent Group tags to unsubscribe agent](agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md) +- [Edit Agent Group removing tags](agent_groups/edit_agent_group_removing_tags.md) +- [Edit Agent Group name, description and tags](agent_groups/edit_agent_group_name,_description_and_tags.md) ## Sinks @@ -92,6 +102,11 @@ - [Check policies details](policies/check_policies_details.md) - [Edit a policy through the details modal](policies/edit_a_policy_through_the_details_modal.md) - [Edit policy name](policies/edit_policy_name.md) +- [Edit policy host_specification](policies/edit_policy_host_specification.md) +- [Edit policy bpf_filter_expression](policies/edit_policy_bpf_filter_expression.md) +- [Edit policy pcap_source](policies/edit_policy_pcap_source.md) +- [Edit policy only_qname_suffix](policies/edit_policy_only_qname_suffix.md) +- [Edit policy only_rcode](policies/edit_policy_only_rcode.md) - [Edit policy description](policies/edit_policy_description.md) - [Edit policy handler](policies/edit_policy_handler.md) - [Remove policy using correct name](policies/remove_policy_using_correct_name.md) @@ -131,3 +146,11 @@ - [Agent subscription to group with policies after editing agent's tags](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md) - [Edit agent name and apply policies to then](integration/edit_agent_name_and_apply_policies_to_then.md) - [Insert tags in agents created without tags and apply policies to group matching new tags.md](integration/insert_tags_in_agents_created_without_tags_and_apply_policies_to_group_matching_new_tags.md) +- [Agent unsubscription to group with policies after editing agent group's tags (editing tags after agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent subscription to group with policies after editing agent group's tags (editing tags after agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent group's tags (editing tags before agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent subscription to group with policies after editing agent group's tags (editing tags before agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md) diff --git a/python-test/docs/smoke.md b/python-test/docs/smoke.md index 8a6045140..c1a9d8658 100644 --- a/python-test/docs/smoke.md +++ b/python-test/docs/smoke.md @@ -38,6 +38,16 @@ - [Edit agent group name](agent_groups/edit_agent_group_name.md) - [Edit agent group tag](agent_groups/edit_agent_group_tag.md) - [Remove agent group using correct name](agent_groups/remove_agent_group_using_correct_name.md) +- [Run two orb agents on the same port](agents/run_two_orb_agents_on_the_same_port.md) +- [Run two orb agents on different ports](agents/run_two_orb_agents_on_different_ports.md) +- [Edit Agent Group name removing name](agent_groups/edit_agent_group_name_removing_name.md) +- [Edit agent group name](agent_groups/edit_agent_group_name.md) +- [Edit agent group description](agent_groups/edit_agent_group_description.md) +- [Edit Agent Group description removing description](agent_groups/edit_agent_group_description_removing_description.md) +- [Edit Agent Group tags to subscribe agent](agent_groups/edit_agent_group_tags_to_subscribe_agent.md) +- [Edit Agent Group tags to unsubscribe agent](agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md) +- [Edit Agent Group removing tags](agent_groups/edit_agent_group_removing_tags.md) +- [Edit Agent Group name, description and tags](agent_groups/edit_agent_group_name,_description_and_tags.md) ## Sinks @@ -86,3 +96,11 @@ - [Agent subscription to group with policies after editing agent's tags](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md) - [Edit agent name and apply policies to then](integration/edit_agent_name_and_apply_policies_to_then.md) - [Insert tags in agents created without tags and apply policies to group matching new tags.md](integration/insert_tags_in_agents_created_without_tags_and_apply_policies_to_group_matching_new_tags.md) +- [Agent unsubscription to group with policies after editing agent group's tags (editing tags after agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent subscription to group with policies after editing agent group's tags (editing tags after agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent group's tags (editing tags before agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent subscription to group with policies after editing agent group's tags (editing tags before agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md) +- [Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision)](integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md) +- [Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision)](integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md) diff --git a/python-test/features/agentGroups.feature b/python-test/features/agentGroups.feature index 87649d249..2952cf0eb 100644 --- a/python-test/features/agentGroups.feature +++ b/python-test/features/agentGroups.feature @@ -1,6 +1,7 @@ @agentGroups -Feature: agent groups creation - +Feature: agent groups creation + + @smoke Scenario: Create Agent Group with one tag Given the Orb user has a registered account And the Orb user logs in @@ -9,7 +10,7 @@ Feature: agent groups creation Then 1 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC subscription to group" within 10 seconds - + @sanity Scenario: Create Agent Group with multiple tags Given the Orb user has a registered account And the Orb user logs in @@ -18,14 +19,14 @@ Feature: agent groups creation Then 1 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC subscription to group" within 10 seconds - + @sanity Scenario: Create Agent Group without tags Given the Orb user has a registered account And the Orb user logs in When an Agent Group is created with 0 orb tag(s) Then Agent Group creation response must be an error with message 'malformed entity specification' - + @smoke Scenario: Create Agent Group without description Given the Orb user has a registered account And the Orb user logs in @@ -34,7 +35,7 @@ Feature: agent groups creation Then 1 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC subscription to group" within 10 seconds - + @smoke Scenario: Edit Agent Group name Given the Orb user has a registered account And the Orb user logs in @@ -44,7 +45,7 @@ Feature: agent groups creation Then 1 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC subscription to group" within 10 seconds - + @smoke Scenario: Agent Group name editing without name Given the Orb user has a registered account And the Orb user logs in @@ -55,7 +56,7 @@ Feature: agent groups creation And 1 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC subscription to group" within 10 seconds - + @smoke Scenario: Edit Agent Group description (without description) Given the Orb user has a registered account And the Orb user logs in @@ -65,7 +66,7 @@ Feature: agent groups creation Then 1 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC subscription to group" within 10 seconds - + @smoke Scenario: Edit Agent Group description (with description) Given the Orb user has a registered account And the Orb user logs in @@ -75,8 +76,8 @@ Feature: agent groups creation Then 1 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC subscription to group" within 10 seconds - - Scenario: Edit Agent Group tags (with tags - unsubscription) + @smoke + Scenario: Edit Agent Group tags (with tags - unsubscription) Given the Orb user has a registered account And the Orb user logs in And that an agent with region:br orb tag(s) already exists and is online @@ -85,8 +86,8 @@ Feature: agent groups creation Then 0 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC unsubscription to group" within 10 seconds - - Scenario: Edit Agent Group tags (with tags - subscription) + @smoke + Scenario: Edit Agent Group tags (with tags - subscription) Given the Orb user has a registered account And the Orb user logs in And that an agent with ns1:true orb tag(s) already exists and is online @@ -95,8 +96,8 @@ Feature: agent groups creation Then 1 agent must be matching on response field matching_agents And the container logs contain the message "completed RPC subscription to group" referred to each matching group within 10 seconds - - Scenario: Edit Agent Group tags (without tags) + @smoke + Scenario: Edit Agent Group tags (without tags) Given the Orb user has a registered account And the Orb user logs in And that an agent with region:br orb tag(s) already exists and is online @@ -106,8 +107,8 @@ Feature: agent groups creation And 1 agent must be matching on response field matching_agents And the agent status in Orb should be online - - Scenario: Edit Agent Group name, description and tags + @smoke + Scenario: Edit Agent Group name, description and tags Given the Orb user has a registered account And the Orb user logs in And that an agent with region:br orb tag(s) already exists and is online diff --git a/python-test/features/agentProviderUi.feature b/python-test/features/agentProviderUi.feature index c8084997b..b5214a14b 100644 --- a/python-test/features/agentProviderUi.feature +++ b/python-test/features/agentProviderUi.feature @@ -1,6 +1,7 @@ @agents_ui Feature: Create agents using orb ui + @smoke Scenario: Create agent Given the Orb user logs in through the UI And that fleet Management is clickable on ORB Menu @@ -8,6 +9,7 @@ Feature: Create agents using orb ui When a new agent is created through the UI with region:br, demo:true, ns1:true orb tag(s) Then the agents list and the agents view should display agent's status as New within 10 seconds + @smoke Scenario: Provision agent Given the Orb user logs in through the UI And that fleet Management is clickable on ORB Menu @@ -18,7 +20,7 @@ Feature: Create agents using orb ui And the agent status in Orb should be online And the container logs should contain the message "sending capabilities" within 10 seconds - + @smoke Scenario: Run two orb agents on the same port Given the Orb user logs in through the UI And that the user is on the orb Agent page @@ -31,6 +33,7 @@ Feature: Create agents using orb ui And the container logs should contain the message "agent startup error" within 2 seconds And container on port default is running after 2 seconds + @smoke Scenario: Run two orb agents on the same port Given the Orb user logs in through the UI And that the user is on the orb Agent page diff --git a/python-test/features/agentsProvider.feature b/python-test/features/agentsProvider.feature index e7633d98b..898d3a65e 100644 --- a/python-test/features/agentsProvider.feature +++ b/python-test/features/agentsProvider.feature @@ -1,6 +1,7 @@ @agents Feature: agent provider - + + @smoke Scenario: Provision agent Given the Orb user has a registered account And the Orb user logs in @@ -9,6 +10,7 @@ Feature: agent provider Then the agent status in Orb should be online And the container logs should contain the message "sending capabilities" within 10 seconds + @smoke Scenario: Run two orb agents on the same port Given the Orb user has a registered account And the Orb user logs in @@ -19,6 +21,7 @@ Feature: agent provider And the container logs should contain the message "agent startup error" within 2 seconds And container on port default is running after 2 seconds + @smoke Scenario: Run two orb agents on different ports Given the Orb user has a registered account And the Orb user logs in @@ -29,6 +32,7 @@ Feature: agent provider And container on port default is running after 2 seconds + @smoke Scenario: Provision agent without tags Given the Orb user has a registered account And the Orb user logs in @@ -38,6 +42,7 @@ Feature: agent provider And the container logs should contain the message "sending capabilities" within 10 seconds + @smoke Scenario: Provision agent with multiple tags Given the Orb user has a registered account And the Orb user logs in @@ -47,6 +52,7 @@ Feature: agent provider And the container logs should contain the message "sending capabilities" within 10 seconds + @smoke Scenario: Edit agent tag Given the Orb user has a registered account And the Orb user logs in @@ -58,6 +64,7 @@ Feature: agent provider And the agent status in Orb should be online + @smoke Scenario: Save agent without tag Given the Orb user has a registered account And the Orb user logs in @@ -69,6 +76,7 @@ Feature: agent provider And the agent status in Orb should be online + @smoke Scenario: Insert tags in agents created without tags Given the Orb user has a registered account And the Orb user logs in @@ -80,6 +88,7 @@ Feature: agent provider And the agent status in Orb should be online + @smoke Scenario: Edit agent name Given the Orb user has a registered account And the Orb user logs in @@ -91,6 +100,7 @@ Feature: agent provider And the agent status in Orb should be online + @smoke Scenario: Edit agent name and tags Given the Orb user has a registered account And the Orb user logs in diff --git a/python-test/features/datasets.feature b/python-test/features/datasets.feature index ac4abf606..54d71d43e 100644 --- a/python-test/features/datasets.feature +++ b/python-test/features/datasets.feature @@ -1,13 +1,14 @@ @datasets Feature: datasets creation + @smoke Scenario: Create Dataset Given the Orb user has a registered account And the Orb user logs in And that an agent with 1 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - And that a policy already exists + And that a policy using: handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=0 already exists When a new dataset is created using referred group, sink and policy ID Then the container logs should contain the message "managing agent policy from core" within 10 seconds And the container logs should contain the message "policy applied successfully" within 10 seconds diff --git a/python-test/features/environment.py b/python-test/features/environment.py index a5f59bc56..3e474e4d2 100644 --- a/python-test/features/environment.py +++ b/python-test/features/environment.py @@ -4,6 +4,11 @@ def before_scenario(context, scenario): cleanup_container() + context.containers_id = dict() + context.agent_groups = dict() + + +def after_scenario(context, scenario): context.execute_steps(''' Given the Orb user logs in Then cleanup agents @@ -12,8 +17,6 @@ def before_scenario(context, scenario): Then cleanup policies Then cleanup datasets ''') - context.containers_id = dict() - context.agent_groups = dict() def after_feature(context, feature): diff --git a/python-test/features/integration.feature b/python-test/features/integration.feature index 1cf2d22c3..b196143a1 100644 --- a/python-test/features/integration.feature +++ b/python-test/features/integration.feature @@ -1,98 +1,118 @@ @integration Feature: Integration tests -Scenario: Apply two policies to an agent + +@smoke +Scenario: Apply multiple advanced policies to an agent + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with 1 orb tag(s) already exists and is online + And referred agent is subscribed to a group + And that a sink already exists + When 14 mixed policies are applied to the group + Then this agent's heartbeat shows that 14 policies are successfully applied and has status running + And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds + And datasets related to all existing policies have validity valid + + +@smoke +Scenario: Apply two simple policies to an agent Given the Orb user has a registered account And the Orb user logs in And that an agent with 1 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - When 2 policies are applied to the group - Then this agent's heartbeat shows that 2 policies are successfully applied + When 2 simple policies are applied to the group + Then this agent's heartbeat shows that 2 policies are successfully applied and has status running And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds And referred sink must have active state on response within 10 seconds And datasets related to all existing policies have validity valid +@smoke Scenario: apply one policy using multiple datasets to the same group Given the Orb user has a registered account And the Orb user logs in And that an agent with 2 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - When 2 policies are applied to the group by 3 datasets each - Then this agent's heartbeat shows that 2 policies are successfully applied + When 2 simple policies are applied to the group by 3 datasets each + Then this agent's heartbeat shows that 2 policies are successfully applied and has status running And 3 datasets are linked with each policy on agent's heartbeat - And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds - And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds - And referred sink must have active state on response within 10 seconds + And the container logs contain the message "policy applied successfully" referred to each policy within 180 seconds + And referred sink must have active state on response within 180 seconds And datasets related to all existing policies have validity valid +@smoke Scenario: Remove group to which agent is linked Given the Orb user has a registered account And the Orb user logs in And that an agent with 1 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - And 1 policies are applied to the group - And this agent's heartbeat shows that 1 policies are successfully applied + And 1 simple policies are applied to the group + And this agent's heartbeat shows that 1 policies are successfully applied and has status running When the group to which the agent is linked is removed Then the container logs should contain the message "completed RPC unsubscription to group" within 10 seconds And dataset related have validity invalid - Scenario: Remove policy from agent +@smoke +Scenario: Remove policy from agent Given the Orb user has a registered account And the Orb user logs in And that an agent with 3 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - And 2 policies are applied to the group - And this agent's heartbeat shows that 2 policies are successfully applied + And 2 simple policies are applied to the group + And this agent's heartbeat shows that 2 policies are successfully applied and has status running When one of applied policies is removed Then referred policy must not be listed on the orb policies list And datasets related to removed policy has validity invalid And datasets related to all existing policies have validity valid - And this agent's heartbeat shows that 1 policies are successfully applied + And this agent's heartbeat shows that 1 policies are successfully applied and has status running And container logs should inform that removed policy was stopped and removed within 10 seconds And the container logs that were output after the policy have been removed contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds And the container logs that were output after the policy have been removed does not contain the message "scraped metrics for policy" referred to deleted policy anymore +@smoke Scenario: Remove dataset from agent with just one dataset linked Given the Orb user has a registered account And the Orb user logs in And that an agent with 3 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - And 1 policies are applied to the group - And this agent's heartbeat shows that 1 policies are successfully applied + And 1 simple policies are applied to the group + And this agent's heartbeat shows that 1 policies are successfully applied and has status running When a dataset linked to this agent is removed Then referred dataset must not be listed on the orb datasets list - And this agent's heartbeat shows that 0 policies are successfully applied + And this agent's heartbeat shows that 0 policies are successfully applied and has status running And container logs should inform that removed policy was stopped and removed within 10 seconds And the container logs that were output after removing dataset contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds And the container logs that were output after removing dataset does not contain the message "scraped metrics for policy" referred to deleted policy anymore +@smoke Scenario: Remove dataset from agent with more than one dataset linked Given the Orb user has a registered account And the Orb user logs in And that an agent with 4 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - And 3 policies are applied to the group - And this agent's heartbeat shows that 3 policies are successfully applied + And 3 simple policies are applied to the group + And this agent's heartbeat shows that 3 policies are successfully applied and has status running When a dataset linked to this agent is removed Then referred dataset must not be listed on the orb datasets list - And this agent's heartbeat shows that 2 policies are successfully applied + And this agent's heartbeat shows that 2 policies are successfully applied and has status running And container logs should inform that removed policy was stopped and removed within 10 seconds And the container logs that were output after removing dataset contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds And the container logs that were output after removing dataset does not contain the message "scraped metrics for policy" referred to deleted policy anymore +@smoke Scenario: Provision agent with tags matching an existent group Given the Orb user has a registered account And the Orb user logs in @@ -103,28 +123,30 @@ Scenario: Provision agent with tags matching an existent group And the container logs should contain the message "completed RPC subscription to group" within 10 seconds +@smoke Scenario: Provision agent with tag matching existing group linked to a valid dataset Given the Orb user has a registered account And the Orb user logs in And an Agent Group is created with 3 orb tag(s) And that a sink already exists - And 2 policies are applied to the group + And 2 simple policies are applied to the group When a new agent is created with tags matching an existing group And the agent container is started on port default - Then this agent's heartbeat shows that 2 policies are successfully applied + Then this agent's heartbeat shows that 2 policies are successfully applied and has status running And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds And referred sink must have active state on response within 10 seconds And datasets related to all existing policies have validity valid +@smoke Scenario: Sink with invalid endpoint Given the Orb user has a registered account And the Orb user logs in And that an agent with 1 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink with invalid endpoint already exists - And that a policy already exists + And that a policy using: handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=2 already exists When a new dataset is created using referred group, sink and policy ID Then the container logs should contain the message "managing agent policy from core" within 10 seconds And the container logs should contain the message "policy applied successfully" within 10 seconds @@ -133,13 +155,14 @@ Scenario: Sink with invalid endpoint And dataset related have validity valid +@smoke Scenario: Sink with invalid username Given the Orb user has a registered account And the Orb user logs in And that an agent with 1 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink with invalid username already exists - And that a policy already exists + And that a policy using: handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=3 already exists When a new dataset is created using referred group, sink and policy ID Then the container logs should contain the message "managing agent policy from core" within 10 seconds And the container logs should contain the message "policy applied successfully" within 10 seconds @@ -148,13 +171,15 @@ Scenario: Sink with invalid username And dataset related have validity valid +#@smoke +@fail Scenario: Sink with invalid password Given the Orb user has a registered account And the Orb user logs in And that an agent with 1 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink with invalid password already exists - And that a policy already exists + And that a policy using: handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=5 already exists When a new dataset is created using referred group, sink and policy ID Then the container logs should contain the message "managing agent policy from core" within 10 seconds And the container logs should contain the message "policy applied successfully" within 10 seconds @@ -163,6 +188,7 @@ Scenario: Sink with invalid password And dataset related have validity valid +@smoke Scenario: Agent subscription to multiple groups created after provisioning agent Given the Orb user has a registered account And the Orb user logs in @@ -174,6 +200,7 @@ Scenario: Agent subscription to multiple groups created after provisioning agent Then the container logs contain the message "completed RPC subscription to group" referred to each matching group within 10 seconds +@smoke Scenario: Agent subscription to multiple groups created before provisioning agent Given the Orb user has a registered account And the Orb user logs in @@ -185,6 +212,7 @@ Scenario: Agent subscription to multiple groups created before provisioning agen Then the container logs contain the message "completed RPC subscription to group" referred to each matching group within 10 seconds +@smoke Scenario: Agent subscription to group after editing agent's tags (agent provisioned before editing and group created after) Given the Orb user has a registered account And the Orb user logs in @@ -197,6 +225,7 @@ Scenario: Agent subscription to group after editing agent's tags (agent provisio And this agent's heartbeat shows that 1 groups are matching the agent +@smoke Scenario: Agent subscription to group after editing agent's tags (editing tags after agent provision) Given the Orb user has a registered account And the Orb user logs in @@ -209,6 +238,7 @@ Scenario: Agent subscription to group after editing agent's tags (editing tags a And this agent's heartbeat shows that 1 groups are matching the agent +@smoke Scenario: Agent subscription to group after editing agent's tags (editing tags before agent provision) Given the Orb user has a registered account And the Orb user logs in @@ -220,42 +250,45 @@ Scenario: Agent subscription to group after editing agent's tags (editing tags b And this agent's heartbeat shows that 1 groups are matching the agent +@smoke Scenario: Agent subscription to multiple group with policies after editing agent's tags (editing tags after agent provision) Given the Orb user has a registered account And the Orb user logs in And that an agent with test:true orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - And 2 policies are applied to the group - And this agent's heartbeat shows that 2 policies are successfully applied + And 2 simple policies are applied to the group + And this agent's heartbeat shows that 2 policies are successfully applied and has status running And an Agent Group is created with region:br orb tag(s) - And 1 policies are applied to the group + And 1 simple policies are applied to the group When edit the agent tags and use region:br, test:true orb tag(s) Then the container logs contain the message "completed RPC subscription to group" referred to each matching group within 10 seconds - And this agent's heartbeat shows that 3 policies are successfully applied + And this agent's heartbeat shows that 3 policies are successfully applied and has status running And this agent's heartbeat shows that 2 groups are matching the agent And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds +@smoke Scenario: Agent subscription to group with policies after editing agent's tags (editing tags after agent provision) Given the Orb user has a registered account And the Orb user logs in And that an agent with 1 orb tag(s) already exists and is online And referred agent is subscribed to a group And that a sink already exists - And 2 policies are applied to the group - And this agent's heartbeat shows that 2 policies are successfully applied + And 2 simple policies are applied to the group + And this agent's heartbeat shows that 2 policies are successfully applied and has status running And an Agent Group is created with region:br orb tag(s) - And 1 policies are applied to the group + And 1 simple policies are applied to the group When edit the agent tags and use region:br orb tag(s) Then the container logs contain the message "completed RPC subscription to group" referred to each matching group within 10 seconds - And this agent's heartbeat shows that 1 policies are successfully applied + And this agent's heartbeat shows that 1 policies are successfully applied and has status running And this agent's heartbeat shows that 1 groups are matching the agent And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds +@smoke Scenario: Insert tags in agents created without tags and apply policies to group matching new tags Given the Orb user has a registered account And the Orb user logs in @@ -263,15 +296,16 @@ Scenario: Insert tags in agents created without tags and apply policies to group And the agent container is started on port default And that a sink already exists When edit the agent tags and use 2 orb tag(s) - And an Agent Group is created with same tag as the agent - And 1 policies are applied to the group - Then this agent's heartbeat shows that 1 policies are successfully applied + And an Agent Group is created with same tag as the agent and without description + And 1 simple policies are applied to the group + Then this agent's heartbeat shows that 1 policies are successfully applied and has status running And the container logs contain the message "completed RPC subscription to group" referred to each matching group within 10 seconds And this agent's heartbeat shows that 1 groups are matching the agent And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds And the container logs that were output after all policies have been applied contain the message "scraped metrics for policy" referred to each applied policy within 180 seconds +@smoke Scenario: Edit agent name and apply policies to then Given the Orb user has a registered account And the Orb user logs in @@ -279,35 +313,203 @@ Scenario: Edit agent name and apply policies to then And an Agent Group is created with same tag as the agent And 1 agent must be matching on response field matching_agents And that a sink already exists - And 1 policies are applied to the group + And 1 simple policies are applied to the group When edit the agent name and edit agent tags using 3 orb tag(s) - Then this agent's heartbeat shows that 1 policies are successfully applied + Then this agent's heartbeat shows that 1 policies are successfully applied and has status running And the container logs contain the message "policy applied successfully" referred to each policy within 10 seconds -Scenario: Editing tags of an Agent Group with policies (unsubscription) +@smoke +Scenario: Editing tags of an Agent Group with policies (unsubscription - provision agent before editing) Given the Orb user has a registered account And the Orb user logs in And that an agent with region:br orb tag(s) already exists and is online And an Agent Group is created with same tag as the agent and without description And that a sink already exists - And 2 policies are applied to the group + And 2 simple policies are applied to the group When the name, tags, description of Agent Group is edited using: name=new_name/ tags=another:tag, ns1:true/ description=None Then 0 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC unsubscription to group" within 10 seconds And the agent status in Orb should be online -Scenario: Editing tags of an Agent Group with policies (subscription) +@smoke +Scenario: Editing tags of an Agent Group with policies (subscription - provision agent before editing) Given the Orb user has a registered account And the Orb user logs in And that an agent with region:br, another:tag orb tag(s) already exists and is online And an Agent Group is created with ns1:true orb tag(s) and without description And that a sink already exists - And 2 policies are applied to the group + And 2 simple policies are applied to the group + When the name, tags, description of Agent Group is edited using: name=new_name/ tags=region:br/ description=None + Then 1 agent must be matching on response field matching_agents + And the container logs should contain the message "completed RPC subscription to group" within 10 seconds + And the agent status in Orb should be online + And this agent's heartbeat shows that 1 groups are matching the agent + And this agent's heartbeat shows that 2 policies are successfully applied and has status running + + +@smoke +Scenario: Editing tags of an Agent Group with policies (provision agent after editing) + Given the Orb user has a registered account + And the Orb user logs in + And an Agent Group is created with ns1:true orb tag(s) and without description + And that a sink already exists + And 2 simple policies are applied to the group + When the name, tags, description of Agent Group is edited using: name=new_name/ tags=another:tag, ns1:true/ description=None + And a new agent is created with region:us orb tag(s) + And the agent container is started on port default + Then 0 agent must be matching on response field matching_agents + And the agent status in Orb should be online + + +@smoke +Scenario: Editing tags of an Agent Group with policies (subscription - provision agent after editing) + Given the Orb user has a registered account + And the Orb user logs in + And an Agent Group is created with ns1:true orb tag(s) and without description + And that a sink already exists + And 2 simple policies are applied to the group When the name, tags, description of Agent Group is edited using: name=new_name/ tags=region:br/ description=None + And a new agent is created with region:br orb tag(s) + And the agent container is started on port default Then 1 agent must be matching on response field matching_agents And the container logs should contain the message "completed RPC subscription to group" within 10 seconds And the agent status in Orb should be online And this agent's heartbeat shows that 1 groups are matching the agent - And this agent's heartbeat shows that 2 policies are successfully applied + And this agent's heartbeat shows that 2 policies are successfully applied and has status running + + +@smoke +Scenario: Editing tags of an Agent and Agent Group with policies (unsubscription - provision agent before editing) + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with region:br orb tag(s) already exists and is online + And an Agent Group is created with same tag as the agent and without description + And that a sink already exists + And 2 simple policies are applied to the group + When the name, tags, description of Agent Group is edited using: name=new_name/ tags=another:tag, ns1:true/ description=None + And edit the agent tags and use region:us orb tag(s) + Then 0 agent must be matching on response field matching_agents + And the container logs should contain the message "completed RPC unsubscription to group" within 10 seconds + And the agent status in Orb should be online + + +@smoke +Scenario: Editing tags of an Agent and Agent Group with policies (subscription - provision agent before editing) + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with region:br, another:tag, test:true orb tag(s) already exists and is online + And an Agent Group is created with ns1:true orb tag(s) and without description + And that a sink already exists + And 2 simple policies are applied to the group + When edit the agent tags and use region:br, another:tag orb tag(s) + And the name, tags, description of Agent Group is edited using: name=new_name/ tags=region:br/ description=None + Then 1 agent must be matching on response field matching_agents + And the container logs should contain the message "completed RPC subscription to group" within 10 seconds + And the agent status in Orb should be online + And this agent's heartbeat shows that 1 groups are matching the agent + And this agent's heartbeat shows that 2 policies are successfully applied and has status running + + +@smoke +Scenario: Editing tags of an Agent and Agent Group with policies (provision agent after editing) + Given the Orb user has a registered account + And the Orb user logs in + And an Agent Group is created with ns1:true orb tag(s) and without description + And that a sink already exists + And 2 simple policies are applied to the group + When the name, tags, description of Agent Group is edited using: name=new_name/ tags=another:tag, ns1:true/ description=None + And a new agent is created with test:true orb tag(s) + And edit the agent tags and use region:us orb tag(s) + And the agent container is started on port default + Then 0 agent must be matching on response field matching_agents + And the agent status in Orb should be online + + +@smoke +Scenario: Editing tags of an Agent and Agent Group with policies (subscription - provision agent after editing) + Given the Orb user has a registered account + And the Orb user logs in + And an Agent Group is created with ns1:true orb tag(s) and without description + And that a sink already exists + And 2 simple policies are applied to the group + When the name, tags, description of Agent Group is edited using: name=new_name/ tags=region:br/ description=None + And a new agent is created with test:true orb tag(s) + And edit the agent tags and use region:br orb tag(s) + And the agent container is started on port default + Then 1 agent must be matching on response field matching_agents + And the container logs should contain the message "completed RPC subscription to group" within 10 seconds + And the agent status in Orb should be online + And this agent's heartbeat shows that 1 groups are matching the agent + And this agent's heartbeat shows that 2 policies are successfully applied and has status running + + + +@smoke +Scenario: Edit an advanced policy with handler dns changing the handler to net + Given the Orb user has a registered account + And the Orb user logs in + And that a sink already exists + And that an agent with 1 orb tag(s) already exists and is online + And an Agent Group is created with same tag as the agent + And a new policy is created using: handler=dns, description='policy_dns', bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=0 + And a new dataset is created using referred group, sink and policy ID + When editing a policy using name=my_policy, handler=net, only_qname_suffix=None, only_rcode=None + Then policy version must be 1 + And policy name must be my_policy + And policy handler must be net + And policy only_qname_suffix must be None + And policy only_rcode must be None + And this agent's heartbeat shows that 1 policies are successfully applied and has status running + + + +@smoke +Scenario: Edit an advanced policy with handler dns changing the handler to dhcp + Given the Orb user has a registered account + And the Orb user logs in + And that a sink already exists + And that an agent with 1 orb tag(s) already exists and is online + And an Agent Group is created with same tag as the agent + And a new policy is created using: handler=dns, host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=2 + And a new dataset is created using referred group, sink and policy ID + When editing a policy using name=second_policy, handler=dhcp, only_qname_suffix=None, only_rcode=None + Then policy version must be 1 + And policy name must be second_policy + And policy handler must be dhcp + And this agent's heartbeat shows that 1 policies are successfully applied and has status running + + +@smoke +Scenario: Edit a simple policy with handler dhcp changing the handler to net + Given the Orb user has a registered account + And the Orb user logs in + And that a sink already exists + And that an agent with 1 orb tag(s) already exists and is online + And an Agent Group is created with same tag as the agent + And a new policy is created using: handler=dhcp + And a new dataset is created using referred group, sink and policy ID + When editing a policy using handler=net, description="policy_net" + Then policy version must be 1 + And policy handler must be net + And this agent's heartbeat shows that 1 policies are successfully applied and has status running + + +@smoke +Scenario: Edit a simple policy with handler net changing the handler to dns and inserting advanced parameters + Given the Orb user has a registered account + And the Orb user logs in + And that a sink already exists + And that an agent with 1 orb tag(s) already exists and is online + And an Agent Group is created with same tag as the agent + And a new policy is created using: handler=net + And a new dataset is created using referred group, sink and policy ID + When editing a policy using handler=dns, host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=2 + Then policy version must be 1 + And policy handler must be dns + And policy host_specification must be 10.0.1.0/24,10.0.2.1/32,2001:db8::/64 + And policy bpf_filter_expression must be udp port 53 + And policy only_qname_suffix must be ['.foo.com', '.example.com'] + And policy only_rcode must be 2 + And this agent's heartbeat shows that 1 policies are successfully applied and has status running diff --git a/python-test/features/login.feature b/python-test/features/login.feature index e7cff47f0..80f8af9f2 100644 --- a/python-test/features/login.feature +++ b/python-test/features/login.feature @@ -1,6 +1,7 @@ @login Feature: login tests + @smoke Scenario Outline: Request registration of a registered email using password Given there is a registered account When request referred account registration using registered email, password, user name and company name @@ -16,6 +17,7 @@ | unregistered | None | NS1 | Then account register should not be changed + @smoke Scenario Outline: Login with invalid credentials Given there is a registered account When the Orb user request an authentication token using email and password @@ -26,6 +28,7 @@ | correct | incorrect | Then user should not be able to authenticate + @smoke Scenario Outline: Check if email is a required field When user request account registration email, password, user name and company name Examples: @@ -40,7 +43,7 @@ | without | without | with | without | Then user should not be able to authenticate - + @smoke Scenario Outline: Check if password is a required field When user request account registration email, password, user name and company name Examples: @@ -54,3 +57,28 @@ | without | without | with | with | | without | without | with | without | Then user should not be able to authenticate + + + @smoke + Scenario Outline: Request account registration of an unregistered account with invalid values + Given that there is an unregistered email with password + When the Orb user request this account registration with as company and as fullname + Then the status code must be + Examples: + | email | password | company | fullname | status_code | + | valid | 1234 | "email used for testing process" | "Test process" | 400 | + | invalid | 12345678 | "email used for testing process" | "Test process" | 400 | + | invalid | 1234567 | "email used for testing process" | "Test process" | 400 | + + + @smoke + Scenario Outline: Request account registration of an unregistered account with company and full name + Given that there is an unregistered email with password + When the Orb user request this account registration with as company and as fullname + Then the status code must be + And account is registered with email, with password, company and full name + Examples: + | email | password | company | fullname | status_code | + | valid | 12345678 | email used for testing process | Test process | 201 | + | valid | 12345678 | None | Test process | 201 | + | valid | 12345678 | email used for testing process | None | 201 | diff --git a/python-test/features/policies.feature b/python-test/features/policies.feature index 11dc97e7f..f042a1d5c 100644 --- a/python-test/features/policies.feature +++ b/python-test/features/policies.feature @@ -1,9 +1,64 @@ @policies Feature: policy creation - Scenario: Create a policy + @smoke + Scenario: Create a policy with dns handler, description, host specification, bpf filter, pcap source, only qname suffix and only rcode Given the Orb user has a registered account And the Orb user logs in And that an agent with 1 orb tag(s) already exists and is online - When a new policy is created + When a new policy is created using: handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=0 + Then referred policy must be listed on the orb policies list + + + @smoke + Scenario: Create a policy with dns handler, host specification, bpf filter, pcap source, only qname suffix and only rcode + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with 1 orb tag(s) already exists and is online + When a new policy is created using: handler=dns, host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.foo.com/ .example.com], only_rcode=2 + Then referred policy must be listed on the orb policies list + + + @smoke + Scenario: Create a policy with dns handler, bpf filter, pcap source, only qname suffix and only rcode + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with 1 orb tag(s) already exists and is online + When a new policy is created using: handler=dns, bpf_filter_expression=udp port 53, pcap_source=af_packet, only_qname_suffix=[.foo.com/ .example.com], only_rcode=3 + Then referred policy must be listed on the orb policies list + + + @smoke + Scenario: Create a policy with dns handler, pcap source, only qname suffix and only rcode + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with 1 orb tag(s) already exists and is online + When a new policy is created using: handler=dns, pcap_source=af_packet, only_qname_suffix=[.foo.com/ .example.com], only_rcode=5 + Then referred policy must be listed on the orb policies list + + + @smoke + Scenario: Create a policy with dns handler, only qname suffix + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with 1 orb tag(s) already exists and is online + When a new policy is created using: handler=dns, only_qname_suffix=[.foo.com/ .example.com] + Then referred policy must be listed on the orb policies list + + + @smoke + Scenario: Create a policy with dhcp handler, description, host specification, bpf filter and pcap source + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with 1 orb tag(s) already exists and is online + When a new policy is created using: handler=dhcp, description='policy_dhcp', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap + Then referred policy must be listed on the orb policies list + + + @smoke + Scenario: Create a policy with net handler, description, host specification, bpf filter and pcap source + Given the Orb user has a registered account + And the Orb user logs in + And that an agent with 1 orb tag(s) already exists and is online + When a new policy is created using: handler=net, description='policy_net', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap Then referred policy must be listed on the orb policies list \ No newline at end of file diff --git a/python-test/features/sinks.feature b/python-test/features/sinks.feature index 05106d453..21126e985 100644 --- a/python-test/features/sinks.feature +++ b/python-test/features/sinks.feature @@ -1,10 +1,11 @@ @sinks Feature: sink creation + @smoke Scenario: Create Sink using Prometheus Given that the user has the prometheus/grafana credentials And the Orb user has a registered account And the Orb user logs in When a new sink is created - Then referred sink must have unknown state on response within 10 seconds + Then referred sink must have new state on response within 10 seconds diff --git a/python-test/features/steps/control_agents_ui.py b/python-test/features/steps/control_agents_ui.py index 38e935ec9..fb8d8da09 100644 --- a/python-test/features/steps/control_agents_ui.py +++ b/python-test/features/steps/control_agents_ui.py @@ -109,4 +109,5 @@ def check_status_on_orb_ui(context, status, time_to_wait): agent_view_status = WebDriverWait(context.driver, 3).until( EC.presence_of_all_elements_located( (By.XPATH, AgentsPage.agent_status())))[0].text + agent_view_status = agent_view_status.replace(".", "") assert_that(agent_view_status, equal_to(status), f"Agent {context.agent['id']} status failed") diff --git a/python-test/features/steps/control_plane_agent_groups.py b/python-test/features/steps/control_plane_agent_groups.py index 0e264c6bd..955f0b071 100644 --- a/python-test/features/steps/control_plane_agent_groups.py +++ b/python-test/features/steps/control_plane_agent_groups.py @@ -155,15 +155,7 @@ def subscribe_agent_to_a_group(context): @step('the container logs contain the message "{text_to_match}" referred to each matching group within' '{time_to_wait} seconds') def check_logs_for_group(context, text_to_match, time_to_wait): - groups_matching = list() - context.groups_matching_id = list() - for group in context.agent_groups.keys(): - group_data = get_agent_group(context.token, group) - group_tags = dict(group_data["tags"]) - agent_tags = context.agent["orb_tags"] - if all(item in agent_tags.items() for item in group_tags.items()) is True: - groups_matching.append(context.agent_groups[group]) - context.groups_matching_id.append(group) + groups_matching, context.groups_matching_id = return_matching_groups(context.token, context.agent_groups, context.agent) text_found, groups_to_which_subscribed = check_subscription(time_to_wait, groups_matching, text_to_match, context.container_id) assert_that(text_found, is_(True), f"Message {text_to_match} was not found in the agent logs for group(s)" @@ -310,3 +302,24 @@ def edit_agent_group(token, agent_group_id, name, description, tags, expected_st 'Request to edit agent group failed with status=' + str(group_edited_response.status_code)) return group_edited_response.json() + + +def return_matching_groups(token, existing_agent_groups, agent_json): + """ + + :param (str) token: used for API authentication + :param (dict) existing_agent_groups: dictionary with the existing groups, the id of the groups being the key and the name the values + :param (dict) agent_json: dictionary containing all the information of the agent to which the groups must be matching + + :return (list): groups_matching, groups_matching_id + """ + groups_matching = list() + groups_matching_id = list() + for group in existing_agent_groups.keys(): + group_data = get_agent_group(token, group) + group_tags = dict(group_data["tags"]) + agent_tags = agent_json["orb_tags"] + if all(item in agent_tags.items() for item in group_tags.items()) is True: + groups_matching.append(existing_agent_groups[group]) + groups_matching_id.append(group) + return groups_matching, groups_matching_id \ No newline at end of file diff --git a/python-test/features/steps/control_plane_agents.py b/python-test/features/steps/control_plane_agents.py index 0ed4b4cb2..4c60c07c5 100644 --- a/python-test/features/steps/control_plane_agents.py +++ b/python-test/features/steps/control_plane_agents.py @@ -1,6 +1,7 @@ from test_config import TestConfig from utils import random_string, filter_list_by_parameter_start_with, generate_random_string_with_predefined_prefix, create_tags_set from local_agent import run_local_agent_container +from control_plane_agent_groups import return_matching_groups from behave import given, when, then, step from hamcrest import * import time @@ -69,8 +70,8 @@ def multiple_dataset_for_policy(context, amount_of_datasets): f"Amount of datasets linked with policy {policy_id} failed") -@step("this agent's heartbeat shows that {amount_of_policies} policies are successfully applied") -def list_policies_applied_to_an_agent(context, amount_of_policies): +@step("this agent's heartbeat shows that {amount_of_policies} policies are successfully applied and has status {policies_status}") +def list_policies_applied_to_an_agent(context, amount_of_policies, policies_status): time_waiting = 0 sleep_time = 0.5 timeout = 30 @@ -86,11 +87,9 @@ def list_policies_applied_to_an_agent(context, amount_of_policies): assert_that(len(context.list_agent_policies_id), equal_to(int(amount_of_policies)), f"Amount of policies applied to this agent failed with {context.list_agent_policies_id} policies") - assert_that(sorted(context.list_agent_policies_id), equal_to(sorted(context.policies_created.keys())), - "Policies linked with the agent is not the same as the created by test process") for policy_id in context.list_agent_policies_id: - assert_that(agent['last_hb_data']['policy_state'][policy_id]["state"], equal_to('running'), - f"policy {policy_id} is not running") + assert_that(agent['last_hb_data']['policy_state'][policy_id]["state"], equal_to(policies_status), + f"policy {policy_id} is not {policies_status}") @step("this agent's heartbeat shows that {amount_of_groups} groups are matching the agent") @@ -99,6 +98,7 @@ def list_groups_matching_an_agent(context, amount_of_groups): sleep_time = 0.5 timeout = 30 context.list_groups_id = list() + groups_matching, context.groups_matching_id = return_matching_groups(context.token, context.agent_groups, context.agent) while time_waiting < timeout: agent = get_agent(context.token, context.agent['id']) if 'group_state' in agent['last_hb_data'].keys(): diff --git a/python-test/features/steps/control_plane_datasets.py b/python-test/features/steps/control_plane_datasets.py index d6d3006f8..f038a47ce 100644 --- a/python-test/features/steps/control_plane_datasets.py +++ b/python-test/features/steps/control_plane_datasets.py @@ -1,4 +1,4 @@ -from behave import given, when, then, step +from behave import given, then, step from utils import random_string, filter_list_by_parameter_start_with from hamcrest import * import requests @@ -11,7 +11,7 @@ base_orb_url = TestConfig.configs().get('base_orb_url') -@when("a new dataset is created using referred group, sink and policy ID") +@step("a new dataset is created using referred group, sink and policy ID") def create_new_dataset(context): context.considered_timestamp = datetime.now().timestamp() token = context.token diff --git a/python-test/features/steps/control_plane_policies.py b/python-test/features/steps/control_plane_policies.py index bc2e488fb..5d1fc363d 100644 --- a/python-test/features/steps/control_plane_policies.py +++ b/python-test/features/steps/control_plane_policies.py @@ -1,25 +1,59 @@ from hamcrest import * import requests -from behave import given, when, then, step -from utils import random_string, filter_list_by_parameter_start_with, safe_load_json +from behave import given, then, step +from utils import random_string, filter_list_by_parameter_start_with, safe_load_json, remove_empty_from_json from local_agent import get_orb_agent_logs from test_config import TestConfig import time from datetime import datetime from control_plane_datasets import create_new_dataset, list_datasets -from random import choice +from random import choice, choices, sample policy_name_prefix = "test_policy_name_" -default_handler = "net" -handle_label = "default_" + default_handler base_orb_url = TestConfig.configs().get('base_orb_url') -@when("a new policy is created") -def create_new_policy(context): - policy_name = policy_name_prefix + random_string(10) - context.policy = create_policy(context.token, policy_name, handle_label, default_handler) - assert_that(context.policy['name'], equal_to(policy_name)) +@step("a new policy is created using: {kwargs}") +def create_new_policy(context, kwargs): + acceptable_keys = ['name', 'handler_label', 'handler', 'description', 'tap', 'input_type', + 'host_specification', 'bpf_filter_expression', 'pcap_source', 'only_qname_suffix', + 'only_rcode', 'backend_type'] + name = policy_name_prefix + random_string(10) + + kwargs_dict = {'name': name, 'handler': None, 'description': None, 'tap': "default_pcap", + 'input_type': "pcap", 'host_specification': None, 'bpf_filter_expression': None, + 'pcap_source': None, 'only_qname_suffix': None, 'only_rcode': None, 'backend_type': "pktvisor"} + + for i in kwargs.split(", "): + assert_that(i, matches_regexp("^.+=.+$"), f"Unexpected format for param {i}") + item = i.split("=") + kwargs_dict[item[0]] = item[1] + + assert_that(all(key in acceptable_keys for key, value in kwargs_dict.items()), equal_to(True), + f"Unexpected parameters for policy. Options are {acceptable_keys}") + + if kwargs_dict["only_qname_suffix"] is not None: + kwargs_dict["only_qname_suffix"] = kwargs_dict["only_qname_suffix"].replace("[", "") + kwargs_dict["only_qname_suffix"] = kwargs_dict["only_qname_suffix"].replace("]", "") + kwargs_dict["only_qname_suffix"] = kwargs_dict["only_qname_suffix"].split("/ ") + + if policy_name_prefix not in kwargs_dict["name"]: + kwargs_dict["name"] + policy_name_prefix + kwargs_dict["name"] + + assert_that(kwargs_dict["handler"], any_of(equal_to("dns"), equal_to("dhcp"), equal_to("net")), + "Unexpected handler for policy") + handle_label = f"default_{kwargs_dict['handler']}_{random_string(3)}" + + policy_json = make_policy_json(kwargs_dict["name"], handle_label, + kwargs_dict["handler"], kwargs_dict["description"], kwargs_dict["tap"], + kwargs_dict["input_type"], kwargs_dict["host_specification"], + kwargs_dict["bpf_filter_expression"], kwargs_dict["pcap_source"], + kwargs_dict["only_qname_suffix"], kwargs_dict["only_rcode"], + kwargs_dict["backend_type"]) + + context.policy = create_policy(context.token, policy_json) + + assert_that(context.policy['name'], equal_to(name)) if 'policies_created' in context: context.policies_created[context.policy['id']] = context.policy['name'] else: @@ -27,6 +61,93 @@ def create_new_policy(context): context.policies_created[context.policy['id']] = context.policy['name'] +@step("editing a policy using {kwargs}") +def policy_editing(context, kwargs): + acceptable_keys = ['name', 'handler_label', 'handler', 'description', 'tap', 'input_type', + 'host_specification', 'bpf_filter_expression', 'pcap_source', 'only_qname_suffix', + 'only_rcode', 'backend_type'] + + handler_label = list(context.policy["policy"]["handlers"]["modules"].keys())[0] + + edited_attributes = { + 'host_specification': return_policy_attribute(context.policy, 'host_specification'), + 'bpf_filter_expression': return_policy_attribute(context.policy, 'bpf_filter_expression'), + 'pcap_source': return_policy_attribute(context.policy, 'pcap_source'), + 'only_qname_suffix': return_policy_attribute(context.policy, 'only_qname_suffix'), + 'only_rcode': return_policy_attribute(context.policy, 'only_rcode'), + 'description': return_policy_attribute(context.policy, 'description'), + "name": return_policy_attribute(context.policy, 'name'), + "handler": return_policy_attribute(context.policy, 'handler'), + "backend_type": return_policy_attribute(context.policy, 'backend'), + "tap": return_policy_attribute(context.policy, 'tap'), + "input_type": return_policy_attribute(context.policy, 'input_type'), + "handler_label": return_policy_attribute(context.policy, 'handler_label')} + + if "host_spec" in context.policy["policy"]["input"]["config"].keys(): + edited_attributes["host_specification"] = context.policy["policy"]["input"]["config"]["host_spec"] + if "pcap_source" in context.policy["policy"]["input"]["config"].keys(): + edited_attributes["pcap_source"] = context.policy["policy"]["input"]["config"]["pcap_source"] + if "bpf" in context.policy["policy"]["input"]["filter"].keys(): + edited_attributes["bpf_filter_expression"] = context.policy["policy"]["input"]["filter"]["bpf"] + if "description" in context.policy.keys(): + edited_attributes["description"] = context.policy['description'] + if "only_qname_suffix" in context.policy["policy"]["handlers"]["modules"][handler_label]['filter'].keys(): + edited_attributes["only_qname_suffix"] = context.policy["policy"]["handlers"]["modules"][handler_label]["filter"][ + "only_qname_suffix"] + if "only_rcode" in context.policy["policy"]["handlers"]["modules"][handler_label]['filter'].keys(): + edited_attributes["only_rcode"] = context.policy["policy"]["handlers"]["modules"][handler_label]["filter"]["only_rcode"] + + for i in kwargs.split(", "): + assert_that(i, matches_regexp("^.+=.+$"), f"Unexpected format for param {i}") + item = i.split("=") + edited_attributes[item[0]] = item[1] + if item[1].isdigit() is False and str(item[1]).lower() == "none": + edited_attributes[item[0]] = None + if item[0] == "handler": + edited_attributes["handler_label"] = f"default_{edited_attributes['handler']}_{random_string(3)}" + + for attribute in acceptable_keys: + if attribute not in edited_attributes.keys(): + edited_attributes[attribute] = None + + assert_that(all(key in acceptable_keys for key, value in edited_attributes.items()), equal_to(True), + f"Unexpected parameters for policy. Options are {acceptable_keys}") + + if edited_attributes["only_qname_suffix"] is not None: + edited_attributes["only_qname_suffix"] = edited_attributes["only_qname_suffix"].replace("[", "") + edited_attributes["only_qname_suffix"] = edited_attributes["only_qname_suffix"].replace("]", "") + edited_attributes["only_qname_suffix"] = edited_attributes["only_qname_suffix"].split("/ ") + + if policy_name_prefix not in edited_attributes["name"]: + edited_attributes["name"] = policy_name_prefix + edited_attributes["name"] + + policy_json = make_policy_json(edited_attributes["name"], edited_attributes["handler_label"], + edited_attributes["handler"], edited_attributes["description"], + edited_attributes["tap"], + edited_attributes["input_type"], edited_attributes["host_specification"], + edited_attributes["bpf_filter_expression"], edited_attributes["pcap_source"], + edited_attributes["only_qname_suffix"], edited_attributes["only_rcode"], + edited_attributes["backend_type"]) + + context.policy = edit_policy(context.token, context.policy['id'], policy_json) + + assert_that(context.policy['name'], equal_to(edited_attributes["name"])) + + +@step("policy {attribute} must be {value}") +def check_policy_attribute(context, attribute, value): + acceptable_attributes = ['name', 'handler_label', 'handler', 'description', 'tap', 'input_type', + 'host_specification', 'bpf_filter_expression', 'pcap_source', 'only_qname_suffix', + 'only_rcode', 'backend_type', 'version'] + if attribute in acceptable_attributes: + if attribute == "name": + value = policy_name_prefix + value + policy_value = return_policy_attribute(context.policy, attribute) + assert_that(str(policy_value), equal_to(value), f"Unexpected value for policy {attribute}") + else: + raise Exception(f"Attribute {attribute} not found on policy") + + @then("referred policy {condition} be listed on the orb policies list") def check_policies(context, condition='must'): policy_id = context.policy['id'] @@ -97,9 +218,9 @@ def clean_policies(context): delete_policies(token, policies_filtered_list) -@given("that a policy already exists") -def new_policy(context): - create_new_policy(context) +@given("that a policy using: {kwargs} already exists") +def new_policy(context, kwargs): + create_new_policy(context, kwargs) check_policies(context) @@ -118,12 +239,18 @@ def check_agent_logs_for_deleted_policies_considering_timestamp(context, conditi @step('the container logs that were output after {condition} contain the message "{' 'text_to_match}" referred to each applied policy within {time_to_wait} seconds') def check_agent_logs_for_policies_considering_timestamp(context, condition, text_to_match, time_to_wait): + policies_data = list() policies_have_expected_message = \ check_agent_log_for_policies(text_to_match, time_to_wait, context.container_id, context.list_agent_policies_id, context.considered_timestamp) + if len(set(context.list_agent_policies_id).difference(policies_have_expected_message)) > 0: + policies_without_message = set(context.list_agent_policies_id).difference(policies_have_expected_message) + for policy in policies_without_message: + policies_data.append(get_policy(context.token, policy)) + assert_that(policies_have_expected_message, equal_to(set(context.list_agent_policies_id)), f"Message '{text_to_match}' for policy " - f"'{set(context.list_agent_policies_id).difference(policies_have_expected_message)}'" + f"'{policies_data}'" f" was not found in the agent logs!") @@ -138,31 +265,72 @@ def check_agent_logs_for_policies(context, text_to_match, time_to_wait): f" was not found in the agent logs!") -@step('{amount_of_policies} policies are applied to the group') -def apply_n_policies(context, amount_of_policies): +@step('{amount_of_policies} {type_of_policies} policies are applied to the group') +def apply_n_policies(context, amount_of_policies, type_of_policies): + args_for_policies = return_policies_type(int(amount_of_policies), type_of_policies) for i in range(int(amount_of_policies)): - create_new_policy(context) + create_new_policy(context, args_for_policies[i][1]) check_policies(context) create_new_dataset(context) -@step('{amount_of_policies} policies are applied to the group by {amount_of_datasets} datasets each') -def apply_n_policies_x_times(context, amount_of_policies, amount_of_datasets): +@step('{amount_of_policies} {type_of_policies} policies are applied to the group by {amount_of_datasets} datasets each') +def apply_n_policies_x_times(context, amount_of_policies, type_of_policies, amount_of_datasets): for n in range(int(amount_of_policies)): - create_new_policy(context) + args_for_policies = return_policies_type(int(amount_of_policies), type_of_policies) + create_new_policy(context, args_for_policies[n][1]) check_policies(context) for x in range(int(amount_of_datasets)): create_new_dataset(context) -def create_policy(token, policy_name, handler_label, handler, description=None, tap="default_pcap", - input_type="pcap", host_specification=None, filter_expression=None, backend_type="pktvisor"): +def create_policy(token, json_request): """ Creates a new policy in Orb control plane :param (str) token: used for API authentication - :param (str) policy_name: of the policy to be created + :param (dict) json_request: policy json + :return: response of policy creation + + """ + + headers_request = {'Content-type': 'application/json', 'Accept': '*/*', 'Authorization': token} + + response = requests.post(base_orb_url + '/api/v1/policies/agent', json=json_request, headers=headers_request) + assert_that(response.status_code, equal_to(201), + 'Request to create policy failed with status=' + str(response.status_code)) + + return response.json() + + +def edit_policy(token, policy_id, json_request): + """ + Editing a policy on Orb control plane + + :param (str) token: used for API authentication + :param (str) policy_id: that identifies the policy to be edited + :param (dict) json_request: policy json + :return: response of policy editing + """ + headers_request = {'Content-type': 'application/json', 'Accept': '*/*', 'Authorization': token} + + response = requests.put(base_orb_url + f"/api/v1/policies/agent/{policy_id}", json=json_request, + headers=headers_request) + assert_that(response.status_code, equal_to(200), + 'Request to create policy failed with status=' + str(response.status_code)) + + return response.json() + + +def make_policy_json(name, handler_label, handler, description=None, tap="default_pcap", + input_type="pcap", host_specification=None, bpf_filter_expression=None, pcap_source=None, + only_qname_suffix=None, only_rcode=None, backend_type="pktvisor"): + """ + + Generate a policy json + + :param (str) name: of the policy to be created :param (str) handler_label: of the handler :param (str) handler: to be added :param (str) description: description of policy @@ -170,21 +338,55 @@ def create_policy(token, policy_name, handler_label, handler, description=None, :param input_type: this must reference a tap name, or application of the policy will fail :param (str) host_specification: Subnets (comma separated) which should be considered belonging to this host, in CIDR form. Used for ingress/egress determination, defaults to host attached to the network interface. - :param filter_expression: these decide exactly which data to summarize and expose for collection + :param (str) bpf_filter_expression: these decide exactly which data to summarize and expose for collection. + Tcpdump compatible filter expression for limiting the traffic examined + (with BPF). See https://www.tcpdump.org/manpages/tcpdump.1.html. + :param (str) pcap_source: Packet capture engine to use. Defaults to best for platform. + Options: af_packet (linux only) or libpcap. + :param (str) only_qname_suffix: Filter out any queries whose QName does not end in a suffix on the list + :param (int) only_rcode: Filter out any queries which are not the given RCODE. Options: + "NOERROR": 0, + "NXDOMAIN": 3, + "REFUSED": 5, + "SERVFAIL": 2 :param backend_type: Agent backend this policy is for. Cannot change once created. Default: pktvisor :return: (dict) a dictionary containing the created policy data """ - json_request = {"name": policy_name, "description": description, "backend": backend_type, - "policy": {"kind": "collection", "input": {"tap": tap, "input_type": input_type}, - "handlers": {"modules": {handler_label: {"type": handler}}}}, - "config": {"host_spec": host_specification}, "filter": {"bpf": filter_expression}} - headers_request = {'Content-type': 'application/json', 'Accept': '*/*', 'Authorization': token} - - response = requests.post(base_orb_url + '/api/v1/policies/agent', json=json_request, headers=headers_request) - assert_that(response.status_code, equal_to(201), - 'Request to create policy failed with status=' + str(response.status_code)) - - return response.json() + if only_rcode is not None: only_rcode = int(only_rcode) + assert_that(pcap_source, any_of(equal_to(None), equal_to("af_packet"), equal_to("libpcap")), + "Unexpected type of pcap_source") + assert_that(only_rcode, any_of(equal_to(None), equal_to(0), equal_to(2), equal_to(3), equal_to(5)), + "Unexpected type of only_rcode") + assert_that(handler, any_of(equal_to("dns"), equal_to("dhcp"), equal_to("net")), "Unexpected handler for policy") + assert_that(name, not_none(), "Unable to create policy without name") + + json_request = {"name": name, + "description": description, + "backend": backend_type, + "policy": { + "kind": "collection", + "input": { + "tap": tap, + "input_type": input_type, + "config": { + "host_spec": host_specification, + "pcap_source": pcap_source}, + "filter": {"bpf": bpf_filter_expression}}, + "handlers": { + "modules": { + handler_label: { + "type": handler, + "filter": { + "only_qname_suffix": only_qname_suffix, + "only_rcode": only_rcode + } + } + } + } + } + } + json_request = remove_empty_from_json(json_request.copy()) + return json_request def get_policy(token, policy_id, expected_status_code=200): @@ -358,3 +560,77 @@ def list_datasets_for_a_policy(policy_id, datasets_list): if dataset['agent_policy_id'] == policy_id: id_of_related_datasets.append(dataset['id']) return id_of_related_datasets + + +def return_policies_type(k, policies_type='mixed'): + assert_that(policies_type, any_of(equal_to('mixed'), any_of('simple'), any_of('advanced')), + "Unexpected value for policies type") + + advanced = { + 'advanced_dns_libpcap_0': "handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.orb.live/ .google.com], only_rcode=0", + 'advanced_dns_libpcap_2': "handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.orb.live/ .google.com], only_rcode=2", + 'advanced_dns_libpcap_3': "handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.orb.live/ .google.com], only_rcode=3", + 'advanced_dns_libpcap_5': "handler=dns, description='policy_dns', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap, only_qname_suffix=[.orb.live/ .google.com], only_rcode=5", + + 'advanced_net': "handler=net, description='policy_net', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap", + + 'advanced_dhcp': "handler=dhcp, description='policy_dhcp', host_specification=10.0.1.0/24,10.0.2.1/32,2001:db8::/64, bpf_filter_expression=udp port 53, pcap_source=libpcap", + } + + simple = { + + 'simple_dns': "handler=dns", + 'simple_net': "handler=net", + # 'simple_dhcp': "handler=dhcp", + } + + mixed = dict() + mixed.update(advanced) + mixed.update(simple) + + master_dict = {'advanced': advanced, 'simple': simple, 'mixed': mixed} + + if k <= len(master_dict[policies_type]): + return sample(list(master_dict[policies_type].items()), k=k) + + return choices(list(master_dict[policies_type].items()), k=k) + + +def return_policy_attribute(policy, attribute): + """ + + :param (dict) policy: json of policy + :param (str) attribute: policy attribute whose value is to be returned + :return: (str, bool or None) value referring to policy attribute + + """ + + handler_label = list(policy["policy"]["handlers"]["modules"].keys())[0] + if attribute == "name": + return policy['name'] + elif attribute == "handler_label": + return handler_label + elif attribute == "handler": + return list(policy["policy"]["handlers"]["modules"].values())[0]["type"] + elif attribute == "backend_type": + return policy["backend"] + elif attribute == "tap": + return policy["policy"]["input"]["tap"] + elif attribute == "input_type": + return policy["policy"]["input"]["input_type"] + elif attribute == "version" and "version" in policy.keys(): + return policy["version"] + elif attribute == "description" and "description" in policy.keys(): + return policy['description'] + elif attribute == "host_specification" and "host_spec" in policy["policy"]["input"]["config"].keys(): + return policy["policy"]["input"]["config"]["host_spec"] + elif attribute == "bpf_filter_expression" and "bpf" in policy["policy"]["input"]["filter"].keys(): + return policy["policy"]["input"]["filter"]["bpf"] + elif attribute == "pcap_source" and "pcap_source" in policy["policy"]["input"]["config"].keys(): + return policy["policy"]["input"]["config"]["pcap_source"] + elif attribute == "only_qname_suffix" and "only_qname_suffix" in policy["policy"]["handlers"]["modules"][handler_label]["filter"].keys(): + return policy["policy"]["handlers"]["modules"][handler_label]["filter"]["only_qname_suffix"] + elif attribute == "only_rcode" and "only_rcode" in policy["policy"]["handlers"]["modules"][handler_label]["filter"].keys(): + return policy["policy"]["handlers"]["modules"][handler_label]["filter"]["only_rcode"] + else: + return None diff --git a/python-test/features/steps/login.py b/python-test/features/steps/login.py index ab6e55cca..fe3747e0e 100644 --- a/python-test/features/steps/login.py +++ b/python-test/features/steps/login.py @@ -61,10 +61,10 @@ def request_account_registration(context, password_status, username, company): passwords_to_test.append(password[:8]) if len(password) > 8: passwords_to_test.append(password[:-1]) for current_password in passwords_to_test: - response = register_account(email, current_password, company, username, 409) + response, status_code = register_account(email, current_password, company, username, 409) assert_that(response.json()['error'], equal_to('email already taken'), 'Wrong message on API response') else: - response = register_account(email, password, company, username, 409) + response, status_code = register_account(email, password, company, username, 409) assert_that(response.json()['error'], equal_to('email already taken'), 'Wrong message on API response') diff --git a/python-test/features/steps/page_objects.py b/python-test/features/steps/page_objects.py index 8bf4d8faf..43515828c 100644 --- a/python-test/features/steps/page_objects.py +++ b/python-test/features/steps/page_objects.py @@ -47,11 +47,11 @@ def agent_provisioning_command(cls): @classmethod def agent_view_id(cls): - return "//label[contains(text(), 'Agent ID')]/following::p" + return "//label[contains(text(), 'Agent ID')]/following::span" @classmethod def agent_status(cls): - return "//label[contains(text(), 'Health Status')]/following::p" + return "//span[@class='float-right']//child::span" class UtilButton: diff --git a/python-test/features/steps/users.py b/python-test/features/steps/users.py index 3dddcc6a4..5517810d5 100644 --- a/python-test/features/steps/users.py +++ b/python-test/features/steps/users.py @@ -1,14 +1,70 @@ from hamcrest import * -from behave import given +from behave import given, step, then from test_config import TestConfig import requests +from utils import random_string configs = TestConfig.configs() base_orb_url = configs.get('base_orb_url') +@step("that there is an unregistered {email} email with {password} password") +def check_non_registered_account(context, email, password): + assert_that(email, any_of(equal_to("valid"), equal_to("invalid")), "Unexpected value for email") + if email == "valid": + context.email = [f"tester_{random_string(4)}@email.com", email] + status_code = 403 + else: + context.email = [f"tester.com", email] + status_code = 400 + context.password = password + authenticate(context.email[0], context.password, status_code) + + +@step("the Orb user request this account registration with {company} as company and {fullname} as fullname") +def request_account_registration(context, company, fullname): + if company.lower() == "none": company = None + if fullname.lower() == "none": fullname = None + context.company = company + context.full_name = fullname + + if context.email[1] == "invalid" or not (8 <= len(context.password) <= 50): + status_code = 400 + else: + status_code = 201 + context.registration_response, context.status_code = register_account(context.email[0], context.password, + context.company, context.full_name, + status_code) + +@then("the status code must be {status_code}") +def register_orb_account(context, status_code): + status_code = int(status_code) + assert_that(context.status_code, equal_to(status_code), "Unexpected status code for account registration") + + + +@step("account is registered with email, with password, {company_status} company and {fullname_status} full name") +def check_account_information(context, company_status, fullname_status): + if company_status.lower() == "none": company_status = None + if fullname_status.lower() == "none": fullname_status = None + context.token = authenticate(context.email[0], context.password)["token"] + account_details = get_account_information(context.token) + expected_keys = ["id", "email", "metadata"] + for key in expected_keys: + assert_that(account_details, has_key(key), f"key {key} not present in account data") + expected_metadata = dict() + if company_status is not None: + expected_metadata["company"] = context.company + if fullname_status is not None: + expected_metadata["fullName"] = context.full_name + for metadata in expected_metadata.keys(): + assert_that(account_details["metadata"], has_key(metadata), f"Missing {metadata} in account metadata") + assert_that(account_details["metadata"][metadata], equal_to(expected_metadata[metadata]), + f"Unexpected value for {metadata}") + + @given("the Orb user has a registered account") -def register_orb_account(context): +def check_account_registration(context): email = configs.get('email') password = configs.get('password') if configs.get('is_credentials_registered') == 'false': @@ -71,4 +127,17 @@ def register_account(user_email, user_password, company_name=None, user_full_nam assert_that(response.status_code, equal_to(expected_status_code), f"Current value of is_credentials_registered parameter = {configs.get('is_credentials_registered')}." f"\nExpected status code for registering an account failed with status= {str(response.status_code)}.") - return response + return response, response.status_code, + + +def get_account_information(token, expected_status_code=200): + """ + + :param (str) token: used for API authentication + :param (int) expected_status_code: expected request's status code. Default:200 (happy path). + :return: (dict) account_response + """ + response = requests.get(base_orb_url + '/api/v1/users/profile', + headers={'Authorization': token}) + assert_that(response.status_code, equal_to(expected_status_code), "Unexpected status code for get account data") + return response.json() diff --git a/python-test/features/steps/utils.py b/python-test/features/steps/utils.py index 0a40b89ea..58673cc90 100644 --- a/python-test/features/steps/utils.py +++ b/python-test/features/steps/utils.py @@ -103,3 +103,16 @@ def check_logs_contain_message_and_name(logs, expected_message, name, name_key): return True, log_line return False, "Logs doesn't contain the message and name expected" + + +def remove_empty_from_json(json_file): + """ + Delete keys with the value "None" in a dictionary, recursively. + + """ + for key, value in list(json_file.items()): + if value is None: + del json_file[key] + elif isinstance(value, dict): + remove_empty_from_json(value) + return json_file diff --git a/python-test/mkdocs.yml b/python-test/mkdocs.yml index 5c8af50af..3914d2765 100644 --- a/python-test/mkdocs.yml +++ b/python-test/mkdocs.yml @@ -59,13 +59,19 @@ nav: - Check agent groups details: agent_groups/check_agent_groups_details.md - Edit an agent group through the details modal: agent_groups/edit_an_agent_group_through_the_details_modal.md - Check if is possible cancel operations with no change: agent_groups/check_if_is_possible_cancel_operations_with_no_change.md - - Edit agent group name: agent_groups/edit_agent_group_name.md - - Edit agent group description: agent_groups/edit_agent_group_description.md - - Edit agent group tag: agent_groups/edit_agent_group_tag.md - Remove agent group using correct name: agent_groups/remove_agent_group_using_correct_name.md - Remove agent group using incorrect name: agent_groups/remove_agent_group_using_incorrect_name.md - Run two orb agents on the same port: agents/run_two_orb_agents_on_the_same_port.md - Run two orb agents on different ports: agents/run_two_orb_agents_on_different_ports.md + - Edit Agent Group name removing name: agent_groups/edit_agent_group_name_removing_name.md + - Edit agent group tag: agent_groups/edit_agent_group_tag.md + - Edit agent group name: agent_groups/edit_agent_group_name.md + - Edit agent group description: agent_groups/edit_agent_group_description.md + - Edit Agent Group description removing description: agent_groups/edit_agent_group_description_removing_description.md + - Edit Agent Group tags to subscribe agent: agent_groups/edit_agent_group_tags_to_subscribe_agent.md + - Edit Agent Group tags to unsubscribe agent: agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md + - Edit Agent Group removing tags: agent_groups/edit_agent_group_removing_tags.md + - Edit Agent Group name, description and tags: agent_groups/edit_agent_group_name,_description_and_tags.md - Sink Scenarios: - Check if total sinks on sinks' page is correct: sinks/check_if_total_sinks_on_sinks'_page_is_correct.md - Create sink with invalid name (regex): sinks/create_sink_with_invalid_name_(regex).md @@ -103,6 +109,11 @@ nav: - Check policies details: policies/check_policies_details.md - Edit a policy through the details modal: policies/edit_a_policy_through_the_details_modal.md - Edit policy name: policies/edit_policy_name.md + - Edit policy host_specification: policies/edit_policy_host_specification.md + - Edit policy bpf_filter_expression: policies/edit_policy_bpf_filter_expression.md + - Edit policy pcap_source: policies/edit_policy_pcap_source.md + - Edit policy only_qname_suffix: policies/edit_policy_only_qname_suffix.md + - Edit policy only_rcode: policies/edit_policy_only_rcode.md - Edit policy description: policies/edit_policy_description.md - Edit policy handler: policies/edit_policy_handler.md - Check if is possible cancel operations with no change: policies/check_if_is_possible_cancel_operations_with_no_change.md @@ -148,6 +159,14 @@ nav: - Agent subscription to group with policies after editing agent's tags: integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md - Edit agent name and apply policies to then: integration/edit_agent_name_and_apply_policies_to_then.md - Insert tags in agents created without tags and apply policies to group matching new tags.md: integration/insert_tags_in_agents_created_without_tags_and_apply_policies_to_group_matching_new_tags.md + - Agent unsubscription to group with policies after editing agent group's tags (editing tags after agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent subscription to group with policies after editing agent group's tags (editing tags after agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent unsubscription to group with policies after editing agent group's tags (editing tags before agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent subscription to group with policies after editing agent group's tags (editing tags before agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md - Suites: - Smoke: @@ -192,6 +211,23 @@ nav: - Agent subscription to group with policies after editing agent's tags: integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md - Edit agent name and apply policies to then: integration/edit_agent_name_and_apply_policies_to_then.md - Insert tags in agents created without tags and apply policies to group matching new tags.md: integration/insert_tags_in_agents_created_without_tags_and_apply_policies_to_group_matching_new_tags.md + - Edit Agent Group name removing name: agent_groups/edit_agent_group_name_removing_name.md + - Edit agent group name: agent_groups/edit_agent_group_name.md + - Edit agent group description: agent_groups/edit_agent_group_description.md + - Edit Agent Group description removing description: agent_groups/edit_agent_group_description_removing_description.md + - Edit agent group tag: agent_groups/edit_agent_group_tag.md + - Edit Agent Group tags to subscribe agent: agent_groups/edit_agent_group_tags_to_subscribe_agent.md + - Edit Agent Group tags to unsubscribe agent: agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md + - Edit Agent Group removing tags: agent_groups/edit_agent_group_removing_tags.md + - Edit Agent Group name, description and tags: agent_groups/edit_agent_group_name,_description_and_tags.md + - Agent unsubscription to group with policies after editing agent group's tags (editing tags after agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent subscription to group with policies after editing agent group's tags (editing tags after agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent unsubscription to group with policies after editing agent group's tags (editing tags before agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent subscription to group with policies after editing agent group's tags (editing tags before agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md - Sanity: - Sanity list: sanity.md - Sanity tests: @@ -227,6 +263,22 @@ nav: - Subscribe an agent to multiple groups created after agent provisioning: integration/subscribe_an_agent_to_multiple_groups_created_after_agent_provisioning.md - Agent subscription to group after editing agent's tags: integration/agent_subscription_to_group_after_editing_agent's_tags.md - Agent subscription to group with policies after editing agent's tags: integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags.md - - Edit agent name and tag: agents/edit_agent_name_and_tags.md - Edit agent name and apply policies to then: integration/edit_agent_name_and_apply_policies_to_then.md - Insert tags in agents created without tags and apply policies to group matching new tags.md: integration/insert_tags_in_agents_created_without_tags_and_apply_policies_to_group_matching_new_tags.md + - Edit Agent Group name removing name: agent_groups/edit_agent_group_name_removing_name.md + - Edit agent group name: agent_groups/edit_agent_group_name.md + - Edit agent group description: agent_groups/edit_agent_group_description.md + - Edit Agent Group description removing description: agent_groups/edit_agent_group_description_removing_description.md + - Edit agent group tag: agent_groups/edit_agent_group_tag.md + - Edit Agent Group tags to subscribe agent: agent_groups/edit_agent_group_tags_to_subscribe_agent.md + - Edit Agent Group tags to unsubscribe agent: agent_groups/edit_agent_group_tags_to_unsubscribe_agent.md + - Edit Agent Group removing tags: agent_groups/edit_agent_group_removing_tags.md + - Edit Agent Group name, description and tags: agent_groups/edit_agent_group_name,_description_and_tags.md + - Agent unsubscription to group with policies after editing agent group's tags (editing tags after agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent subscription to group with policies after editing agent group's tags (editing tags after agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent unsubscription to group with policies after editing agent group's tags (editing tags before agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent subscription to group with policies after editing agent group's tags (editing tags before agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags after agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_after_agent_provision.md + - Agent unsubscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision): integration/agent_unsubscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md + - Agent subscription to group with policies after editing agent's tags and agent group's tags (editing tags before agent provision): integration/agent_subscription_to_group_with_policies_after_editing_agent's_tags_and_agent_group's_tags_editing_tags_before_agent_provision.md diff --git a/sinker/backend/pktvisor/pktvisor_test.go b/sinker/backend/pktvisor/pktvisor_test.go index c73555862..9a105cb81 100644 --- a/sinker/backend/pktvisor/pktvisor_test.go +++ b/sinker/backend/pktvisor/pktvisor_test.go @@ -43,6 +43,29 @@ func TestDHCPConversion(t *testing.T) { be := backend.GetBackend("pktvisor") + commonLabels := []prometheus.Label{ + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + } + cases := map[string]struct { data []byte expected prometheus.TimeSeries @@ -59,32 +82,129 @@ func TestDHCPConversion(t *testing.T) { } }`), expected: prometheus.TimeSeries{ - Labels: []prometheus.Label{ - { - Name: "__name__", - Value: "dhcp_wire_packets_filtered", - }, - { - Name: "instance", - Value: "agent-test", - }, - { - Name: "agent_id", - Value: agentID.String(), - }, - { - Name: "agent", - Value: "agent-test", - }, - { - Name: "policy_id", - Value: policyID.String(), - }, - { - Name: "policy", - Value: "policy-test", - }, + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_wire_packets_filtered"})), + Datapoint: prometheus.Datapoint{ + Value: 10, + }, + }, + }, + "DHCPPayloadWirePacketsTotal": { + data: []byte(` +{ + "policy_dhcp": { + "dhcp": { + "wire_packets": { + "total": 10 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_wire_packets_total"})), + Datapoint: prometheus.Datapoint{ + Value: 10, + }, + }, + }, + "DHCPPayloadWirePacketsDeepSamples": { + data: []byte(` +{ + "policy_dhcp": { + "dhcp": { + "wire_packets": { + "deep_samples": 10 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_wire_packets_deep_samples"})), + Datapoint: prometheus.Datapoint{ + Value: 10, + }, + }, + }, + "DHCPPayloadWirePacketsDiscover": { + data: []byte(` +{ + "policy_dhcp": { + "dhcp": { + "wire_packets": { + "discover": 10 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_wire_packets_discover"})), + Datapoint: prometheus.Datapoint{ + Value: 10, + }, + }, + }, + "DHCPPayloadWirePacketsOffer": { + data: []byte(` +{ + "policy_dhcp": { + "dhcp": { + "wire_packets": { + "offer": 10 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_wire_packets_offer"})), + Datapoint: prometheus.Datapoint{ + Value: 10, + }, + }, + }, + "DHCPPayloadWirePacketsRequest": { + data: []byte(` +{ + "policy_dhcp": { + "dhcp": { + "wire_packets": { + "request": 10 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_wire_packets_request"})), + Datapoint: prometheus.Datapoint{ + Value: 10, }, + }, + }, + "DHCPPayloadWirePacketsAck": { + data: []byte(` +{ + "policy_dhcp": { + "dhcp": { + "wire_packets": { + "ack": 10 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_wire_packets_ack"})), Datapoint: prometheus.Datapoint{ Value: 10, }, @@ -248,7 +368,7 @@ func TestGeoLocConversion(t *testing.T) { data []byte expected prometheus.TimeSeries }{ - "PacketPayloadTopASN": { + "PacketPayloadTopGeoLoc": { data: []byte(` { "policy_packets": { @@ -319,3 +439,1969 @@ func TestGeoLocConversion(t *testing.T) { } } + +func TestPCAPConversion(t *testing.T) { + var logger *zap.Logger + pktvisor.Register(logger) + + ownerID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + agentID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + var agent = &pb.OwnerRes{ + OwnerID: ownerID.String(), + AgentName: "agent-test", + } + + data := fleet.AgentMetricsRPCPayload{ + PolicyID: policyID.String(), + PolicyName: "policy-test", + Datasets: nil, + Format: "json", + BEVersion: "1.0", + } + + be := backend.GetBackend("pktvisor") + + commonLabels := []prometheus.Label{ + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + } + + cases := map[string]struct { + data []byte + expected prometheus.TimeSeries + }{ + "PCAPPayload_Tcp_Reassembly_Errors": { + data: []byte(` +{ + "policy_pcap": { + "pcap": { + "tcp_reassembly_errors": 2 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "pcap_tcp_reassembly_errors", + })), + Datapoint: prometheus.Datapoint{ + Value: 2, + }, + }, + }, + "PCAPPayload_if_drops": { + data: []byte(` +{ + "policy_pcap": { + "pcap": { + "if_drops": 2 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "pcap_if_drops", + })), + Datapoint: prometheus.Datapoint{ + Value: 2, + }, + }, + }, + "PCAPPayload_os_drops": { + data: []byte(` +{ + "policy_pcap": { + "pcap": { + "os_drops": 2 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "pcap_os_drops", + })), + Datapoint: prometheus.Datapoint{ + Value: 2, + }, + }, + }, + } + + for desc, c := range cases { + t.Run(desc, func(t *testing.T) { + data.Data = c.data + res, err := be.ProcessMetrics(agent, agentID.String(), data) + require.Nil(t, err, fmt.Sprintf("%s: unexpected error: %s", desc, err)) + var receivedLabel []prometheus.Label + var receivedDatapoint prometheus.Datapoint + for _, value := range res { + if c.expected.Labels[0] == value.Labels[0] { + receivedLabel = value.Labels + receivedDatapoint = value.Datapoint + } + } + assert.True(t, reflect.DeepEqual(c.expected.Labels, receivedLabel), fmt.Sprintf("%s: expected %v got %v", desc, c.expected.Labels, receivedLabel)) + assert.Equal(t, c.expected.Datapoint.Value, receivedDatapoint.Value, fmt.Sprintf("%s: expected value %f got %f", desc, c.expected.Datapoint.Value, receivedDatapoint.Value)) + }) + } + +} + +func TestDNSConversion(t *testing.T) { + var logger *zap.Logger + pktvisor.Register(logger) + + ownerID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + agentID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + var agent = &pb.OwnerRes{ + OwnerID: ownerID.String(), + AgentName: "agent-test", + } + + data := fleet.AgentMetricsRPCPayload{ + PolicyID: policyID.String(), + PolicyName: "policy-test", + Datasets: nil, + Format: "json", + BEVersion: "1.0", + } + + be := backend.GetBackend("pktvisor") + + commonLabels := []prometheus.Label{ + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + } + + cases := map[string]struct { + data []byte + expected prometheus.TimeSeries + }{ + "DNSPayloadCardinalityTotal": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "cardinality": { + "qname": 4 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_cardinality_qname", + })), + Datapoint: prometheus.Datapoint{ + Value: 4, + }, + }, + }, + "DNSPayloadTopNxdomain": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "top_nxdomain": [ + { + "estimate": 186, + "name": "89.187.189.231" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_top_nxdomain", + }), prometheus.Label{ + Name: "qname", + Value: "89.187.189.231", + }), + Datapoint: prometheus.Datapoint{ + Value: 186, + }, + }, + }, + "DNSPayloadTopRefused": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "top_refused": [ + { + "estimate": 186, + "name": "89.187.189.231" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_top_refused", + }), prometheus.Label{ + Name: "qname", + Value: "89.187.189.231", + }), + Datapoint: prometheus.Datapoint{ + Value: 186, + }, + }, + }, + "DNSPayloadTopSrvfail": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "top_srvfail": [ + { + "estimate": 186, + "name": "89.187.189.231" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_top_srvfail", + }), prometheus.Label{ + Name: "qname", + Value: "89.187.189.231", + }), + Datapoint: prometheus.Datapoint{ + Value: 186, + }, + }, + }, + } + + for desc, c := range cases { + t.Run(desc, func(t *testing.T) { + data.Data = c.data + res, err := be.ProcessMetrics(agent, agentID.String(), data) + require.Nil(t, err, fmt.Sprintf("%s: unexpected error: %s", desc, err)) + var receivedLabel []prometheus.Label + var receivedDatapoint prometheus.Datapoint + for _, value := range res { + if c.expected.Labels[0] == value.Labels[0] { + if len(c.expected.Labels) < 7 { + receivedLabel = value.Labels + receivedDatapoint = value.Datapoint + } else { + if c.expected.Labels[6].Value == value.Labels[6].Value { + receivedLabel = value.Labels + receivedDatapoint = value.Datapoint + } + } + } + } + assert.True(t, reflect.DeepEqual(c.expected.Labels, receivedLabel), fmt.Sprintf("%s: expected %v got %v", desc, c.expected.Labels, receivedLabel)) + assert.Equal(t, c.expected.Datapoint.Value, receivedDatapoint.Value, fmt.Sprintf("%s: expected value %f got %f", desc, c.expected.Datapoint.Value, receivedDatapoint.Value)) + }) + } + +} + +func TestDNSRatesConversion(t *testing.T) { + var logger *zap.Logger + pktvisor.Register(logger) + + ownerID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + agentID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + var agent = &pb.OwnerRes{ + OwnerID: ownerID.String(), + AgentName: "agent-test", + } + + data := fleet.AgentMetricsRPCPayload{ + PolicyID: policyID.String(), + PolicyName: "policy-test", + Datasets: nil, + Format: "json", + BEVersion: "1.0", + } + + be := backend.GetBackend("pktvisor") + + commonLabels := []prometheus.Label{ + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "quantile", + Value: "0.5", + }, + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "quantile", + Value: "0.9", + }, + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "quantile", + Value: "0.95", + }, + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "quantile", + Value: "0.99", + }, + } + + cases := map[string]struct { + data []byte + expectedLabels []prometheus.Label + expectedDatapoints []float64 + }{ + "DNSPayloadRatesTotal": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "rates": { + "total": { + "p50": 0, + "p90": 1, + "p95": 2, + "p99": 6 + } + } + } + } +}`), + expectedLabels: labelQuantiles(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_rates_total", + }), + expectedDatapoints: []float64{0, 1, 2, 6}, + }, + "PacketsPayloadRatesPpsIn": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "rates": { + "pps_in": { + "p50": 0, + "p90": 1, + "p95": 2, + "p99": 6 + } + } + } + } +}`), + expectedLabels: labelQuantiles(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_rates_pps_in", + }), + expectedDatapoints: []float64{0, 1, 2, 6}, + }, + "PacketsPayloadRatesPpsTotal": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "rates": { + "pps_total": { + "p50": 0, + "p90": 1, + "p95": 2, + "p99": 6 + } + } + } + } +}`), + expectedLabels: labelQuantiles(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_rates_pps_total", + }), + expectedDatapoints: []float64{0, 1, 2, 6}, + }, + "PacketsPayloadRatesPpsOut": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "rates": { + "pps_out": { + "p50": 0, + "p90": 1, + "p95": 2, + "p99": 6 + } + } + } + } +}`), + expectedLabels: labelQuantiles(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_rates_pps_out", + }), + expectedDatapoints: []float64{0, 1, 2, 6}, + }, + "DHCPPayloadRates": { + data: []byte(` +{ + "policy_dhcp": { + "dhcp": { + "rates": { + "total": { + "p50": 0, + "p90": 1, + "p95": 2, + "p99": 6 + } + } + } + } +}`), + expectedLabels: labelQuantiles(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_rates_total", + }), + expectedDatapoints: []float64{0, 1, 2, 6}, + }, + } + + for desc, c := range cases { + t.Run(desc, func(t *testing.T) { + data.Data = c.data + res, err := be.ProcessMetrics(agent, agentID.String(), data) + require.Nil(t, err, fmt.Sprintf("%s: unexpected error: %s", desc, err)) + var receivedLabel []prometheus.Label + var receivedDatapoint []float64 + + for _, value := range res { + if c.expectedLabels[0] == value.Labels[0] { + for _, labels := range value.Labels { + receivedLabel = append(receivedLabel, labels) + } + receivedDatapoint = append(receivedDatapoint, value.Datapoint.Value) + } + } + + assert.ElementsMatch(t, c.expectedLabels, receivedLabel, fmt.Sprintf("%s: expected %v got %v", desc, c.expectedLabels, receivedLabel)) + assert.ElementsMatch(t, c.expectedDatapoints, receivedDatapoint, fmt.Sprintf("%s: expected %v got %v", desc, c.expectedDatapoints, receivedDatapoint)) + }) + } + +} + +func TestDNSTopKMetricsConversion(t *testing.T) { + var logger *zap.Logger + pktvisor.Register(logger) + + ownerID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + agentID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + var agent = &pb.OwnerRes{ + OwnerID: ownerID.String(), + AgentName: "agent-test", + } + + data := fleet.AgentMetricsRPCPayload{ + PolicyID: policyID.String(), + PolicyName: "policy-test", + Datasets: nil, + Format: "json", + BEVersion: "1.0", + } + + be := backend.GetBackend("pktvisor") + + cases := map[string]struct { + data []byte + expected prometheus.TimeSeries + }{ + "PacketPayloadToqQName2": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "top_qname2": [ + { + "estimate": 8, + "name": ".google.com" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: []prometheus.Label{ + { + Name: "__name__", + Value: "dns_top_qname2", + }, + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "qname", + Value: ".google.com", + }, + }, + Datapoint: prometheus.Datapoint{ + Value: 8, + }, + }, + }, + "PacketPayloadToqQName3": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "top_qname3": [ + { + "estimate": 6, + "name": ".l.google.com" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: []prometheus.Label{ + { + Name: "__name__", + Value: "dns_top_qname3", + }, + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "qname", + Value: ".l.google.com", + }, + }, + Datapoint: prometheus.Datapoint{ + Value: 6, + }, + }, + }, + "PacketPayloadToqQType": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "top_qtype": [ + { + "estimate": 6, + "name": "HTTPS" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: []prometheus.Label{ + { + Name: "__name__", + Value: "dns_top_qtype", + }, + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "qtype", + Value: "HTTPS", + }, + }, + Datapoint: prometheus.Datapoint{ + Value: 6, + }, + }, + }, + "PacketPayloadTopUDPPorts": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "top_udp_ports": [ + { + "estimate": 2, + "name": "39783" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: []prometheus.Label{ + { + Name: "__name__", + Value: "dns_top_udp_ports", + }, + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "port", + Value: "39783", + }, + }, + Datapoint: prometheus.Datapoint{ + Value: 2, + }, + }, + }, + "PacketPayloadTopRCode": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "top_rcode": [ + { + "estimate": 8, + "name": "NOERROR" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: []prometheus.Label{ + { + Name: "__name__", + Value: "dns_top_rcode", + }, + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + { + Name: "rcode", + Value: "NOERROR", + }, + }, + Datapoint: prometheus.Datapoint{ + Value: 8, + }, + }, + }, + } + + for desc, c := range cases { + t.Run(desc, func(t *testing.T) { + data.Data = c.data + res, err := be.ProcessMetrics(agent, agentID.String(), data) + require.Nil(t, err, fmt.Sprintf("%s: unexpected error: %s", desc, err)) + var receivedLabel []prometheus.Label + var receivedDatapoint prometheus.Datapoint + for _, value := range res { + if c.expected.Labels[0] == value.Labels[0] { + receivedLabel = value.Labels + receivedDatapoint = value.Datapoint + } + } + assert.True(t, reflect.DeepEqual(c.expected.Labels, receivedLabel), fmt.Sprintf("%s: expected %v got %v", desc, c.expected.Labels, receivedLabel)) + assert.Equal(t, c.expected.Datapoint.Value, receivedDatapoint.Value, fmt.Sprintf("%s: expected value %f got %f", desc, c.expected.Datapoint.Value, receivedDatapoint.Value)) + }) + } + +} + +func TestDNSWirePacketsConversion(t *testing.T) { + var logger *zap.Logger + pktvisor.Register(logger) + + ownerID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + agentID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + var agent = &pb.OwnerRes{ + OwnerID: ownerID.String(), + AgentName: "agent-test", + } + + data := fleet.AgentMetricsRPCPayload{ + PolicyID: policyID.String(), + PolicyName: "policy-test", + Datasets: nil, + Format: "json", + BEVersion: "1.0", + } + + be := backend.GetBackend("pktvisor") + + commonLabels := []prometheus.Label{ + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + } + + cases := map[string]struct { + data []byte + expected prometheus.TimeSeries + }{ + "DNSPayloadWirePacketsIpv4": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "ipv4": 1 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_ipv4", + })), + Datapoint: prometheus.Datapoint{ + Value: 1, + }, + }, + }, + "DNSPayloadWirePacketsIpv6": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "ipv6": 14 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_ipv6", + })), + Datapoint: prometheus.Datapoint{ + Value: 14, + }, + }, + }, + "DNSPayloadWirePacketsNoerror": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "noerror": 8 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_noerror", + })), + Datapoint: prometheus.Datapoint{ + Value: 8, + }, + }, + }, + "DNSPayloadWirePacketsNxdomain": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "nxdomain": 6 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_nxdomain", + })), + Datapoint: prometheus.Datapoint{ + Value: 6, + }, + }, + }, + "DNSPayloadWirePacketsQueries": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "queries": 7 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_queries", + })), + Datapoint: prometheus.Datapoint{ + Value: 7, + }, + }, + }, + "DNSPayloadWirePacketsRefused": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "refused": 8 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_refused", + })), + Datapoint: prometheus.Datapoint{ + Value: 8, + }, + }, + }, + "DNSPayloadWirePacketsReplies": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "replies": 8 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_replies", + })), + Datapoint: prometheus.Datapoint{ + Value: 8, + }, + }, + }, + "DNSPayloadWirePacketsSrvfail": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "srvfail": 9 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_srvfail", + })), + Datapoint: prometheus.Datapoint{ + Value: 9, + }, + }, + }, + "DNSPayloadWirePacketsTcp": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "tcp": 9 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_tcp", + })), + Datapoint: prometheus.Datapoint{ + Value: 9, + }, + }, + }, + "DNSPayloadWirePacketsTotal": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "total": 9 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_total", + })), + Datapoint: prometheus.Datapoint{ + Value: 9, + }, + }, + }, + "DNSPayloadWirePacketsUdp": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "wire_packets": { + "udp": 9 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_wire_packets_udp", + })), + Datapoint: prometheus.Datapoint{ + Value: 9, + }, + }, + }, + } + + for desc, c := range cases { + t.Run(desc, func(t *testing.T) { + data.Data = c.data + res, err := be.ProcessMetrics(agent, agentID.String(), data) + require.Nil(t, err, fmt.Sprintf("%s: unexpected error: %s", desc, err)) + var receivedLabel []prometheus.Label + var receivedDatapoint prometheus.Datapoint + for _, value := range res { + if c.expected.Labels[0] == value.Labels[0] { + receivedLabel = value.Labels + receivedDatapoint = value.Datapoint + } + } + assert.True(t, reflect.DeepEqual(c.expected.Labels, receivedLabel), fmt.Sprintf("%s: expected %v got %v", desc, c.expected.Labels, receivedLabel)) + assert.Equal(t, c.expected.Datapoint.Value, receivedDatapoint.Value, fmt.Sprintf("%s: expected value %f got %f", desc, c.expected.Datapoint.Value, receivedDatapoint.Value)) + }) + } + +} + +func TestDNSXactConversion(t *testing.T) { + var logger *zap.Logger + pktvisor.Register(logger) + + ownerID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + agentID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + var agent = &pb.OwnerRes{ + OwnerID: ownerID.String(), + AgentName: "agent-test", + } + + data := fleet.AgentMetricsRPCPayload{ + PolicyID: policyID.String(), + PolicyName: "policy-test", + Datasets: nil, + Format: "json", + BEVersion: "1.0", + } + + be := backend.GetBackend("pktvisor") + + commonLabels := []prometheus.Label{ + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + } + + cases := map[string]struct { + data []byte + expected prometheus.TimeSeries + }{ + "DNSPayloadXactCountsTimedOut": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "xact": { + "counts": { + "timed_out": 1 + } + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_xact_counts_timed_out", + })), + Datapoint: prometheus.Datapoint{ + Value: 1, + }, + }, + }, + "DNSPayloadXactCountsTotal": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "xact": { + "counts": { + "total": 8 + } + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_xact_counts_total", + })), + Datapoint: prometheus.Datapoint{ + Value: 8, + }, + }, + }, + "DNSPayloadXactInTotal": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "xact": { + "in": { + "total": 8 + } + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_xact_in_total", + })), + Datapoint: prometheus.Datapoint{ + Value: 8, + }, + }, + }, + "DNSPayloadXactInTopSlow": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "xact": { + "in": { + "top_slow": [ + { + "estimate": 111, + "name": "23.43.252.68" + } + ] + } + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_xact_in_top_slow", + }), prometheus.Label{ + Name: "qname", + Value: "23.43.252.68", + }), + Datapoint: prometheus.Datapoint{ + Value: 111, + }, + }, + }, + "DNSPayloadXactOutTopSlow": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "xact": { + "out": { + "top_slow": [ + { + "estimate": 111, + "name": "23.43.252.68" + } + ] + } + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_xact_out_top_slow", + }), prometheus.Label{ + Name: "qname", + Value: "23.43.252.68", + }), + Datapoint: prometheus.Datapoint{ + Value: 111, + }, + }, + }, + "DNSPayloadXactOutTotal": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "xact": { + "out": { + "total": 8 + } + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_xact_out_total", + })), + Datapoint: prometheus.Datapoint{ + Value: 8, + }, + }, + }, + } + + for desc, c := range cases { + t.Run(desc, func(t *testing.T) { + data.Data = c.data + res, err := be.ProcessMetrics(agent, agentID.String(), data) + require.Nil(t, err, fmt.Sprintf("%s: unexpected error: %s", desc, err)) + var receivedLabel []prometheus.Label + var receivedDatapoint prometheus.Datapoint + for _, value := range res { + if c.expected.Labels[0] == value.Labels[0] { + receivedLabel = value.Labels + receivedDatapoint = value.Datapoint + } + } + assert.True(t, reflect.DeepEqual(c.expected.Labels, receivedLabel), fmt.Sprintf("%s: expected %v got %v", desc, c.expected.Labels, receivedLabel)) + assert.Equal(t, c.expected.Datapoint.Value, receivedDatapoint.Value, fmt.Sprintf("%s: expected value %f got %f", desc, c.expected.Datapoint.Value, receivedDatapoint.Value)) + }) + } +} + +func TestPacketsConversion(t *testing.T) { + var logger *zap.Logger + pktvisor.Register(logger) + + ownerID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + agentID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + var agent = &pb.OwnerRes{ + OwnerID: ownerID.String(), + AgentName: "agent-test", + } + + data := fleet.AgentMetricsRPCPayload{ + PolicyID: policyID.String(), + PolicyName: "policy-test", + Datasets: nil, + Format: "json", + BEVersion: "1.0", + } + + be := backend.GetBackend("pktvisor") + + commonLabels := []prometheus.Label{ + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + } + + cases := map[string]struct { + data []byte + expected prometheus.TimeSeries + }{ + "DNSPayloadPacketsCardinalityDst": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "cardinality": { + "dst_ips_out": 41 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_cardinality_dst_ips_out", + })), + Datapoint: prometheus.Datapoint{ + Value: 41, + }, + }, + }, + "DNSPayloadPacketsCardinalitySrc": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "cardinality": { + "src_ips_in": 43 + } + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_cardinality_src_ips_in", + })), + Datapoint: prometheus.Datapoint{ + Value: 43, + }, + }, + }, + "DNSPayloadPacketsDeepSamples": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "deep_samples": 3139 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_deep_samples", + })), + Datapoint: prometheus.Datapoint{ + Value: 3139, + }, + }, + }, + "DNSPayloadPacketsIn": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "in": 1422 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_in", + })), + Datapoint: prometheus.Datapoint{ + Value: 1422, + }, + }, + }, + "DNSPayloadPacketsIpv4": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "ipv4": 2506 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_ipv4", + })), + Datapoint: prometheus.Datapoint{ + Value: 2506, + }, + }, + }, + "DNSPayloadPacketsIpv6": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "ipv6": 2506 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_ipv6", + })), + Datapoint: prometheus.Datapoint{ + Value: 2506, + }, + }, + }, + "DNSPayloadPacketsOtherL4": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "other_l4": 637 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_other_l4", + })), + Datapoint: prometheus.Datapoint{ + Value: 637, + }, + }, + }, + "DNSPayloadPacketsOut": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "out": 1083 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_out", + })), + Datapoint: prometheus.Datapoint{ + Value: 1083, + }, + }, + }, + "DNSPayloadPacketsTcp": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "tcp": 549 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_tcp", + })), + Datapoint: prometheus.Datapoint{ + Value: 549, + }, + }, + }, + "DNSPayloadPacketsTotal": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "total": 3139 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_total", + })), + Datapoint: prometheus.Datapoint{ + Value: 3139, + }, + }, + }, + "DNSPayloadPacketsUdp": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "udp": 1953 + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_udp", + })), + Datapoint: prometheus.Datapoint{ + Value: 1953, + }, + }, + }, + "DNSPayloadPacketsTopIpv4": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "top_ipv4": [ + { + "estimate": 996, + "name": "103.6.85.201" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_top_ipv4", + }), prometheus.Label{ + Name: "ipv4", + Value: "103.6.85.201", + }), + Datapoint: prometheus.Datapoint{ + Value: 996, + }, + }, + }, + "DNSPayloadPacketsTopIpv6": { + data: []byte(` +{ + "policy_dns": { + "packets": { + "top_ipv6": [ + { + "estimate": 996, + "name": "103.6.85.201" + } + ] + } + } +}`), + expected: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_top_ipv6", + }), prometheus.Label{ + Name: "ipv6", + Value: "103.6.85.201", + }), + Datapoint: prometheus.Datapoint{ + Value: 996, + }, + }, + }, + } + + for desc, c := range cases { + t.Run(desc, func(t *testing.T) { + data.Data = c.data + res, err := be.ProcessMetrics(agent, agentID.String(), data) + require.Nil(t, err, fmt.Sprintf("%s: unexpected error: %s", desc, err)) + var receivedLabel []prometheus.Label + var receivedDatapoint prometheus.Datapoint + for _, value := range res { + if c.expected.Labels[0] == value.Labels[0] { + receivedLabel = value.Labels + receivedDatapoint = value.Datapoint + } + } + assert.True(t, reflect.DeepEqual(c.expected.Labels, receivedLabel), fmt.Sprintf("%s: expected %v got %v", desc, c.expected.Labels, receivedLabel)) + assert.Equal(t, c.expected.Datapoint.Value, receivedDatapoint.Value, fmt.Sprintf("%s: expected value %f got %f", desc, c.expected.Datapoint.Value, receivedDatapoint.Value)) + }) + } +} + +func TestPeriodConversion(t *testing.T) { + var logger *zap.Logger + pktvisor.Register(logger) + + ownerID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + policyID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + agentID, err := uuid.NewV4() + require.Nil(t, err, fmt.Sprintf("unexpected error: %s", err)) + + var agent = &pb.OwnerRes{ + OwnerID: ownerID.String(), + AgentName: "agent-test", + } + + data := fleet.AgentMetricsRPCPayload{ + PolicyID: policyID.String(), + PolicyName: "policy-test", + Datasets: nil, + Format: "json", + BEVersion: "1.0", + } + + be := backend.GetBackend("pktvisor") + + commonLabels := []prometheus.Label{ + { + Name: "instance", + Value: "agent-test", + }, + { + Name: "agent_id", + Value: agentID.String(), + }, + { + Name: "agent", + Value: "agent-test", + }, + { + Name: "policy_id", + Value: policyID.String(), + }, + { + Name: "policy", + Value: "policy-test", + }, + } + + cases := map[string]struct { + data []byte + expectedLength prometheus.TimeSeries + expectedStartTs prometheus.TimeSeries + }{ + "DNSPayloadPeriod": { + data: []byte(` +{ + "policy_dns": { + "dns": { + "period": { + "length": 60, + "start_ts": 1624888107 + } + } + } +}`), + expectedLength: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_period_length", + })), + Datapoint: prometheus.Datapoint{ + Value: 60, + }, + }, + expectedStartTs: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dns_period_start_ts", + })), + Datapoint: prometheus.Datapoint{ + Value: 1624888107, + }, + }, + }, + "PacketsPayloadPeriod": { + data: []byte(` +{ + "policy_packets": { + "packets": { + "period": { + "length": 60, + "start_ts": 1624888107 + } + } + } +}`), + expectedLength: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_period_length", + })), + Datapoint: prometheus.Datapoint{ + Value: 60, + }, + }, + expectedStartTs: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "packets_period_start_ts", + })), + Datapoint: prometheus.Datapoint{ + Value: 1624888107, + }, + }, + }, + "DHCPPayloadPeriod": { + data: []byte(` +{ + "policy_dhcp": { + "dhcp": { + "period": { + "length": 60, + "start_ts": 1624888107 + } + } + } +}`), + expectedLength: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_period_length", + })), + Datapoint: prometheus.Datapoint{ + Value: 60, + }, + }, + expectedStartTs: prometheus.TimeSeries{ + Labels: append(prependLabel(commonLabels, prometheus.Label{ + Name: "__name__", + Value: "dhcp_period_start_ts", + })), + Datapoint: prometheus.Datapoint{ + Value: 1624888107, + }, + }, + }, + } + + for desc, c := range cases { + t.Run(desc, func(t *testing.T) { + data.Data = c.data + res, err := be.ProcessMetrics(agent, agentID.String(), data) + require.Nil(t, err, fmt.Sprintf("%s: unexpected error: %s", desc, err)) + var receivedLabelStartTs []prometheus.Label + var receivedDatapointStartTs prometheus.Datapoint + var receivedLabelLength []prometheus.Label + var receivedDatapointLength prometheus.Datapoint + for _, value := range res { + if c.expectedLength.Labels[0] == value.Labels[0] { + receivedLabelLength = value.Labels + receivedDatapointLength = value.Datapoint + } else if c.expectedStartTs.Labels[0] == value.Labels[0] { + receivedLabelStartTs = value.Labels + receivedDatapointStartTs = value.Datapoint + } + } + assert.True(t, reflect.DeepEqual(c.expectedLength.Labels, receivedLabelLength), fmt.Sprintf("%s: expected %v got %v", desc, c.expectedLength.Labels, receivedLabelLength)) + assert.Equal(t, c.expectedLength.Datapoint.Value, receivedDatapointLength.Value, fmt.Sprintf("%s: expected value %f got %f", desc, c.expectedLength.Datapoint.Value, receivedDatapointLength.Value)) + assert.True(t, reflect.DeepEqual(c.expectedStartTs.Labels, receivedLabelStartTs), fmt.Sprintf("%s: expected %v got %v", desc, c.expectedStartTs.Labels, receivedLabelStartTs)) + assert.Equal(t, c.expectedStartTs.Datapoint.Value, receivedDatapointStartTs.Value, fmt.Sprintf("%s: expected value %f got %f", desc, c.expectedStartTs.Datapoint.Value, receivedDatapointStartTs.Value)) + + }) + } +} + +func prependLabel(labelList []prometheus.Label, label prometheus.Label) []prometheus.Label { + labelList = append(labelList, prometheus.Label{}) + copy(labelList[1:], labelList) + labelList[0] = label + return labelList +} + +func labelQuantiles(labelList []prometheus.Label, label prometheus.Label) []prometheus.Label { + for i := 0; i < 28; i += 7 { + labelList = append(labelList[:i+1], labelList[i:]...) + labelList[i] = label + } + return labelList +} diff --git a/sinker/redis/consumer/streams.go b/sinker/redis/consumer/streams.go index 52ffbc507..2db166643 100644 --- a/sinker/redis/consumer/streams.go +++ b/sinker/redis/consumer/streams.go @@ -103,8 +103,8 @@ func (es eventStore) handleSinksUpdate(ctx context.Context, e updateSinkEvent) e if err != nil { return err } - var config config.SinkConfig - if err := json.Unmarshal(data, &config); err != nil { + var cfg config.SinkConfig + if err := json.Unmarshal(data, &cfg); err != nil { return err } @@ -113,15 +113,18 @@ func (es eventStore) handleSinksUpdate(ctx context.Context, e updateSinkEvent) e if err != nil { return err } - sinkConfig.Url = config.Url - sinkConfig.User = config.User - sinkConfig.Password = config.Password + sinkConfig.Url = cfg.Url + sinkConfig.User = cfg.User + sinkConfig.Password = cfg.Password + if sinkConfig.OwnerID == "" { + sinkConfig.OwnerID = e.owner + } es.configRepo.Edit(sinkConfig) } else { - config.SinkID = e.sinkID - config.OwnerID = e.owner - es.configRepo.Add(config) + cfg.SinkID = e.sinkID + cfg.OwnerID = e.owner + es.configRepo.Add(cfg) } return nil } diff --git a/sinks/api/http/endpoint.go b/sinks/api/http/endpoint.go index 4ea920c0a..09e6c1365 100644 --- a/sinks/api/http/endpoint.go +++ b/sinks/api/http/endpoint.go @@ -71,7 +71,7 @@ func updateSinkEndpoint(svc sinks.SinkService) endpoint.Endpoint { Description: req.Description, } - if err := svc.UpdateSink(ctx, req.token, sink); err != nil { + if _, err := svc.UpdateSink(ctx, req.token, sink); err != nil { return nil, err } res := sinkRes{ diff --git a/sinks/api/http/logging.go b/sinks/api/http/logging.go index 49ad55e8b..3bba17426 100644 --- a/sinks/api/http/logging.go +++ b/sinks/api/http/logging.go @@ -47,7 +47,7 @@ func (l loggingMiddleware) CreateSink(ctx context.Context, token string, s sinks return l.svc.CreateSink(ctx, token, s) } -func (l loggingMiddleware) UpdateSink(ctx context.Context, token string, s sinks.Sink) (err error) { +func (l loggingMiddleware) UpdateSink(ctx context.Context, token string, s sinks.Sink) (sink sinks.Sink, err error) { defer func(begin time.Time) { if err != nil { l.logger.Warn("method call: edit_sink", diff --git a/sinks/api/http/metrics.go b/sinks/api/http/metrics.go index ca6bf8da1..842598a9d 100644 --- a/sinks/api/http/metrics.go +++ b/sinks/api/http/metrics.go @@ -60,12 +60,12 @@ func (m metricsMiddleware) CreateSink(ctx context.Context, token string, s sinks return m.svc.CreateSink(ctx, token, s) } -func (m metricsMiddleware) UpdateSink(ctx context.Context, token string, s sinks.Sink) (err error) { +func (m metricsMiddleware) UpdateSink(ctx context.Context, token string, s sinks.Sink) (sink sinks.Sink, err error) { defer func(begin time.Time) { labels := []string{ "method", "updateSink", - "owner_id", s.MFOwnerID, - "sink_id", s.ID, + "owner_id", sink.MFOwnerID, + "sink_id", sink.ID, } m.counter.With(labels...).Add(1) diff --git a/sinks/redis/producer/events.go b/sinks/redis/producer/events.go index 8ed5bb609..d0ee27f29 100644 --- a/sinks/redis/producer/events.go +++ b/sinks/redis/producer/events.go @@ -17,7 +17,7 @@ import ( const ( SinkPrefix = "sinks." SinkCreate = SinkPrefix + "create" - SinkDelete = SinkPrefix + "delete" + SinkDelete = SinkPrefix + "remove" SinkUpdate = SinkPrefix + "update" ) @@ -49,12 +49,14 @@ func (cce createSinkEvent) Encode() map[string]interface{} { } type deleteSinkEvent struct { - id string + sinkID string + ownerID string } func (dse deleteSinkEvent) Encode() map[string]interface{} { return map[string]interface{}{ - "id": dse.id, + "sink_id": dse.sinkID, + "owner_id": dse.ownerID, "operation": SinkDelete, } } diff --git a/sinks/redis/producer/streams.go b/sinks/redis/producer/streams.go index 3df93c9c8..e68779ede 100644 --- a/sinks/redis/producer/streams.go +++ b/sinks/redis/producer/streams.go @@ -41,35 +41,32 @@ func (es eventStore) CreateSink(ctx context.Context, token string, s sinks.Sink) return es.svc.CreateSink(ctx, token, s) } -func (es eventStore) UpdateSink(ctx context.Context, token string, s sinks.Sink) (err error) { - if err := es.svc.UpdateSink(ctx, token, s); err != nil { - return err - } - - event := updateSinkEvent{ - sinkID: s.ID, - owner: s.MFOwnerID, - config: s.Config, - } - - encode, err := event.Encode() - if err != nil { - es.logger.Error("error encoding object", zap.Error(err)) - return err - } - - record := &redis.XAddArgs{ - Stream: streamID, - MaxLenApprox: streamLen, - Values: encode, - } - - err = es.client.XAdd(ctx, record).Err() - if err != nil { - es.logger.Error("error sending event to event store", zap.Error(err)) - return err - } - return nil +func (es eventStore) UpdateSink(ctx context.Context, token string, s sinks.Sink) (sink sinks.Sink,err error) { + defer func() { + event := updateSinkEvent{ + sinkID: sink.ID, + owner: sink.MFOwnerID, + config: sink.Config, + } + + encode, err := event.Encode() + if err != nil { + es.logger.Error("error encoding object", zap.Error(err)) + } + + record := &redis.XAddArgs{ + Stream: streamID, + MaxLenApprox: streamLen, + Values: encode, + } + + err = es.client.XAdd(ctx, record).Err() + if err != nil { + es.logger.Error("error sending event to event store", zap.Error(err)) + } + }() + + return es.svc.UpdateSink(ctx, token, s) } func (es eventStore) ListSinks(ctx context.Context, token string, pm sinks.PageMetadata) (sinks.Page, error) { @@ -89,12 +86,18 @@ func (es eventStore) ViewSink(ctx context.Context, token string, key string) (_ } func (es eventStore) DeleteSink(ctx context.Context, token, id string) (err error) { + sink, err := es.svc.ViewSink(ctx, token, id) + if err != nil{ + return err + } + if err := es.svc.DeleteSink(ctx, token, id); err != nil { return err } event := deleteSinkEvent{ - id: id, + sinkID: id, + ownerID: sink.MFOwnerID, } record := &redis.XAddArgs{ diff --git a/sinks/sinks.go b/sinks/sinks.go index 97af95e63..1c1ea8276 100644 --- a/sinks/sinks.go +++ b/sinks/sinks.go @@ -109,7 +109,7 @@ type SinkService interface { // CreateSink creates new data sink CreateSink(ctx context.Context, token string, s Sink) (Sink, error) // UpdateSink by id - UpdateSink(ctx context.Context, token string, s Sink) error + UpdateSink(ctx context.Context, token string, s Sink) (Sink, error) // ListSinks retrieves data about sinks ListSinks(ctx context.Context, token string, pm PageMetadata) (Page, error) // ListBackends retrieves a list of available backends diff --git a/sinks/sinks_service.go b/sinks/sinks_service.go index a707f7fa8..e366feb60 100644 --- a/sinks/sinks_service.go +++ b/sinks/sinks_service.go @@ -43,18 +43,22 @@ func (svc sinkService) CreateSink(ctx context.Context, token string, sink Sink) return sink, nil } -func (svc sinkService) UpdateSink(ctx context.Context, token string, sink Sink) error { +func (svc sinkService) UpdateSink(ctx context.Context, token string, sink Sink) (Sink, error) { skOwnerID, err := svc.identify(token) if err != nil { - return err + return Sink{}, err } if sink.Backend != "" || sink.Error != "" { - return errors.ErrUpdateEntity + return Sink{}, errors.ErrUpdateEntity } sink.MFOwnerID = skOwnerID - return svc.sinkRepo.Update(ctx, sink) + err = svc.sinkRepo.Update(ctx, sink) + if err != nil { + return Sink{}, err + } + return sink, nil } func (svc sinkService) ListBackends(ctx context.Context, token string) ([]string, error) { diff --git a/sinks/sinks_service_test.go b/sinks/sinks_service_test.go index 3305bb07a..55eda073c 100644 --- a/sinks/sinks_service_test.go +++ b/sinks/sinks_service_test.go @@ -145,7 +145,7 @@ func TestUpdateSink(t *testing.T) { for desc, tc := range cases { t.Run(desc, func(t *testing.T) { - err := service.UpdateSink(context.Background(), tc.token, tc.sink) + _, err := service.UpdateSink(context.Background(), tc.token, tc.sink) assert.True(t, errors.Contains(err, tc.err), fmt.Sprintf("%s: expected %d got %d", desc, tc.err, err)) }) } diff --git a/ui/src/app/@theme/styles/orb/ngx-datatable/ngx-datatable.scss b/ui/src/app/@theme/styles/orb/ngx-datatable/ngx-datatable.scss index c22e0fd3e..d462c2093 100644 --- a/ui/src/app/@theme/styles/orb/ngx-datatable/ngx-datatable.scss +++ b/ui/src/app/@theme/styles/orb/ngx-datatable/ngx-datatable.scss @@ -17,7 +17,7 @@ $orb-primary-hover: #226fd2; $orb-primary-pressed: #0c5dc5; $orb-primary-focused: #0c5dc5; -.orb-tag-sink { +.orb-tag-chip { font-size: 12px !important; color: $orb-bg1; /*background: $orb-primary; diff --git a/ui/src/app/@theme/styles/themes.scss b/ui/src/app/@theme/styles/themes.scss index b26baa7f2..10e37a51e 100644 --- a/ui/src/app/@theme/styles/themes.scss +++ b/ui/src/app/@theme/styles/themes.scss @@ -7,7 +7,7 @@ @import './material/material-light'; $nb-themes: nb-register-theme(( - layout-padding-top: 2.25rem, + layout-padding-top: 1rem, menu-item-icon-margin: 0 0.5rem 0 0, card-height-tiny: 13.5rem, card-height-small: 21.1875rem, @@ -25,7 +25,7 @@ $nb-themes: nb-register-theme(( ), default, default); $nb-themes: nb-register-theme(( - layout-padding-top: 2.25rem, + layout-padding-top: 1rem, menu-item-icon-margin: 0 0.5rem 0 0, card-height-tiny: 13.5rem, card-height-small: 21.1875rem, @@ -42,7 +42,7 @@ $nb-themes: nb-register-theme(( ), cosmic, cosmic); $nb-themes: nb-register-theme(( - layout-padding-top: 2.25rem, + layout-padding-top: 1rem, menu-item-icon-margin: 0 0.5rem 0 0, card-height-tiny: 13.5rem, card-height-small: 21.1875rem, @@ -59,7 +59,7 @@ $nb-themes: nb-register-theme(( ), corporate, corporate); $nb-themes: nb-register-theme(( - layout-padding-top: 2.25rem, + layout-padding-top: 1rem, menu-item-icon-margin: 0 0.5rem 0 0, card-height-tiny: 13.5rem, card-height-small: 21.1875rem, @@ -76,7 +76,7 @@ $nb-themes: nb-register-theme(( ), dark, dark); $nb-themes: nb-register-theme(( - layout-padding-top: 2.25rem, + layout-padding-top: 1rem, menu-item-icon-margin: 0 0.5rem 0 0, card-height-tiny: 13.5rem, card-height-small: 21.1875rem, @@ -94,7 +94,7 @@ $nb-themes: nb-register-theme(( ), material-light, material-light); $nb-themes: nb-register-theme(( - layout-padding-top: 2.25rem, + layout-padding-top: 1rem, menu-item-icon-margin: 0 0.5rem 0 0, card-height-tiny: 13.5rem, card-height-small: 21.1875rem, @@ -115,7 +115,7 @@ $nb-themes: nb-register-theme(( // outter layout layout-padding-left: 0, layout-padding-right: 0, - layout-padding-top: 1.75rem, + layout-padding-top: 1rem !important, // nav menu scrollable-padding: 10px 0 12px 0, //menu-card-height-tiny: 13.5rem, @@ -152,6 +152,7 @@ $nb-themes: nb-register-theme(( slide-out-shadow-color: 0 4px 14px 0 #292929, slide-out-shadow-color-rtl: 0 4px 14px 0 #292929, // Stepper + stepper-step-margin-left: 1rem, stepper-step-index-border-color: #969fb9, stepper-step-index-active-border-color: #df316f, stepper-step-active-text-color: #df316f, diff --git a/ui/src/app/app.module.ts b/ui/src/app/app.module.ts index 81702e732..a9bfa1c0a 100644 --- a/ui/src/app/app.module.ts +++ b/ui/src/app/app.module.ts @@ -38,8 +38,6 @@ import { BreadcrumbModule } from 'xng-breadcrumb'; import { NgxDatatableModule } from '@swimlane/ngx-datatable'; import { ProfileComponent } from 'app/pages/profile/profile.component'; import { GoogleAnalyticsService } from './common/services/analytics/google-service-analytics.service'; -// Pactsafe -import { PSModule } from '@pactsafe/pactsafe-angular-sdk'; export const MQTT_SERVICE_OPTIONS: IMqttServiceOptions = { connectOnCreate: false, @@ -52,7 +50,6 @@ export const MQTT_SERVICE_OPTIONS: IMqttServiceOptions = { ProfileComponent, ], imports: [ - PSModule.forRoot(), BrowserModule, BrowserAnimationsModule, HttpClientModule, diff --git a/ui/src/app/common/interfaces/mainflux.interface.ts b/ui/src/app/common/interfaces/mainflux.interface.ts index 5b5ab593d..e87faeb33 100644 --- a/ui/src/app/common/interfaces/mainflux.interface.ts +++ b/ui/src/app/common/interfaces/mainflux.interface.ts @@ -1,146 +1,223 @@ export interface UserGroup { id?: string; + name?: string; + description?: string; + metadata?: Object; + owner_id?: string; + parent_id?: string; + users?: User[]; } export interface User { id?: string; + email?: string; + password?: string; + picture?: string; + metadata?: Object; } export interface Channel { id?: string; + name?: string; + metadata?: any; + type?: string; } export interface Thing { id?: string; + key?: string; + name?: string; + metadata?: any; + type?: string; } export interface Attribute { name: string; + channel: string; + subtopic?: string; + persist_state: boolean; } export interface Definition { id: string; + created: Date; + attributes: Attribute[]; + delta: number; } export interface Twin { name?: string; + owner?: string; + id?: string; + revision?: number; + created?: Date; + updated?: Date; + definitions?: Definition[]; + definition?: any; // for request metadata?: any; } export interface SenMLRec { bn: string; + bt: number; + bu: string; + bver: number; + n: string; + t: number; + u: string; + v: number; + vd: number; + vb: number; + vs: string; } export interface MainfluxMsg { name: string; + time: number; + unit: string; + value: number; + string_value: string; + bool_value: boolean; + data_value: string; + sum: number; + protocol: string; } export interface MsgFilters { offset?: number; + limit?: number; + publisher?: string; + subtopic?: string; + name?: string; + v?: string; + vs?: string; + vd?: string; + vb?: string; + from?: number; + to?: number; } export interface ReaderUrl { prefix?: string; + suffix?: string; } export interface Message { value: number; + time: number; } export interface Dataset { label?: string; + messages?: Message[]; } // Mainflux::ORB export interface PageFilters { limit?: number; + offset?: number; + name?: string; + order?: string; + dir?: string; + // mainflux and other components metadata metadata?: string; + type?: string; + // orb tags?: Object; } export interface DropdownFilterItem { id?: string; + label?: string; + selected?: boolean; + prop?: string; + + filter?: (item: any, filter: any) => boolean; } export interface TableConfig { keys?: string[]; + colNames?: string[]; } export interface TablePage { limit?: number; + offset?: number; + total?: number; + rows?: Object[]; } diff --git a/ui/src/app/common/interfaces/orb/agent.interface.ts b/ui/src/app/common/interfaces/orb/agent.interface.ts index 2d9e9e573..722a73b9e 100644 --- a/ui/src/app/common/interfaces/orb/agent.interface.ts +++ b/ui/src/app/common/interfaces/orb/agent.interface.ts @@ -6,6 +6,28 @@ import { OrbEntity } from 'app/common/interfaces/orb/orb.entity.interface'; +/** + * @enum AgentStates + */ +export enum AgentStates { + new = 'new', + online = 'online', + offline = 'offline', + stale = 'stale', + removed = 'removed', +} + +export interface AgentGroupState { + name?: string; + channel?: string; +} + +export interface AgentPolicyState { + name?: string; + state?: string; + datasets?: string[]; +} + /** * @interface Agent */ @@ -49,7 +71,11 @@ export interface Agent extends OrbEntity { /** * Last Heartbeat Data {{[propName: string]: string}} */ - last_hb_data?: any; + last_hb_data?: any | { + backend_state?: any; + group_state?: {[id: string]: AgentGroupState}; + policy_state?: {[id: string]: AgentPolicyState}; + }; /** * Last Heartbeat timestamp {string} @@ -62,6 +88,16 @@ export interface Agent extends OrbEntity { */ error_state?: boolean; + /** + * MQTT KEY + */ key?: string; + /** + * Combines tags for display in UI + * Internal use + * See + */ + combined_tags?: any; + } diff --git a/ui/src/app/common/interfaces/orb/pagination.interface.ts b/ui/src/app/common/interfaces/orb/pagination.interface.ts index b1caa8032..3286b4674 100644 --- a/ui/src/app/common/interfaces/orb/pagination.interface.ts +++ b/ui/src/app/common/interfaces/orb/pagination.interface.ts @@ -1,19 +1,37 @@ export interface OrbPagination { limit: number; + offset: number; + order: string; + total?: number; + name?: string; + tags?: string; + dir?: 'desc' | 'asc'; + data: T[]; + + next?: any; } export interface NgxDatabalePageInfo { offset?: number; + pageSize?: number; + limit?: number; + count?: number; + name?: string; + tags?: string; + + order?: string; + + dir?: string; } diff --git a/ui/src/app/common/services/agents/agent.groups.service.ts b/ui/src/app/common/services/agents/agent.groups.service.ts index 3fbd6dcce..7dbdb5358 100644 --- a/ui/src/app/common/services/agents/agent.groups.service.ts +++ b/ui/src/app/common/services/agents/agent.groups.service.ts @@ -7,6 +7,7 @@ import { environment } from 'environments/environment'; import { NotificationsService } from 'app/common/services/notifications/notifications.service'; import { NgxDatabalePageInfo, OrbPagination } from 'app/common/interfaces/orb/pagination.interface'; import { AgentGroup } from 'app/common/interfaces/orb/agent.group.interface'; +import { delay, expand, reduce } from 'rxjs/operators'; // default filters const defLimit: number = 20; @@ -101,6 +102,24 @@ export class AgentGroupsService { ); } + getAllAgentGroups() { + const pageInfo = AgentGroupsService.getDefaultPagination(); + pageInfo.limit = 100; + + return this.getAgentGroups(pageInfo) + .pipe( + expand(data => { + return data.next ? this.getAgentGroups(data.next) : Observable.empty(); + }), + delay(250), + reduce>((acc, value) => { + acc.data.splice(value.offset, value.limit, ...value.data); + acc.offset = 0; + return acc; + }, this.cache), + ); + } + getAgentGroups(pageInfo: NgxDatabalePageInfo, isFilter = false) { const offset = pageInfo?.offset || this.cache.offset; const limit = pageInfo?.limit || this.cache.limit; diff --git a/ui/src/app/common/services/agents/agent.policies.service.ts b/ui/src/app/common/services/agents/agent.policies.service.ts index 18185be77..b64ded429 100644 --- a/ui/src/app/common/services/agents/agent.policies.service.ts +++ b/ui/src/app/common/services/agents/agent.policies.service.ts @@ -7,6 +7,7 @@ import { environment } from 'environments/environment'; import { NotificationsService } from 'app/common/services/notifications/notifications.service'; import { NgxDatabalePageInfo, OrbPagination } from 'app/common/interfaces/orb/pagination.interface'; import { AgentPolicy } from 'app/common/interfaces/orb/agent.policy.interface'; +import { delay, expand, reduce } from 'rxjs/operators'; // default filters const defLimit: number = 20; @@ -118,6 +119,24 @@ export class AgentPoliciesService { ); } + getAllAgentPolicies() { + const pageInfo = AgentPoliciesService.getDefaultPagination(); + pageInfo.limit = 100; + + return this.getAgentsPolicies(pageInfo) + .pipe( + expand(data => { + return data.next ? this.getAgentsPolicies(data.next) : Observable.empty(); + }), + delay(250), + reduce>((acc, value) => { + acc.data.splice(value.offset, value.limit, ...value.data); + acc.offset = 0; + return acc; + }, this.cache), + ); + } + getAgentsPolicies(pageInfo: NgxDatabalePageInfo, isFilter = false) { const offset = pageInfo?.offset || this.cache.offset; const limit = pageInfo?.limit || this.cache.limit; diff --git a/ui/src/app/common/services/agents/agents.service.ts b/ui/src/app/common/services/agents/agents.service.ts index 875e381ba..e030fa9d0 100644 --- a/ui/src/app/common/services/agents/agents.service.ts +++ b/ui/src/app/common/services/agents/agents.service.ts @@ -7,12 +7,17 @@ import { environment } from 'environments/environment'; import { NotificationsService } from 'app/common/services/notifications/notifications.service'; import { NgxDatabalePageInfo, OrbPagination } from 'app/common/interfaces/orb/pagination.interface'; import { Agent } from 'app/common/interfaces/orb/agent.interface'; +import { delay, expand, reduce } from 'rxjs/operators'; // default filters const defLimit: number = 20; const defOrder: string = 'name'; const defDir = 'desc'; +export enum AvailableOS { + DOCKER = 'docker', +} + @Injectable() export class AgentsService { paginationCache: any = {}; @@ -157,6 +162,24 @@ export class AgentsService { ); } + getAllAgents() { + const pageInfo = AgentsService.getDefaultPagination(); + pageInfo.limit = 100; + + return this.getAgents(pageInfo) + .pipe( + expand(data => { + return data.next ? this.getAgents(data.next) : Observable.empty(); + }), + delay(250), + reduce>((acc, value) => { + acc.data.splice(value.offset, value.limit, ...value.data); + acc.offset = 0; + return acc; + }, this.cache), + ); + } + getAgents(pageInfo: NgxDatabalePageInfo, isFilter = false) { const { limit, offset, name, tags } = pageInfo || this.cache; let params = new HttpParams() @@ -185,16 +208,28 @@ export class AgentsService { this.paginationCache[offset || 0] = true; // This is the position to insert the new data const start = resp.offset; + const newData = [...this.cache.data]; - newData.splice(start, resp.limit, ...resp.agents); + + newData.splice(start, resp.limit, + // TODO - check rule for combination/concatenation of tags + // should we start to represent multiple values for a key here already or + // simply override value for key, assuming agent_tag is precedent. + ...resp.agents.map(agent => { + agent.combined_tags = {...agent?.orb_tags, ...agent?.agent_tags}; + return agent; + })); + this.cache = { ...this.cache, offset: Math.floor(resp.offset / resp.limit), total: resp.total, data: newData, }; + if (name) this.cache.name = name; if (tags) this.cache.tags = tags; + return this.cache; }, ) diff --git a/ui/src/app/common/services/dataset/dataset.policies.service.ts b/ui/src/app/common/services/dataset/dataset.policies.service.ts index ed51201aa..0f8954f54 100644 --- a/ui/src/app/common/services/dataset/dataset.policies.service.ts +++ b/ui/src/app/common/services/dataset/dataset.policies.service.ts @@ -7,6 +7,7 @@ import { environment } from 'environments/environment'; import { NotificationsService } from 'app/common/services/notifications/notifications.service'; import { NgxDatabalePageInfo, OrbPagination } from 'app/common/interfaces/orb/pagination.interface'; import { Dataset } from 'app/common/interfaces/orb/dataset.policy.interface'; +import { delay, expand, reduce } from 'rxjs/operators'; // default filters const defLimit: number = 20; @@ -51,7 +52,7 @@ export class DatasetPoliciesService { addDataset(datasetItem: Dataset) { return this.http.post(environment.datasetPoliciesUrl, - { ...datasetItem}, + { ...datasetItem }, { observe: 'response' }) .map( resp => { @@ -115,6 +116,24 @@ export class DatasetPoliciesService { ); } + getAllDatasets() { + const pageInfo = DatasetPoliciesService.getDefaultPagination(); + pageInfo.limit = 100; + + return this.getDatasetPolicies(pageInfo) + .pipe( + expand(data => { + return data.next ? this.getDatasetPolicies(data.next) : Observable.empty(); + }), + delay(250), + reduce>((acc, value) => { + acc.data.splice(value.offset, value.limit, ...value.data); + acc.offset = 0; + return acc; + }, this.cache), + ); + } + getDatasetPolicies(pageInfo: NgxDatabalePageInfo, isFilter = false) { const offset = pageInfo?.offset || this.cache.offset; const limit = pageInfo?.limit || this.cache.limit; diff --git a/ui/src/app/common/services/sinks/sinks.service.ts b/ui/src/app/common/services/sinks/sinks.service.ts index b1e955f9c..d18581c93 100644 --- a/ui/src/app/common/services/sinks/sinks.service.ts +++ b/ui/src/app/common/services/sinks/sinks.service.ts @@ -7,9 +7,10 @@ import { environment } from 'environments/environment'; import { Sink } from 'app/common/interfaces/orb/sink.interface'; import { NotificationsService } from 'app/common/services/notifications/notifications.service'; import { NgxDatabalePageInfo, OrbPagination } from 'app/common/interfaces/orb/pagination.interface'; +import { delay, expand, reduce } from 'rxjs/operators'; // default filters -const defLimit: number = 20; +const defLimit: number = 10; const defOrder: string = 'name'; const defDir = 'desc'; @@ -96,45 +97,89 @@ export class SinksService { ); } + getAllSinks() { + const pageInfo = SinksService.getDefaultPagination(); + pageInfo.limit = 100; + + return this.getSinks(pageInfo) + .pipe( + expand(data => { + return data.next ? this.getSinks(data.next) : Observable.empty(); + }), + delay(250), + reduce>((acc, value) => { + acc.data.splice(value.offset, value.limit, ...value.data); + acc.offset = 0; + return acc; + }, this.cache), + ); + } + getSinks(pageInfo: NgxDatabalePageInfo, isFilter = false) { - const offset = pageInfo?.offset || this.cache.offset; - const limit = pageInfo?.limit || this.cache.limit; - let params = new HttpParams() - .set('offset', (offset * limit).toString()) - .set('limit', limit.toString()) - .set('order', this.cache.order) - .set('dir', this.cache.dir); + let limit = pageInfo?.limit || this.cache.limit; + let order = pageInfo?.order || this.cache.order; + let dir = pageInfo?.dir || this.cache.dir; + let offset = pageInfo?.offset || 0; + let doClean = false; + let params = new HttpParams(); if (isFilter) { if (pageInfo?.name) { - params = params.append('name', pageInfo.name); - } - if (pageInfo?.tags) { - params.append('tags', JSON.stringify(pageInfo.tags)); + params = params.set('name', pageInfo.name); + // is filter different than last filter? + doClean = !this.paginationCache?.name || this.paginationCache?.name !== pageInfo.name; } - this.paginationCache[offset] = false; + // was filtered, no longer + } else if (this.paginationCache?.isFilter === true) { + doClean = true; } - if (this.paginationCache[pageInfo?.offset]) { + if (pageInfo.order !== this.cache.order + || pageInfo.dir !== this.cache.dir) { + doClean = true; + } + + if (doClean) { + this.clean(); + offset = 0; + limit = this.cache.limit = pageInfo.limit; + dir = pageInfo.dir; + order = pageInfo.order; + } + + if (this.paginationCache[offset]) { return of(this.cache); } + params = params + .set('offset', (offset).toString()) + .set('limit', limit.toString()) + .set('order', order) + .set('dir', dir); return this.http.get(environment.sinksUrl, { params }) .map( (resp: any) => { - this.paginationCache[pageInfo?.offset || 0] = true; + this.paginationCache[pageInfo?.offset / pageInfo?.limit || 0] = true; // This is the position to insert the new data - const start = pageInfo?.offset || 0; + const start = pageInfo?.offset * pageInfo?.limit || 0; const newData = [...this.cache.data]; newData.splice(start, resp.limit, ...resp.sinks); this.cache = { ...this.cache, - offset: Math.floor(resp.offset / resp.limit), + next: resp.offset + resp.limit < resp.total && { + limit: resp.limit, + offset: (parseInt(resp.offset, 10) + parseInt(resp.limit, 10)).toString(), + order: 'name', + dir: 'desc', + }, + offset: resp.offset, + dir: resp.direction, + order: resp.order, total: resp.total, data: newData, + name: pageInfo?.name, }; - if (pageInfo?.name) this.cache.name = pageInfo.name; - if (pageInfo?.tags) this.cache.tags = pageInfo.tags; + return this.cache; }, ) diff --git a/ui/src/app/pages/datasets/add/dataset.add.component.html b/ui/src/app/pages/datasets/add/dataset.add.component.html index 6fdccf363..2f46b2d86 100644 --- a/ui/src/app/pages/datasets/add/dataset.add.component.html +++ b/ui/src/app/pages/datasets/add/dataset.add.component.html @@ -11,7 +11,7 @@

{{isEdit ? 'Edit Dataset' : 'New Dataset'}}

- forkJoin({ agentGroup: this.agentGroupsService.getAgentGroupById(dataset.agent_group_id), agentPolicy: this.agentPoliciesService.getAgentPolicyById(dataset.agent_policy_id), - sinks: this.sinksService.getSinks(null), + sinks: this.sinksService.getAllSinks(), })), ) .subscribe(result => { diff --git a/ui/src/app/pages/datasets/list/dataset.list.component.html b/ui/src/app/pages/datasets/list/dataset.list.component.html index f01baeead..33b4f64e5 100644 --- a/ui/src/app/pages/datasets/list/dataset.list.component.html +++ b/ui/src/app/pages/datasets/list/dataset.list.component.html @@ -4,7 +4,7 @@

All Datasets

-
+

All Datasets

- {{ conf.label }} + placeholder="Filter by" + size="medium" + style="width: 160px; height: 100%"> + {{ conf.label }}
- - - + + - - - - + nbInput + type="text"/>
- + nbButton + status="primary"> +  New Set +
+ [sorts]="tableSorts" + class="orb" + style="height: calc(62vh)">
- + +
+ {{ row.name }} +
+ + + +
+ + {{ row.valid ? 'True' : 'False' }} +
+ + + +
+ {{ value?.length }} +
+ +
- - -
diff --git a/ui/src/app/pages/datasets/list/dataset.list.component.scss b/ui/src/app/pages/datasets/list/dataset.list.component.scss index 0559c8577..6f6794c47 100644 --- a/ui/src/app/pages/datasets/list/dataset.list.component.scss +++ b/ui/src/app/pages/datasets/list/dataset.list.component.scss @@ -1,18 +1,18 @@ h4 { font-family: 'Montserrat', sans-serif; + font-size: 24px; font-style: normal; font-weight: normal; - font-size: 24px; line-height: 32px; } .dataset-summary { .dataset-info-regular { - color: #ffffff; + color: #fff; &::before { - content: 'You have\00a0'; color: #969fb9; + content: 'You have\00a0'; } } @@ -27,19 +27,19 @@ h4 { .filter-dropdown { display: flex; + padding-right: 50px; position: absolute; right: 0; - padding-right: 50px; transform: translateY(-50px); } .button-new-dataset { - width: 155px; - height: 236px; - border: 1px solid #969fb9; background: #232940; + border: 1px solid #969fb9; box-sizing: border-box; + height: 236px; justify-content: flex-end; + width: 155px; &::ng-deep div { display: flex !important; @@ -49,22 +49,22 @@ h4 { &:hover { background: #232940; border: 1px solid #3089fc; - box-sizing: border-box; border-radius: 4px; + box-sizing: border-box; } &:active { background: #232940; border: 1px solid #3089fc; - box-sizing: border-box; border-radius: 4px; + box-sizing: border-box; } &:default { background: #232940; border: 1px solid #969fb9; - box-sizing: border-box; border-radius: 4px; + box-sizing: border-box; } } @@ -76,55 +76,67 @@ tr div p { ::ng-deep { header p { + align-items: center; + display: flex; font-family: 'Montserrat', sans-serif; - font-style: normal; - font-weight: normal; font-size: 14px !important; - line-height: 20px; + font-style: normal; - width: 250px; + font-weight: normal; height: 20px; + line-height: 20px; margin-left: 16px; - margin-top: 24px; - display: flex; - align-items: center; + margin-top: 24px; + width: 250px; } .orb-breadcrumb { display: flex; font-family: 'Montserrat', sans-serif; + font-size: 12px; font-style: normal; font-weight: 500; - font-size: 12px; line-height: 12px; ::ng-deep { .xng-breadcrumb-trail { - color: #ffffff; + color: #fff; margin-bottom: 0; } + .xng-breadcrumb-link { - text-decoration: none !important; color: #969fb9 !important; - } - .xng-breadcrumb-separator { text-decoration: none !important; + } + + .xng-breadcrumb-separator { color: #969fb9 !important; + text-decoration: none !important; } } } } .add-dataset-container { - position: relative; height: 0; + position: relative; button { float: right; - top: 5px; position: relative; - z-index: 2; right: 5px; + top: 5px; + z-index: 2; + } +} + +.orb-service- { + &true { + color: #6fcf97; + } + + &false { + color: #df316f; } } diff --git a/ui/src/app/pages/datasets/list/dataset.list.component.ts b/ui/src/app/pages/datasets/list/dataset.list.component.ts index a0e1d2b5b..fde33f208 100644 --- a/ui/src/app/pages/datasets/list/dataset.list.component.ts +++ b/ui/src/app/pages/datasets/list/dataset.list.component.ts @@ -11,7 +11,6 @@ import { import { DropdownFilterItem } from 'app/common/interfaces/mainflux.interface'; import { ColumnMode, DatatableComponent, TableColumn } from '@swimlane/ngx-datatable'; import { NgxDatabalePageInfo, OrbPagination } from 'app/common/interfaces/orb/pagination.interface'; -import { Debounce } from 'app/shared/decorators/utils'; import { Dataset } from 'app/common/interfaces/orb/dataset.policy.interface'; import { DatasetPoliciesService } from 'app/common/services/dataset/dataset.policies.service'; import { ActivatedRoute, Router } from '@angular/router'; @@ -36,9 +35,13 @@ export class DatasetListComponent implements OnInit, AfterViewInit, AfterViewChe searchPlaceholder = 'Search by name'; - filterSelectedIndex = '0'; - // templates + @ViewChild('nameTemplateCell') nameTemplateCell: TemplateRef; + + @ViewChild('validTemplateCell') validTemplateCell: TemplateRef; + + @ViewChild('sinksTemplateCell') sinksTemplateCell: TemplateRef; + @ViewChild('actionsTemplateCell') actionsTemplateCell: TemplateRef; tableFilters: DropdownFilterItem[] = [ @@ -47,6 +50,25 @@ export class DatasetListComponent implements OnInit, AfterViewInit, AfterViewChe label: 'Name', prop: 'name', selected: false, + filter: (dataset, name) => dataset?.name.includes(name), + }, + { + id: '1', + label: 'Valid', + prop: 'valid', + selected: false, + filter: (dataset, valid) => `${ dataset?.valid }`.includes(name), + }, + ]; + + selectedFilter = this.tableFilters[0]; + + filterValue = null; + + tableSorts = [ + { + prop: 'name', + dir: 'asc', }, ]; @@ -79,7 +101,7 @@ export class DatasetListComponent implements OnInit, AfterViewInit, AfterViewChe ngOnInit() { this.datasetPoliciesService.clean(); - this.getDatasets(); + this.getAllDatasets(); } ngAfterViewInit() { @@ -90,6 +112,23 @@ export class DatasetListComponent implements OnInit, AfterViewInit, AfterViewChe resizeable: false, flexGrow: 5, minWidth: 90, + cellTemplate: this.nameTemplateCell, + }, + { + prop: 'valid', + name: 'Valid', + resizeable: false, + flexGrow: 1, + minWidth: 25, + cellTemplate: this.validTemplateCell, + }, + { + prop: 'sink_ids', + name: 'Sinks', + resizeable: false, + flexGrow: 1, + minWidth: 25, + cellTemplate: this.sinksTemplateCell, }, { name: '', @@ -105,22 +144,25 @@ export class DatasetListComponent implements OnInit, AfterViewInit, AfterViewChe this.cdr.detectChanges(); } + getAllDatasets(): void { + this.datasetPoliciesService.getAllDatasets().subscribe(resp => { + this.paginationControls.data = resp.data; + this.paginationControls.total = resp.data.length; + this.paginationControls.offset = resp.offset / resp.limit; + this.loading = false; + this.cdr.markForCheck(); + }); + } - @Debounce(500) getDatasets(pageInfo: NgxDatabalePageInfo = null): void { - const isFilter = this.paginationControls.name?.length > 0 || this.paginationControls.tags?.length > 0; - - if (isFilter) { - pageInfo = { - offset: this.paginationControls.offset, - limit: this.paginationControls.limit, - }; - if (this.paginationControls.name?.length > 0) pageInfo.name = this.paginationControls.name; - if (this.paginationControls.tags?.length > 0) pageInfo.tags = this.paginationControls.tags; - } + const finalPageInfo = { ...pageInfo }; + finalPageInfo.dir = 'desc'; + finalPageInfo.order = 'name'; + finalPageInfo.limit = this.paginationControls.limit; + finalPageInfo.offset = pageInfo?.offset * pageInfo?.limit || 0; this.loading = true; - this.datasetPoliciesService.getDatasetPolicies(pageInfo, isFilter).subscribe( + this.datasetPoliciesService.getDatasetPolicies(pageInfo).subscribe( (resp: OrbPagination) => { this.paginationControls = resp; this.paginationControls.offset = pageInfo?.offset || 0; @@ -146,8 +188,22 @@ export class DatasetListComponent implements OnInit, AfterViewInit, AfterViewChe ); } - onFilterSelected(selectedIndex) { - this.searchPlaceholder = `Search by ${ this.tableFilters[selectedIndex].label }`; + onFilterSelected(filter) { + this.searchPlaceholder = `Search by ${ filter.label }`; + this.filterValue = null; + } + + applyFilter() { + if (!this.paginationControls || !this.paginationControls?.data) return; + + if (!this.filterValue || this.filterValue === '') { + this.table.rows = this.paginationControls.data; + } else { + this.table.rows = this.paginationControls.data.filter(sink => this.filterValue.split(/[,;]+/gm).reduce((prev, curr) => { + return this.selectedFilter.filter(sink, curr) && prev; + }, true)); + } + this.paginationControls.offset = 0; } openDeleteModal(row: any) { @@ -181,11 +237,4 @@ export class DatasetListComponent implements OnInit, AfterViewInit, AfterViewChe } }); } - - searchDatasetItemByName(input) { - this.getDatasets({ - ...this.paginationControls, - [this.tableFilters[this.filterSelectedIndex].prop]: input, - }); - } } diff --git a/ui/src/app/pages/datasets/policies.agent/add/agent.policy.add.component.html b/ui/src/app/pages/datasets/policies.agent/add/agent.policy.add.component.html index d1f4c0064..ba7cca793 100644 --- a/ui/src/app/pages/datasets/policies.agent/add/agent.policy.add.component.html +++ b/ui/src/app/pages/datasets/policies.agent/add/agent.policy.add.component.html @@ -12,7 +12,7 @@

{{ isEdit ? 'Edit Agent Policy' : 'Create Agent Policy'}}

- + Handler Configuration -
- + * @@ -31,7 +31,7 @@
- + *
-
+
@@ -54,8 +54,8 @@ {{control.value.name}} @@ -104,20 +104,24 @@ - - +
+ + +
diff --git a/ui/src/app/pages/datasets/policies.agent/add/handler.policy.add.component.scss b/ui/src/app/pages/datasets/policies.agent/add/handler.policy.add.component.scss index 38fb858c9..1c8338412 100644 --- a/ui/src/app/pages/datasets/policies.agent/add/handler.policy.add.component.scss +++ b/ui/src/app/pages/datasets/policies.agent/add/handler.policy.add.component.scss @@ -33,6 +33,7 @@ nb-select { button { float: right; + margin: 0 1rem; } textarea { diff --git a/ui/src/app/pages/datasets/policies.agent/list/agent.policy.list.component.html b/ui/src/app/pages/datasets/policies.agent/list/agent.policy.list.component.html index abd01b96b..cceef2a87 100644 --- a/ui/src/app/pages/datasets/policies.agent/list/agent.policy.list.component.html +++ b/ui/src/app/pages/datasets/policies.agent/list/agent.policy.list.component.html @@ -4,89 +4,92 @@

All Policies

-
-
+
+
- {{ conf.label }} + placeholder="Filter by" + size="medium" + style="width: 160px; height: 100%"> + {{ conf.label }}
- - - + + - - - - + nbInput + type="text"/>
- + nbButton + status="primary"> +  New Policy +
+ [sorts]="tableSorts" + class="orb" + style="height: calc(62vh)">
- + +
+ {{ row.name }} +
+ + + +
+ {{ row.version }} +
+
+ {{ 'N/A' }} +
+ + +
- - -
diff --git a/ui/src/app/pages/datasets/policies.agent/list/agent.policy.list.component.ts b/ui/src/app/pages/datasets/policies.agent/list/agent.policy.list.component.ts index 291d3521d..c54515303 100644 --- a/ui/src/app/pages/datasets/policies.agent/list/agent.policy.list.component.ts +++ b/ui/src/app/pages/datasets/policies.agent/list/agent.policy.list.component.ts @@ -16,7 +16,6 @@ import { NbDialogService } from '@nebular/theme'; import { AgentPoliciesService } from 'app/common/services/agents/agent.policies.service'; import { NotificationsService } from 'app/common/services/notifications/notifications.service'; import { ActivatedRoute, Router } from '@angular/router'; -import { Debounce } from 'app/shared/decorators/utils'; import { AgentPolicyDeleteComponent } from 'app/pages/datasets/policies.agent/delete/agent.policy.delete.component'; import { AgentPolicyDetailsComponent } from 'app/pages/datasets/policies.agent/details/agent.policy.details.component'; import { DatePipe } from '@angular/common'; @@ -39,7 +38,9 @@ export class AgentPolicyListComponent implements OnInit, AfterViewInit, AfterVie searchPlaceholder = 'Search by name'; - filterSelectedIndex = '0'; + @ViewChild('nameTemplateCell') nameTemplateCell: TemplateRef; + + @ViewChild('versionTemplateCell') versionTemplateCell: TemplateRef; @ViewChild('actionsTemplateCell') actionsTemplateCell: TemplateRef; @@ -49,12 +50,32 @@ export class AgentPolicyListComponent implements OnInit, AfterViewInit, AfterVie label: 'Name', prop: 'name', selected: false, + filter: (policy, name) => policy?.name.includes(name), }, { id: '1', + label: 'Description', + prop: 'description', + selected: false, + filter: (policy, description) => policy?.description.includes(description), + }, + { + id: '2', label: 'Version', prop: 'version', selected: false, + filter: (policy, version) => policy?.version.includes(version), + }, + ]; + + selectedFilter = this.tableFilters[0]; + + filterValue = null; + + tableSorts = [ + { + prop: 'name', + dir: 'asc', }, ]; @@ -88,7 +109,7 @@ export class AgentPolicyListComponent implements OnInit, AfterViewInit, AfterVie ngOnInit() { this.agentPoliciesService.clean(); - this.getAgentsPolicies(); + this.getAllPolicies(); } ngAfterViewInit() { @@ -97,35 +118,38 @@ export class AgentPolicyListComponent implements OnInit, AfterViewInit, AfterVie prop: 'name', name: 'Policy Name', resizeable: false, - flexGrow: 3, - minWidth: 90, + canAutoResize: true, + flexGrow: 2, + minWidth: 120, + cellTemplate: this.nameTemplateCell, }, { prop: 'description', name: 'Description', resizeable: false, flexGrow: 4, - minWidth: 180, + minWidth: 120, }, { prop: 'version', name: 'Version', resizeable: false, - flexGrow: 2, - minWidth: 60, + flexGrow: 1, + minWidth: 50, + cellTemplate: this.versionTemplateCell, }, { prop: 'ts_last_modified', - pipe: {transform: (value) => this.datePipe.transform(value, 'MMM d, y, HH:mm:ss z')}, + pipe: { transform: (value) => this.datePipe.transform(value, 'M/d/yy, HH:mm z') }, name: 'Last Modified', - minWidth: 90, + minWidth: 140, flexGrow: 2, resizeable: false, }, { name: '', prop: 'actions', - minWidth: 130, + minWidth: 100, resizeable: false, sortable: false, flexGrow: 2, @@ -136,21 +160,25 @@ export class AgentPolicyListComponent implements OnInit, AfterViewInit, AfterVie this.cdr.detectChanges(); } - @Debounce(500) + getAllPolicies(): void { + this.agentPoliciesService.getAllAgentPolicies().subscribe(resp => { + this.paginationControls.data = resp.data; + this.paginationControls.total = resp.data.length; + this.paginationControls.offset = resp.offset / resp.limit; + this.loading = false; + this.cdr.markForCheck(); + }); + } + getAgentsPolicies(pageInfo: NgxDatabalePageInfo = null): void { - const isFilter = this.paginationControls.name?.length > 0 || this.paginationControls.tags?.length > 0; - - if (isFilter) { - pageInfo = { - offset: this.paginationControls.offset, - limit: this.paginationControls.limit, - }; - if (this.paginationControls.name?.length > 0) pageInfo.name = this.paginationControls.name; - if (this.paginationControls.tags?.length > 0) pageInfo.tags = this.paginationControls.tags; - } + const finalPageInfo = { ...pageInfo }; + finalPageInfo.dir = 'desc'; + finalPageInfo.order = 'name'; + finalPageInfo.limit = this.paginationControls.limit; + finalPageInfo.offset = pageInfo?.offset * pageInfo?.limit || 0; this.loading = true; - this.agentPoliciesService.getAgentsPolicies(pageInfo, isFilter).subscribe( + this.agentPoliciesService.getAgentsPolicies(finalPageInfo).subscribe( (resp: OrbPagination) => { this.paginationControls = resp; this.paginationControls.offset = pageInfo?.offset || 0; @@ -173,8 +201,22 @@ export class AgentPolicyListComponent implements OnInit, AfterViewInit, AfterVie }); } - onFilterSelected(selectedIndex) { - this.searchPlaceholder = `Search by ${ this.tableFilters[selectedIndex].label }`; + onFilterSelected(filter) { + this.searchPlaceholder = `Search by ${ filter.label }`; + this.filterValue = null; + } + + applyFilter() { + if (!this.paginationControls || !this.paginationControls?.data) return; + + if (!this.filterValue || this.filterValue === '') { + this.table.rows = this.paginationControls.data; + } else { + this.table.rows = this.paginationControls.data.filter(sink => this.filterValue.split(/[,;]+/gm).reduce((prev, curr) => { + return this.selectedFilter.filter(sink, curr) && prev; + }, true)); + } + this.paginationControls.offset = 0; } openDeleteModal(row: any) { @@ -208,11 +250,4 @@ export class AgentPolicyListComponent implements OnInit, AfterViewInit, AfterVie } }); } - - searchAgentByName(input) { - this.getAgentsPolicies({ - ...this.paginationControls, - [this.tableFilters[this.filterSelectedIndex].prop]: input, - }); - } } diff --git a/ui/src/app/pages/fleet/agents/add/agent.add.component.html b/ui/src/app/pages/fleet/agents/add/agent.add.component.html index 0485b95f9..fa29a8de8 100644 --- a/ui/src/app/pages/fleet/agents/add/agent.add.component.html +++ b/ui/src/app/pages/fleet/agents/add/agent.add.component.html @@ -8,7 +8,7 @@

{{isEdit ? 'Edit Agent' : 'New Agent'}}

- {{isEdit ? 'Edit Agent' : 'New Agent'}} *ngFor="let tag of selectedTags | keyvalue; index as i;" [attr.data-orb-qa-id]="'orb_tag_' + i" [style.background-color]="tag | tagcolor" - class="orb-tag-sink "> + class="orb-tag-chip "> {{tag | tagchip}} {{isEdit ? 'Edit Agent' : 'New Agent'}} + class="orb-tag-chip "> No tag added @@ -195,13 +195,13 @@

{{isEdit ? 'Edit Agent' : 'New Agent'}}

*ngFor="let tag of selectedTags | keyvalue; index as i;" [attr.data-orb-qa-id]="'review-tag_' + i" [style.background-color]="tag | tagcolor" - class="orb-tag-sink "> + class="orb-tag-chip "> {{tag | tagchip}} + class="orb-tag-chip "> No tag added diff --git a/ui/src/app/pages/fleet/agents/details/agent.details.component.html b/ui/src/app/pages/fleet/agents/details/agent.details.component.html index 13fcc5104..05ef744e5 100644 --- a/ui/src/app/pages/fleet/agents/details/agent.details.component.html +++ b/ui/src/app/pages/fleet/agents/details/agent.details.component.html @@ -34,13 +34,13 @@
{{tag}} {{tag}} diff --git a/ui/src/app/pages/fleet/agents/key/agent.key.component.scss b/ui/src/app/pages/fleet/agents/key/agent.key.component.scss index c2da70894..e44af24e6 100644 --- a/ui/src/app/pages/fleet/agents/key/agent.key.component.scss +++ b/ui/src/app/pages/fleet/agents/key/agent.key.component.scss @@ -37,6 +37,7 @@ nb-card { code { color: #ffffff; + line-height: 2.5 !important; } } } diff --git a/ui/src/app/pages/fleet/agents/key/agent.key.component.ts b/ui/src/app/pages/fleet/agents/key/agent.key.component.ts index 82a62e454..da073426a 100644 --- a/ui/src/app/pages/fleet/agents/key/agent.key.component.ts +++ b/ui/src/app/pages/fleet/agents/key/agent.key.component.ts @@ -45,12 +45,12 @@ export class AgentKeyComponent implements OnInit { -e PKTVISOR_PCAP_IFACE_DEFAULT=mock \\ ns1labs/orb-agent:develop`; - this.command2show = `docker run -d --net=host \n --e ORB_CLOUD_ADDRESS=${ document.location.hostname } \n --e ORB_CLOUD_MQTT_ID=${ this.agent.id } \n --e ORB_CLOUD_MQTT_CHANNEL_ID=${ this.agent.channel_id } \n --e ORB_CLOUD_MQTT_KEY=${ this.agent.key } \n --e PKTVISOR_PCAP_IFACE_DEFAULT=mock \n + this.command2show = `docker run -d --net=host \\ +-e ORB_CLOUD_ADDRESS=${ document.location.hostname } \\ +-e ORB_CLOUD_MQTT_ID=${ this.agent.id } \\ +-e ORB_CLOUD_MQTT_CHANNEL_ID=${ this.agent.channel_id } \\ +-e ORB_CLOUD_MQTT_KEY=${ this.agent.key } \\ +-e PKTVISOR_PCAP_IFACE_DEFAULT=mock \\ ns1labs/orb-agent:develop`; } diff --git a/ui/src/app/pages/fleet/agents/list/agent.list.component.html b/ui/src/app/pages/fleet/agents/list/agent.list.component.html index 6119c0c7b..c362d8695 100644 --- a/ui/src/app/pages/fleet/agents/list/agent.list.component.html +++ b/ui/src/app/pages/fleet/agents/list/agent.list.component.html @@ -13,7 +13,7 @@

All Agents

-   {{paginationControls.data.filter(filterByError).length}} of them have errors. +   {{paginationControls.data.filter(filterByError).length}} of them have errors.

@@ -23,139 +23,115 @@

All Agents

- {{ conf.label }} + placeholder="Filter by" + size="medium" + style="width: 160px; height: 100%"> + {{ conf.label }}
- - - + + - - - - + nbInput + type="text"/>
-
+ #table + [columnMode]="columnMode.flex" + [columns]="columns" + [footerHeight]="50" + [headerHeight]="50" + [limit]="paginationControls.limit" + [loadingIndicator]="loading" + [rowHeight]="50" + [rows]="paginationControls.data" + [scrollbarV]="true" + [sorts]="tableSorts" + class="orb" + style="height: calc(62vh)">
- -
+ +
{{ value }}
- +
-
- - {{ row.state | titlecase }} -
-
- - {{ row.state | titlecase }} -
+ + {{ row.state | titlecase }}
- -
- + +
+ - {{tag}} + *ngFor="let tag of value | keyvalue | slice:0:3" + [style.background-color]="tag | tagcolor" + class="orb-tag-chip "> + {{tag | tagchip}} - {{tag}} - - + class="orb-tag-chip "> No tags were created
- -
+ +
Never
-
- {{ value | date:'short' }} +
+ {{ value | date:'M/d/yy, HH:mm z' }}
- +
- - -
diff --git a/ui/src/app/pages/fleet/agents/list/agent.list.component.scss b/ui/src/app/pages/fleet/agents/list/agent.list.component.scss index a171bb52f..604eb6f47 100644 --- a/ui/src/app/pages/fleet/agents/list/agent.list.component.scss +++ b/ui/src/app/pages/fleet/agents/list/agent.list.component.scss @@ -22,7 +22,7 @@ h4 { } .summary-accent { - color: #969fb9; + color: #969fb9 !important; } .filter-dropdown { diff --git a/ui/src/app/pages/fleet/agents/list/agent.list.component.ts b/ui/src/app/pages/fleet/agents/list/agent.list.component.ts index bfd894490..d557f6156 100644 --- a/ui/src/app/pages/fleet/agents/list/agent.list.component.ts +++ b/ui/src/app/pages/fleet/agents/list/agent.list.component.ts @@ -14,7 +14,6 @@ import { ActivatedRoute, Router } from '@angular/router'; import { STRINGS } from 'assets/text/strings'; import { ColumnMode, DatatableComponent, TableColumn } from '@swimlane/ngx-datatable'; import { NgxDatabalePageInfo, OrbPagination } from 'app/common/interfaces/orb/pagination.interface'; -import { Debounce } from 'app/shared/decorators/utils'; import { Agent } from 'app/common/interfaces/orb/agent.interface'; import { AgentsService } from 'app/common/services/agents/agents.service'; import { AgentDeleteComponent } from 'app/pages/fleet/agents/delete/agent.delete.component'; @@ -40,8 +39,6 @@ export class AgentListComponent implements OnInit, AfterViewInit, AfterViewCheck searchPlaceholder = 'Search by name'; - filterSelectedIndex = '0'; - // templates @ViewChild('agentNameTemplateCell') agentNameTemplateCell: TemplateRef; @@ -59,12 +56,33 @@ export class AgentListComponent implements OnInit, AfterViewInit, AfterViewCheck label: 'Name', prop: 'name', selected: false, + filter: (agent, name) => agent?.name.includes(name), }, { id: '1', label: 'Tags', prop: 'tags', selected: false, + filter: (agent, tag) => Object.entries(agent?.combined_tags) + .filter(([key, value]) => `${key}:${value}`.includes(tag.replace(' ', ''))).length > 0, + }, + { + id: '2', + label: 'Status', + prop: 'state', + selected: false, + filter: (agent, state) => agent?.state.includes(state), + }, + ]; + + selectedFilter = this.tableFilters[0]; + + filterValue = null; + + tableSorts = [ + { + prop: 'name', + dir: 'asc', }, ]; @@ -97,7 +115,7 @@ export class AgentListComponent implements OnInit, AfterViewInit, AfterViewCheck ngOnInit() { this.agentService.clean(); - this.getAgents(); + this.getAllAgents(); } ngAfterViewInit() { @@ -106,7 +124,7 @@ export class AgentListComponent implements OnInit, AfterViewInit, AfterViewCheck prop: 'name', name: 'Name', resizeable: false, - flexGrow: 3, + flexGrow: 2, minWidth: 90, cellTemplate: this.agentNameTemplateCell, }, @@ -119,11 +137,17 @@ export class AgentListComponent implements OnInit, AfterViewInit, AfterViewCheck cellTemplate: this.agentStateTemplateRef, }, { - prop: 'orb_tags', + prop: 'combined_tags', name: 'Tags', - minWidth: 90, + minWidth: 150, flexGrow: 4, cellTemplate: this.agentTagsTemplateCell, + comparator: (a, b) => Object.entries(a) + .map(([key, value]) => `${key}:${value}`) + .join(',') + .localeCompare(Object.entries(b) + .map(([key, value]) => `${key}:${value}`) + .join(',')), }, { prop: 'ts_last_hb', @@ -148,21 +172,25 @@ export class AgentListComponent implements OnInit, AfterViewInit, AfterViewCheck this.cdr.detectChanges(); } - @Debounce(500) + getAllAgents(): void { + this.agentService.getAllAgents().subscribe(resp => { + this.paginationControls.data = resp.data; + this.paginationControls.total = resp.data.length; + this.paginationControls.offset = resp.offset / resp.limit; + this.loading = false; + this.cdr.markForCheck(); + }); + } + getAgents(pageInfo: NgxDatabalePageInfo = null): void { - const isFilter = this.paginationControls.name?.length > 0 || this.paginationControls.tags?.length > 0; - - if (isFilter) { - pageInfo = { - offset: this.paginationControls.offset, - limit: this.paginationControls.limit, - }; - if (this.paginationControls.name?.length > 0) pageInfo.name = this.paginationControls.name; - if (this.paginationControls.tags?.length > 0) pageInfo.tags = this.paginationControls.tags; - } + const finalPageInfo = { ...pageInfo }; + finalPageInfo.dir = 'desc'; + finalPageInfo.order = 'name'; + finalPageInfo.limit = this.paginationControls.limit; + finalPageInfo.offset = pageInfo?.offset * pageInfo?.limit || 0; this.loading = true; - this.agentService.getAgents(pageInfo, isFilter).subscribe( + this.agentService.getAgents(finalPageInfo).subscribe( (resp: OrbPagination) => { this.paginationControls = resp; this.paginationControls.offset = pageInfo?.offset || 0; @@ -192,8 +220,23 @@ export class AgentListComponent implements OnInit, AfterViewInit, AfterViewCheck }); } - onFilterSelected(selectedIndex) { - this.searchPlaceholder = `Search by ${ this.tableFilters[selectedIndex].label }`; + onFilterSelected(filter) { + this.searchPlaceholder = `Search by ${ filter.label }`; + this.filterValue = null; + } + + applyFilter() { + if (!this.paginationControls || !this.paginationControls?.data) return; + + if (!this.filterValue || this.filterValue === '') { + this.table.rows = this.paginationControls.data; + } else { + this.table.rows = this.paginationControls.data + .filter(agent => this.filterValue.split(/[,;]+/gm).reduce((prev, curr) => { + return this.selectedFilter.filter(agent, curr) && prev; + }, true)); + } + this.paginationControls.offset = 0; } openDeleteModal(row: any) { @@ -228,13 +271,6 @@ export class AgentListComponent implements OnInit, AfterViewInit, AfterViewCheck }); } - searchAgentByName(input) { - this.getAgents({ - ...this.paginationControls, - [this.tableFilters[this.filterSelectedIndex].prop]: input, - }); - } - filterByError = (agent) => !!agent && agent?.error_state && agent.error_state; mapRegion = (agent) => !!agent && agent?.orb_tags && !!agent.orb_tags['region'] && agent.orb_tags['region']; diff --git a/ui/src/app/pages/fleet/agents/match/agent.match.component.html b/ui/src/app/pages/fleet/agents/match/agent.match.component.html index 688690e1c..6a948e970 100644 --- a/ui/src/app/pages/fleet/agents/match/agent.match.component.html +++ b/ui/src/app/pages/fleet/agents/match/agent.match.component.html @@ -45,7 +45,7 @@
{{tag | tagchip}} diff --git a/ui/src/app/pages/fleet/agents/match/agent.match.component.ts b/ui/src/app/pages/fleet/agents/match/agent.match.component.ts index 15b1ed3ee..d280294e3 100644 --- a/ui/src/app/pages/fleet/agents/match/agent.match.component.ts +++ b/ui/src/app/pages/fleet/agents/match/agent.match.component.ts @@ -74,6 +74,12 @@ export class AgentMatchComponent implements OnInit, AfterViewInit { minWidth: 100, flexGrow: 2, cellTemplate: this.agentTagsTemplateCell, + comparator: (a, b) => Object.entries(a) + .map(([key, value]) => `${key}:${value}`) + .join(',') + .localeCompare(Object.entries(b) + .map(([key, value]) => `${key}:${value}`) + .join(',')), }, { prop: 'state', diff --git a/ui/src/app/pages/fleet/agents/view/agent.view.component.html b/ui/src/app/pages/fleet/agents/view/agent.view.component.html index 5ea13f9af..56e329c48 100644 --- a/ui/src/app/pages/fleet/agents/view/agent.view.component.html +++ b/ui/src/app/pages/fleet/agents/view/agent.view.component.html @@ -1,71 +1,174 @@ -
- - -

Agent View

+
+
+ + +

Agent View

+
+
+ + {{ agent?.state | ngxCapitalize }}. + + Last activity + + today, at {{ agent?.ts_last_hb | date: 'HH:mm z' }} + + + on {{ agent?.ts_last_hb | date: 'M/d/yy, HH:mm z' }} + + + + This Agent has not been provisioned. + + +
+
-
+
+
+ + Agent Information + +
+
+ +

{{ agent?.name }}

+
+
+
+ +

{{ agent?.agent_metadata?.orb_agent?.version }}

+
+
+
+

+ + {{ agent?.id }} +

+

+ + {{ agent?.channel_id }} +

+
+ +
+ + + {{ tag | tagchip }} + + + {{tag | tagchip}} + + +
+
+ + +
- Agent Information + Capabilities - -

{{ agent?.name }}

- - -

{{ agent?.agent_metadata?.orb_agent?.version }}

- - -

{{ agent?.id }}

- - -

{{ agent?.channel_id }}

+ + + + Agent has not been provisioned. + + +
+ 

+        

+      
+    
 
-        
+    
+      
+        Provisioning Command
+        
+          
+            {{ option | titlecase }}
+          
+        
+        
+      
+      
         
           {{ command2show }}
+          
+          {{ command2show }}
         

       
     
   

-  

-    
-      Tags
-      
-        

-          
-            
-              {{ tag | tagchip }}
-            
-            
-              {{tag | tagchip}}
-            
-          
-        

-      
-    
 
-    
-      Heartbeat
-      
-        
-        
{{ agent?.ts_last_hb | date:'short' }}

-
-        
-        
{{ agent?.state | titlecase }}

-      
-    
+  

+    

+      
+        Active Policies/Datasets
+        
+          Agent has not been provisioned.
+          
+            
+              
+                Policy: 
+                {{ policy.name }}
+                  Status: 
+                {{ agent.last_hb_data.policy_state[policy.id].state }}
+              
+              
+                
+                  
+                    Dataset: "{{ datasets[id].name }}"
+                  
+                
+              
+            
+          
+        
+      
+    

+    

+      
+        Active Groups
+        
+          

+            
+              {{ group.name }}
+              ,
+            
+          

+        
+      
+    

   

 

diff --git a/ui/src/app/pages/fleet/agents/view/agent.view.component.scss b/ui/src/app/pages/fleet/agents/view/agent.view.component.scss
index 96af1d0e8..db13984a0 100644
--- a/ui/src/app/pages/fleet/agents/view/agent.view.component.scss
+++ b/ui/src/app/pages/fleet/agents/view/agent.view.component.scss
@@ -1,30 +1,34 @@
 h4 {
   font-family: 'Montserrat', sans-serif;
+  font-size: 1.5rem;
   font-style: normal;
   font-weight: normal;
-  font-size: 1.5rem;
-  margin-bottom: 1.5rem;
   line-height: 2rem;
+  margin-bottom: 1.5rem;
 }
 
 nb-card {
   border: transparent;
   border-radius: 0.5rem;
-  
+
   nb-card-header {
     background-color: #232940;
-    color: #969fb9;
-    padding: 0.5rem 1rem;
     border-bottom: transparent;
     border-top-left-radius: 0.5rem;
     border-top-right-radius: 0.5rem;
+    color: #969fb9;
+    padding: 0.5rem 1rem;
   }
-  
+
   nb-card-body {
     label {
       color: #969fb9;
     }
 
+    span {
+      text-align: end;
+    }
+
     .active-dataset {
       list-style-type: none;
       margin: 0;
@@ -33,64 +37,101 @@ nb-card {
 
     pre {
       background: #1c2339;
-      padding: 0.75rem;
       border-radius: 0.5rem;
+      color: #fff;
+      font-weight: 400;
+      padding: 0.75rem;
+
       white-space: pre-line;
 
       button {
         background: transparent;
         border: 0 transparent;
         color: #969fb9;
-        position: relative;
-        top: -0.25rem;
         float: right;
+        position: relative;
         right: -0.5rem;
+        top: -0.25rem;
       }
 
       code {
         color: #ffffff;
+        line-height: 2.5 !important;
       }
     }
   }
 }
 
 ::ng-deep {
+
+  .agent-group-link {
+    color: #0c5dc5;
+    text-decoration: underline;
+    text-decoration-color: #0c5dc5;
+  }
+
+  .agent-new {
+    color: #9b51e0 !important;
+  }
+
+  .agent-online, .dataset-valid, .policy-running {
+    color: #6fcf97 !important;
+  }
+
+  .agent-offline, .dataset-invalid, .policy-error {
+    color: #df316f !important;
+  }
+
+  .agent-stale {
+    color: #f2994a !important;
+  }
+
+  .agent-removed {
+    color: #62d9ff !important;
+  }
+
+  .summary-accent {
+    color: #969fb9 !important;
+  }
+
   header p {
+    align-items: center;
+    display: flex;
     font-family: 'Montserrat', sans-serif;
-    font-style: normal;
-    font-weight: normal;
     font-size: 14px !important;
-    line-height: 20px;
+    font-style: normal;
 
-    width: 250px;
+    font-weight: normal;
     height: 20px;
+    line-height: 20px;
     margin-left: 16px;
-    margin-top: 24px;
 
-    display: flex;
-    align-items: center;
+    margin-top: 24px;
+    width: 250px;
   }
 
   .orb-breadcrumb {
     display: flex;
     font-family: 'Montserrat', sans-serif;
+    font-size: 12px;
     font-style: normal;
     font-weight: 500;
-    font-size: 12px;
     line-height: 12px;
 
     ::ng-deep {
       .xng-breadcrumb-trail {
-        color: #ffffff;
+        color: #fff;
         margin-bottom: 0;
       }
+
       .xng-breadcrumb-link {
-        text-decoration: none !important;
         color: #969fb9 !important;
-      }
-     .xng-breadcrumb-separator {
         text-decoration: none !important;
+      }
+
+      .xng-breadcrumb-separator {
         color: #969fb9 !important;
+        text-decoration: none !important;
       }
     }
   }
@@ -98,7 +139,40 @@ nb-card {
 
 ::ng-deep {
   mark {
-    background:#1c2339!important;
-    color:#df316f!important;
+    background: #1c2339 !important;
+    color: #df316f !important;
   }
 }
+
+::ng-deep nb-accordion {
+  border: none !important;
+  border-radius: 8px !important;
+  display: grid;
+  padding: 0;
+  text-subtitle-line-height: 1rem;
+
+  nb-accordion-item {
+    border: none !important;
+    border-radius: 8px !important;
+    padding: 0;
+
+    nb-accordion-item-header {
+      border: none !important;
+      border-radius: 8px !important;
+      padding: 10px 0;
+    }
+
+    nb-accordion-item-body {
+      border: none !important;
+      border-radius: 8px !important;
+      display: grid;
+      padding: 0;
+    }
+  }
+}
+
+nb-list-item {
+  background: #171c30;
+  border: none;
+  border-radius: 8px;
+}
diff --git a/ui/src/app/pages/fleet/agents/view/agent.view.component.ts b/ui/src/app/pages/fleet/agents/view/agent.view.component.ts
index ad7843d75..e259ad667 100644
--- a/ui/src/app/pages/fleet/agents/view/agent.view.component.ts
+++ b/ui/src/app/pages/fleet/agents/view/agent.view.component.ts
@@ -1,68 +1,208 @@
-import { Component } from '@angular/core';
+import { Component, OnDestroy, OnInit } from '@angular/core';
 import { STRINGS } from 'assets/text/strings';
 import { ActivatedRoute, Router } from '@angular/router';
-import { Agent } from 'app/common/interfaces/orb/agent.interface';
-import { AgentsService } from 'app/common/services/agents/agents.service';
+import { Agent, AgentStates } from 'app/common/interfaces/orb/agent.interface';
+import { AgentsService, AvailableOS } from 'app/common/services/agents/agents.service';
+import { defer, forkJoin, Observable, of, Subscription } from 'rxjs';
+import { AgentPoliciesService } from 'app/common/services/agents/agent.policies.service';
+import { DatasetPoliciesService } from 'app/common/services/dataset/dataset.policies.service';
+import { concatMap } from 'rxjs/operators';
+import { AgentPolicy } from 'app/common/interfaces/orb/agent.policy.interface';
+import { AgentGroup } from 'app/common/interfaces/orb/agent.group.interface';
+import { Dataset } from 'app/common/interfaces/orb/dataset.policy.interface';
+import { AgentGroupDetailsComponent } from 'app/pages/fleet/groups/details/agent.group.details.component';
+import { NbDialogService } from '@nebular/theme';
+import { AgentGroupsService } from 'app/common/services/agents/agent.groups.service';
 
 @Component({
   selector: 'ngx-agent-view',
   templateUrl: './agent.view.component.html',
   styleUrls: ['./agent.view.component.scss'],
 })
-export class AgentViewComponent {
+export class AgentViewComponent implements OnInit, OnDestroy {
   strings = STRINGS.agents;
 
+  agentStates = AgentStates;
+
   isLoading: boolean = true;
 
   agent: Agent;
+
+  datasets: {[id: string]: Dataset};
+
+  policies: AgentPolicy[];
+
+  groups: AgentGroup[];
+
   agentID;
 
   command2copy: string;
+
   copyCommandIcon: string;
 
+  availableOS = [AvailableOS.DOCKER];
+
+  selectedOS = AvailableOS.DOCKER;
+
   command2show: string;
 
+  hideCommand: boolean;
+
+  subscription: Subscription;
 
   constructor(
     private agentsService: AgentsService,
+    private datasetService: DatasetPoliciesService,
+    private groupsService: AgentGroupsService,
+    private policiesService: AgentPoliciesService,
+    private dialogService: NbDialogService,
     protected route: ActivatedRoute,
     protected router: Router,
   ) {
-    this.agent = this.router.getCurrentNavigation().extras.state?.agent as Agent || null;
+    this.agent = this.router.getCurrentNavigation()?.extras?.state?.agent as Agent || null;
     this.agentID = this.route.snapshot.paramMap.get('id');
+
+    this.datasets = {};
+    this.groups = [];
+    this.policies = [];
     this.command2copy = '';
     this.command2show = '';
     this.copyCommandIcon = 'clipboard-outline';
 
-    !!this.agentID && this.agentsService.getAgentById(this.agentID).subscribe(resp => {
-      this.agent = resp;
-      this.makeCommand2Copy();
-      this.isLoading = false;
-    });
   }
 
-  toggleIcon (target) {
-  if (target === 'command') {
+  ngOnInit() {
+    this.hideCommand = this.agent?.state !== this.agentStates.new;
+    this.isLoading = true;
+
+    this.subscription = this.loadData()
+      .subscribe({
+        next: resp => {
+          this.agent = resp.agent;
+          this.datasets = resp?.datasets.reduce((acc, dataset) => {
+            acc[dataset.id] = dataset;
+            return acc;
+          }, {});
+          this.policies = resp?.policies;
+          this.groups = resp?.groups;
+        },
+        complete: () => {
+          this.makeCommand2Copy();
+          this.isLoading = false;
+        },
+      });
+  }
+
+  loadData() {
+    return !!this.agentID
+      && this.agentsService
+        // for each AGENT
+        .getAgentById(this.agentID)
+        .pipe(
+          // retrieve policies
+          concatMap(agent => forkJoin({
+              // defer execution until subscription
+              // either has policies to query or not
+              policies: defer(() => !!agent?.last_hb_data?.policy_state
+                // fork all requests and await complete all
+                && forkJoin(Object.keys(agent?.last_hb_data?.policy_state)
+                  // map policy IDs to request
+                  .map(policyId => this.policiesService
+                    .getAgentPolicyById(policyId)))
+                // or no requests at all
+                || of(null)),
+              // defer execution until subscription
+              // and datasets for each policy too
+              datasets: defer(() => !!agent?.last_hb_data?.policy_state
+                // fork all requests and await complete all
+                && forkJoin(Object.values(agent?.last_hb_data?.policy_state)
+                  // summarize all datasets to request
+                  .reduce((acc: Observable[], { datasets }) => {
+                    return acc.concat(datasets
+                      // map each datasetID to request
+                      .map(dataset => this.datasetService
+                        .getDatasetById(dataset)));
+                  }, []) as Observable[])
+                // or no requests at all
+                || of(null)),
+              groups: defer(() => !!agent?.last_hb_data?.group_state
+                // fork all requests and await complete all
+                && forkJoin(Object.keys(agent?.last_hb_data?.group_state)
+                  // summarize all groups to request
+                  .map(groupId => this.groupsService
+                    .getAgentGroupById(groupId)))
+                // or no requests at all
+                // or no requests at all
+                || of(null)),
+            }),
+            // emit once when all emitters emit(), completes,
+            // and take(1) unsubscribes all inner observables at
+            // first emission.
+            (outer, inner) => ({ agent: outer, ...inner })),
+        );
+  }
+
+  toggleIcon(target) {
+    if (target === 'command') {
       this.copyCommandIcon = 'checkmark-outline';
     }
   }
 
+  isToday() {
+    const today = new Date(Date.now());
+    const date = new Date(this?.agent?.ts_last_hb);
+
+    return today.getDay() === date.getDay()
+      && today.getMonth() === date.getMonth()
+      && today.getFullYear() === date.getFullYear();
+
+  }
+
   makeCommand2Copy() {
-    this.command2copy = `docker run -d --net=host \\
+    // TODO: future - store this elsewhere
+    if (this.selectedOS === AvailableOS.DOCKER) {
+      this.command2copy = `docker run -d --net=host \\
 -e ORB_CLOUD_ADDRESS=${ document.location.hostname } \\
 -e ORB_CLOUD_MQTT_ID=${ this.agent.id } \\
 -e ORB_CLOUD_MQTT_CHANNEL_ID=${ this.agent.channel_id } \\
--e ORB_CLOUD_MQTT_KEY=${ this.agent.key } \\
+-e ORB_CLOUD_MQTT_KEY="PASTE_AGENT_KEY" \\
 -e PKTVISOR_PCAP_IFACE_DEFAULT=mock \\
 ns1labs/orb-agent:develop`;
 
-    this.command2show = `docker run -d --net=host \n
--e ORB_CLOUD_ADDRESS=${ document.location.hostname } \n
--e ORB_CLOUD_MQTT_ID=${ this.agent.id } \n
--e ORB_CLOUD_MQTT_CHANNEL_ID=${ this.agent.channel_id } \n
--e ORB_CLOUD_MQTT_KEY=${ this.agent.key } \n
--e PKTVISOR_PCAP_IFACE_DEFAULT=mock \n
-
+      this.command2show = `docker run -d --net=host \\
+-e ORB_CLOUD_ADDRESS=${ document.location.hostname } \\
+-e ORB_CLOUD_MQTT_ID=${ this.agent.id } \\
+-e ORB_CLOUD_MQTT_CHANNEL_ID=${ this.agent.channel_id } \\
+-e ORB_CLOUD_MQTT_KEY={{ AGENT KEY }} \\
+-e PKTVISOR_PCAP_IFACE_DEFAULT=mock \\
 ns1labs/orb-agent:develop`;
+    }
+  }
+
+  toggleProvisioningCommand() {
+    this.hideCommand = !this.hideCommand;
+  }
+
+  ngOnDestroy() {
+    this.subscription?.unsubscribe();
+  }
+
+  showAgentGroupDetail(agentGroup) {
+    this.dialogService.open(AgentGroupDetailsComponent, {
+      context: { agentGroup },
+      autoFocus: true,
+      closeOnEsc: true,
+    }).onClose.subscribe((resp) => {
+      if (resp) {
+        this.onOpenEditAgentGroup(agentGroup);
+      }
+    });
+  }
+
+  onOpenEditAgentGroup(agentGroup: any) {
+    this.router.navigate([`../../../groups/edit/${ agentGroup.id }`], {
+      state: { agentGroup: agentGroup, edit: true },
+      relativeTo: this.route,
+    });
   }
 }
diff --git a/ui/src/app/pages/fleet/groups/add/agent.group.add.component.html b/ui/src/app/pages/fleet/groups/add/agent.group.add.component.html
index ccd71c96b..ae96fb95d 100644
--- a/ui/src/app/pages/fleet/groups/add/agent.group.add.component.html
+++ b/ui/src/app/pages/fleet/groups/add/agent.group.add.component.html
@@ -5,11 +5,11 @@
     
{{strings[isEdit ? 'edit' : 'add']['header']}}

   

   

-    

-      
+    

+      
         
           
@@ -18,19 +18,19 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

               
{{strings.add.step.desc1}}

             

           
-          

+          
             
               

                 

                   
                   *
                 

-                
+                       formControlName="name"
+                       fullWidth="true"
+                       nbInput/>
               

             
             

@@ -38,28 +38,28 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

               

                 
               

-              
+                     nbInput/>
             
             

             

-              
-              
             

@@ -76,19 +76,19 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

             

               
                 
+                  [style.background-color]="tag | tagcolor"
+                  class="orb-tag-chip ">
                   {{tag | tagchip}}
-                  
                 
                 
+                  class="orb-tag-chip ">
                   No tag added
                 
               
@@ -102,15 +102,15 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

                     *
                   

                   

-                    
+                           formControlName="key"
+                           fullWidth="true"
+                           nbInput/>
                   

                 

                 

-                  
+                  
                 

                 

                   

@@ -118,24 +118,24 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

                     *
                   

                   

-                    
+                           formControlName="value"
+                           fullWidth="true"
+                           nbInput/>
                   

                 

                 

-                  
                 

@@ -155,31 +155,31 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

               

             

             

-            

-              
               
-              
             

@@ -191,7 +191,7 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

               {{strings.add.step.title3}}
             

           
-          

+          

             

               

                 
@@ -205,21 +205,21 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

                 
{{firstFormGroup.controls.description.value}}

               

             

+            

           

-          

           

             

               
                 
+                  [style.background-color]="tag | tagcolor"
+                  class="orb-tag-chip ">
                   {{tag | tagchip}}
                 
                 
+                  class="orb-tag-chip ">
                   No tag added
                 
               
@@ -231,9 +231,9 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

               

                 

                   {{strings.match.matchAny}} {{tagMatch.total}} {{strings.match.agents}}
-                  . 
                 

@@ -243,46 +243,47 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

               

             

           

-          

-          

+          

             

-              
+              
               
             

           

           

-          

-            
             
-            
@@ -293,30 +294,31 @@ 
{{strings[isEdit ? 'edit' : 'add']['header']}}

   

 

 
-
+
   

-    

-      
-      {{ row.state | titlecase }}
-    

-    

-      
+    

+      
       {{ row.state | titlecase }}
     

   

 
 
-
-  

+
+  

     
       
+        [style.background-color]="tag | tagcolor"
+        class="orb-tag-chip ">
         {{tag | tagchip}}
       
     
   

 
+
+
+  

+    {{ row.ts_last_hb | date: 'M/d/yy, HH:mm z' }}
+  

+
diff --git a/ui/src/app/pages/fleet/groups/add/agent.group.add.component.scss b/ui/src/app/pages/fleet/groups/add/agent.group.add.component.scss
index 3438510e3..faab454fa 100644
--- a/ui/src/app/pages/fleet/groups/add/agent.group.add.component.scss
+++ b/ui/src/app/pages/fleet/groups/add/agent.group.add.component.scss
@@ -41,6 +41,10 @@ mat-chip nb-icon {
     }
   }
 
+  nb-layout-column {
+    padding: 1rem 1rem 0.75rem !important;
+  }
+
   .step {
     flex-direction: row-reverse !important;
     align-items: start !important;
@@ -186,5 +190,26 @@ tr div p {
 .tag-table {
   position: relative;
   left: 0;
-  width: 900px;
+  min-height: 18vw;
+  height: 25vw;
+  max-height: 30vw;
+  width: 57vw;
+}
+
+.orb-service- {
+  &new {
+    color: #9b51e0;
+  }
+  &online {
+    color: #6fcf97;
+  }
+  &stale {
+    color: #f2994a;
+  }
+  &error {
+    color: #df316f;
+  }
+  &offline {
+    color: #969fb9;
+  }
 }
diff --git a/ui/src/app/pages/fleet/groups/add/agent.group.add.component.ts b/ui/src/app/pages/fleet/groups/add/agent.group.add.component.ts
index 5581eaad9..6284e9966 100644
--- a/ui/src/app/pages/fleet/groups/add/agent.group.add.component.ts
+++ b/ui/src/app/pages/fleet/groups/add/agent.group.add.component.ts
@@ -1,4 +1,4 @@
-import { AfterViewInit, Component, TemplateRef, ViewChild } from '@angular/core';
+import { AfterViewInit, ChangeDetectorRef, Component, OnChanges, OnInit, TemplateRef, ViewChild } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
 import { STRINGS } from 'assets/text/strings';
 import { FormBuilder, FormGroup, Validators } from '@angular/forms';
@@ -8,16 +8,15 @@ import { TagMatch } from 'app/common/interfaces/orb/tag.match.interface';
 import { Agent } from 'app/common/interfaces/orb/agent.interface';
 import { DropdownFilterItem } from 'app/common/interfaces/mainflux.interface';
 import { AgentsService } from 'app/common/services/agents/agents.service';
-import { ColumnMode, TableColumn } from '@swimlane/ngx-datatable';
+import { ColumnMode, DatatableComponent, TableColumn } from '@swimlane/ngx-datatable';
 import { NotificationsService } from 'app/common/services/notifications/notifications.service';
 
-
 @Component({
   selector: 'ngx-agent-group-add-component',
   templateUrl: './agent.group.add.component.html',
   styleUrls: ['./agent.group.add.component.scss'],
 })
-export class AgentGroupAddComponent implements AfterViewInit {
+export class AgentGroupAddComponent implements OnInit, OnChanges, AfterViewInit {
   // page vars
   strings = { ...STRINGS.agentGroups, stepper: STRINGS.stepper };
 
@@ -27,11 +26,16 @@ export class AgentGroupAddComponent implements AfterViewInit {
 
   columns: TableColumn[];
 
+  // table
+  @ViewChild(DatatableComponent) table: DatatableComponent;
+
   // templates
   @ViewChild('agentTagsTemplateCell') agentTagsTemplateCell: TemplateRef;
 
   @ViewChild('agentStateTemplateCell') agentStateTemplateRef: TemplateRef;
 
+  @ViewChild('agentLastHBTemplateCell') agentLastHBTemplateRef: TemplateRef;
+
   tableFilters: DropdownFilterItem[] = [
     {
       id: '0',
@@ -70,6 +74,7 @@ export class AgentGroupAddComponent implements AfterViewInit {
   constructor(
     private agentGroupsService: AgentGroupsService,
     private agentsService: AgentsService,
+    private cdr: ChangeDetectorRef,
     private notificationsService: NotificationsService,
     private router: Router,
     private route: ActivatedRoute,
@@ -85,7 +90,9 @@ export class AgentGroupAddComponent implements AfterViewInit {
 
     this.agentGroupID = this.route.snapshot.paramMap.get('id');
     this.isEdit = !!this.agentGroupID;
+  }
 
+  ngOnInit() {
     this.getAgentGroup()
       .then((agentGroup) => {
         this.agentGroup = agentGroup;
@@ -97,6 +104,12 @@ export class AgentGroupAddComponent implements AfterViewInit {
       .catch(reason => console.warn(`Couldn't retrieve data. Reason: ${ reason }`));
   }
 
+  ngOnChanges() {
+    this.table.rows = this.matchingAgents;
+    this.table.recalculate();
+
+  }
+
   initializeForms() {
     const { name, description } = this.agentGroup;
 
@@ -117,31 +130,46 @@ export class AgentGroupAddComponent implements AfterViewInit {
         prop: 'name',
         name: 'Agent Name',
         resizeable: false,
+        canAutoResize: true,
         flexGrow: 1,
-        minWidth: 90,
+        minWidth: 150,
+        width: 175,
       },
       {
         prop: 'orb_tags',
         name: 'Tags',
         resizeable: false,
-        minWidth: 100,
-        flexGrow: 2,
+        minWidth: 250,
+        width: 350,
+        canAutoResize: true,
+        flexGrow: 10,
         cellTemplate: this.agentTagsTemplateCell,
+        comparator: (a, b) => Object.entries(a)
+          .map(([key, value]) => `${key}:${value}`)
+          .join(',')
+          .localeCompare(Object.entries(b)
+            .map(([key, value]) => `${key}:${value}`)
+            .join(',')),
       },
       {
         prop: 'state',
         name: 'Status',
-        minWidth: 90,
+        minWidth: 100,
+        width: 100,
+        canAutoResize: true,
         flexGrow: 1,
         cellTemplate: this.agentStateTemplateRef,
       },
       {
         name: 'Last Activity',
         prop: 'ts_last_hb',
+        cellTemplate: this.agentLastHBTemplateRef,
         minWidth: 130,
+        width: 140,
         resizeable: false,
+        canAutoResize: true,
         sortable: false,
-        flexGrow: 1,
+        flexGrow: 2,
       },
     ];
   }
@@ -224,6 +252,8 @@ export class AgentGroupAddComponent implements AfterViewInit {
 
       this.tagMatch = summary;
       this.matchingAgents = matches;
+      this.cdr.markForCheck();
+
     }).catch(reason => console.warn(`Couldn't retrieve data. Reason: ${ reason }`));
   }
 
diff --git a/ui/src/app/pages/fleet/groups/details/agent.group.details.component.html b/ui/src/app/pages/fleet/groups/details/agent.group.details.component.html
index c210a5679..535f0f756 100644
--- a/ui/src/app/pages/fleet/groups/details/agent.group.details.component.html
+++ b/ui/src/app/pages/fleet/groups/details/agent.group.details.component.html
@@ -30,7 +30,7 @@
         

           
             
                 {{tag | tagchip}}
diff --git a/ui/src/app/pages/fleet/groups/list/agent.group.list.component.html b/ui/src/app/pages/fleet/groups/list/agent.group.list.component.html
index 5875bfd5a..f84e4a874 100644
--- a/ui/src/app/pages/fleet/groups/list/agent.group.list.component.html
+++ b/ui/src/app/pages/fleet/groups/list/agent.group.list.component.html
@@ -4,7 +4,7 @@
     
     
{{strings.list.header}}

   
-  

+  

     

       

         
{{strings.list.header}}
       

         

           
-            {{ conf.label }}
+            placeholder="Filter by"
+            size="medium"
+            style="width: 160px; height: 100%">
+            {{ conf.label }}
           
         

-        
-          
-          
+          
+          
-        
-        
-          
-          
+                 nbInput
+                 type="text"/>
         
       

     

     

-      
     

     
+      #table
+      [columnMode]="columnMode.flex"
+      [columns]="columns"
+      [footerHeight]="50"
+      [headerHeight]="50"
+      [limit]="paginationControls.limit"
+      [loadingIndicator]="loading"
+      [rowHeight]="50"
+      [rows]="paginationControls.data"
+      [scrollbarV]="true"
+      [sorts]="tableSorts"
+      class="orb"
+      style="height: calc(62vh)">
     
   

 

 
+
+  

+    {{ row.name }}
+  

+
+
 
-  {{ row.matching_agents.total }}
+  
+    {{ row.matching_agents.total }}
+  
 
 
-
-  

-    
+
+  

+    
+      
+        {{tag | tagchip}}
+      
       
-          {{tag | tagchip}}
+        *ngIf="(row?.tags | json) === '{}'"
+        [style.background-color]="'notag' | tagcolor"
+        class="orb-tag-chip ">
+        No tags were created
       
     
   

 
 
-
+
   

-    
-    
-    
   

diff --git a/ui/src/app/pages/fleet/groups/list/agent.group.list.component.ts b/ui/src/app/pages/fleet/groups/list/agent.group.list.component.ts
index 9ea347eb4..e3d96d44b 100644
--- a/ui/src/app/pages/fleet/groups/list/agent.group.list.component.ts
+++ b/ui/src/app/pages/fleet/groups/list/agent.group.list.component.ts
@@ -1,4 +1,12 @@
-import { AfterViewChecked, AfterViewInit, ChangeDetectorRef, Component, OnInit, TemplateRef, ViewChild } from '@angular/core';
+import {
+  AfterViewChecked,
+  AfterViewInit,
+  ChangeDetectorRef,
+  Component,
+  OnInit,
+  TemplateRef,
+  ViewChild,
+} from '@angular/core';
 import { NbDialogService } from '@nebular/theme';
 
 import { DropdownFilterItem } from 'app/common/interfaces/mainflux.interface';
@@ -10,7 +18,6 @@ import { ColumnMode, DatatableComponent, TableColumn } from '@swimlane/ngx-datat
 import { AgentGroupsService } from 'app/common/services/agents/agent.groups.service';
 import { NgxDatabalePageInfo, OrbPagination } from 'app/common/interfaces/orb/pagination.interface';
 import { AgentGroup } from 'app/common/interfaces/orb/agent.group.interface';
-import { Debounce } from 'app/shared/decorators/utils';
 import { AgentMatchComponent } from 'app/pages/fleet/agents/match/agent.match.component';
 import { NotificationsService } from 'app/common/services/notifications/notifications.service';
 
@@ -24,6 +31,7 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
   strings = STRINGS.agentGroups;
 
   columnMode = ColumnMode;
+
   columns: TableColumn[];
 
   loading = false;
@@ -31,11 +39,14 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
   paginationControls: OrbPagination;
 
   searchPlaceholder = 'Search by name';
-  filterSelectedIndex = '0';
 
   // templates
+  @ViewChild('agentGroupNameTemplateCell') agentGroupNameTemplateCell: TemplateRef;
+
   @ViewChild('agentGroupTemplateCell') agentGroupsTemplateCell: TemplateRef;
+
   @ViewChild('agentGroupTagsTemplateCell') agentGroupTagsTemplateCell: TemplateRef;
+
   @ViewChild('actionsTemplateCell') actionsTemplateCell: TemplateRef;
 
   tableFilters: DropdownFilterItem[] = [
@@ -44,15 +55,42 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
       label: 'Name',
       prop: 'name',
       selected: false,
+      filter: (agent, name) => agent?.name.includes(name),
     },
     {
       id: '1',
       label: 'Tags',
       prop: 'tags',
       selected: false,
+      filter: (agent, tag) => Object.entries(agent?.tags)
+        .filter(([key, value]) => `${ key }:${ value }`.includes(tag.replace(' ', ''))).length > 0,
+    },
+    {
+      id: '2',
+      label: 'Description',
+      prop: 'description',
+      selected: false,
+      filter: (agent, description) => agent?.description.includes(description),
+    },
+  ];
+
+  selectedFilter = this.tableFilters[0];
+
+  filterValue = null;
+
+  tableSorts = [
+    {
+      prop: 'name',
+      dir: 'asc',
     },
   ];
 
+  @ViewChild('tableWrapper') tableWrapper;
+
+  @ViewChild(DatatableComponent) table: DatatableComponent;
+
+  private currentComponentWidth;
+
   constructor(
     private cdr: ChangeDetectorRef,
     private dialogService: NbDialogService,
@@ -65,9 +103,6 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
     this.paginationControls = AgentGroupsService.getDefaultPagination();
   }
 
-  @ViewChild('tableWrapper') tableWrapper;
-  @ViewChild(DatatableComponent) table: DatatableComponent;
-  private currentComponentWidth;
   ngAfterViewChecked() {
     if (this.table && this.table.recalculate && (this.tableWrapper.nativeElement.clientWidth !== this.currentComponentWidth)) {
       this.currentComponentWidth = this.tableWrapper.nativeElement.clientWidth;
@@ -79,7 +114,7 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
 
   ngOnInit() {
     this.agentGroupsService.clean();
-    this.getAgentGroups();
+    this.getAllAgentGroups();
   }
 
   ngAfterViewInit() {
@@ -87,9 +122,11 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
       {
         prop: 'name',
         name: 'Name',
+        canAutoResize: true,
         resizeable: false,
-        flexGrow: 1,
+        flexGrow: 2,
         minWidth: 90,
+        cellTemplate: this.agentGroupNameTemplateCell,
       },
       {
         prop: 'description',
@@ -102,22 +139,29 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
         prop: 'matching_agents',
         name: 'Agents',
         resizeable: false,
-        minWidth: 100,
+        minWidth: 25,
         flexGrow: 1,
+        comparator: (a, b) => a.total - b.total,
         cellTemplate: this.agentGroupsTemplateCell,
       },
       {
         prop: 'tags',
         name: 'Tags',
-        minWidth: 90,
+        minWidth: 300,
         flexGrow: 3,
-        cellClass: Object,
+        resizeable: false,
         cellTemplate: this.agentGroupTagsTemplateCell,
+        comparator: (a, b) => Object.entries(a)
+          .map(([key, value]) => `${key}:${value}`)
+          .join(',')
+          .localeCompare(Object.entries(b)
+            .map(([key, value]) => `${key}:${value}`)
+            .join(',')),
       },
       {
         name: '',
         prop: 'actions',
-        minWidth: 130,
+        minWidth: 150,
         resizeable: false,
         sortable: false,
         flexGrow: 1,
@@ -128,21 +172,25 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
     this.cdr.detectChanges();
   }
 
-  @Debounce(500)
+  getAllAgentGroups(): void {
+    this.agentGroupsService.getAllAgentGroups().subscribe(resp => {
+      this.paginationControls.data = resp.data;
+      this.paginationControls.total = resp.data.length;
+      this.paginationControls.offset = resp.offset / resp.limit;
+      this.loading = false;
+      this.cdr.markForCheck();
+    });
+  }
+
   getAgentGroups(pageInfo: NgxDatabalePageInfo = null): void {
-    const isFilter = this.paginationControls.name?.length > 0 || this.paginationControls.tags?.length > 0;
-
-    if (isFilter) {
-      pageInfo = {
-        offset: this.paginationControls.offset,
-        limit: this.paginationControls.limit,
-      };
-      if (this.paginationControls.name?.length > 0) pageInfo.name = this.paginationControls.name;
-      if (this.paginationControls.tags?.length > 0) pageInfo.tags = this.paginationControls.tags;
-    }
+    const finalPageInfo = { ...pageInfo };
+    finalPageInfo.dir = 'desc';
+    finalPageInfo.order = 'name';
+    finalPageInfo.limit = this.paginationControls.limit;
+    finalPageInfo.offset = pageInfo?.offset * pageInfo?.limit || 0;
 
     this.loading = true;
-    this.agentGroupsService.getAgentGroups(pageInfo, isFilter).subscribe(
+    this.agentGroupsService.getAgentGroups(pageInfo).subscribe(
       (resp: OrbPagination) => {
         this.paginationControls = resp;
         this.paginationControls.offset = pageInfo?.offset || 0;
@@ -165,8 +213,22 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
     });
   }
 
-  onFilterSelected(selectedIndex) {
-    this.searchPlaceholder = `Search by ${ this.tableFilters[selectedIndex].label }`;
+  onFilterSelected(filter) {
+    this.searchPlaceholder = `Search by ${ filter.label }`;
+    this.filterValue = null;
+  }
+
+  applyFilter() {
+    if (!this.paginationControls || !this.paginationControls?.data) return;
+
+    if (!this.filterValue || this.filterValue === '') {
+      this.table.rows = this.paginationControls.data;
+    } else {
+      this.table.rows = this.paginationControls.data.filter(sink => this.filterValue.split(/[,;]+/gm).reduce((prev, curr) => {
+        return this.selectedFilter.filter(sink, curr) && prev;
+      }, true));
+    }
+    this.paginationControls.offset = 0;
   }
 
   openDeleteModal(row: any) {
@@ -203,18 +265,11 @@ export class AgentGroupListComponent implements OnInit, AfterViewInit, AfterView
 
   onMatchingAgentsModal(row: any) {
     this.dialogService.open(AgentMatchComponent, {
-      context: {agentGroup: row},
+      context: { agentGroup: row },
       autoFocus: true,
       closeOnEsc: true,
     }).onClose.subscribe(_ => {
       this.getAgentGroups();
     });
   }
-
-  searchAgentByName(input) {
-    this.getAgentGroups({
-      ...this.paginationControls,
-      [this.tableFilters[this.filterSelectedIndex].prop]: input,
-    });
-  }
 }
diff --git a/ui/src/app/pages/register/register.component.html b/ui/src/app/pages/register/register.component.html
index 0a8a62278..3b6899b66 100644
--- a/ui/src/app/pages/register/register.component.html
+++ b/ui/src/app/pages/register/register.component.html
@@ -2,8 +2,8 @@