Merge pull request #5679 from StackStorm/token_ttl

Add purging of old tokens

CHANGELOG.rst

@@ -33,6 +33,9 @@ Added
 * Add ``ST2_USE_DEBUGGER`` env var as alternative to the ``--use-debugger`` cli flag. #5675
   Contributed by @cognifloyd
 
+* Added purging of old tokens. #56791
+  Contributed by Amanda McGuinness (@amanda11 intive)
+
 Changed
 ~~~~~~~
 

conf/st2.conf.sample

@@ -183,6 +183,8 @@ rule_enforcements_ttl = None
 sleep_delay = 2
 # Workflow task execution output objects (generated by action output streaming) older than this value (days) will be automatically deleted. Defaults to None (disabled).
 task_executions_ttl = None
+# Tokens that expired over this value (days) will be automatically deleted. Defaults to None (disabled).
+tokens_ttl = None
 # Trace objects older than this value (days) will be automatically deleted. Defaults to None (disabled).
 traces_ttl = None
 # Trigger instances older than this value (days) will be automatically deleted. Defaults to None (disabled).

st2common/bin/st2-purge-tokens

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import sys
+
+from st2common.cmd.purge_tokens import main
+
+if __name__ == "__main__":
+    sys.exit(main())

st2common/setup.py

@@ -57,6 +57,7 @@
         "bin/st2-purge-trigger-instances",
         "bin/st2-purge-traces",
         "bin/st2-purge-rule-enforcements",
+        "bin/st2-purge-tokens",
         "bin/st2-run-pack-tests",
         "bin/st2ctl",
         "bin/st2-generate-symmetric-crypto-key",

st2common/st2common/cmd/purge_tokens.py

@@ -0,0 +1,81 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+"""
+A utility script that purges trigger instances older than certain
+timestamp.
+
+*** RISK RISK RISK. You will lose data. Run at your own risk. ***
+"""
+
+from __future__ import absolute_import
+
+from datetime import datetime
+
+import six
+import pytz
+from oslo_config import cfg
+
+from st2common import config
+from st2common import log as logging
+from st2common.config import do_register_cli_opts
+from st2common.script_setup import setup as common_setup
+from st2common.script_setup import teardown as common_teardown
+from st2common.constants.exit_codes import SUCCESS_EXIT_CODE
+from st2common.constants.exit_codes import FAILURE_EXIT_CODE
+from st2common.garbage_collection.token import purge_tokens
+
+__all__ = ["main"]
+
+LOG = logging.getLogger(__name__)
+
+
+def _register_cli_opts():
+    cli_opts = [
+        cfg.StrOpt(
+            "timestamp",
+            default=None,
+            help="Will delete tokens whose expiry is older than "
+            + "this UTC timestamp. "
+            + "Example value: 2015-03-13T19:01:27.255542Z",
+        )
+    ]
+    do_register_cli_opts(cli_opts)
+
+
+def main():
+    _register_cli_opts()
+    common_setup(config=config, setup_db=True, register_mq_exchanges=False)
+
+    # Get config values
+    timestamp = cfg.CONF.timestamp
+
+    if not timestamp:
+        LOG.error("Please supply a timestamp for purging models. Aborting.")
+        return 1
+    else:
+        timestamp = datetime.strptime(timestamp, "%Y-%m-%dT%H:%M:%S.%fZ")
+        timestamp = timestamp.replace(tzinfo=pytz.UTC)
+
+    # Purge models.
+    try:
+        purge_tokens(logger=LOG, timestamp=timestamp)
+    except Exception as e:
+        LOG.exception(six.text_type(e))
+        return FAILURE_EXIT_CODE
+    finally:
+        common_teardown()
+
+    return SUCCESS_EXIT_CODE

st2common/st2common/garbage_collection/token.py

@@ -0,0 +1,65 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Module with utility functions for purging old trigger instance objects.
+"""
+
+from __future__ import absolute_import
+
+import six
+from mongoengine.errors import InvalidQueryError
+
+from st2common.persistence.auth import Token
+from st2common.util import isotime
+
+__all__ = ["purge_tokens"]
+
+
+def purge_tokens(logger, timestamp):
+    """
+    :param timestamp: Tokens which expired after this timestamp will be deleted.
+    :type timestamp: ``datetime.datetime
+    """
+    if not timestamp:
+        raise ValueError("Specify a valid timestamp to purge.")
+
+    logger.info(
+        "Purging token instances which expired after timestamp: %s"
+        % timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
+    )
+
+    query_filters = {"expiry__lt": isotime.parse(timestamp)}
+
+    try:
+        deleted_count = Token.delete_by_query(**query_filters)
+    except InvalidQueryError as e:
+        msg = (
+            "Bad query (%s) used to delete token instances: %s"
+            "Please contact support."
+            % (
+                query_filters,
+                six.text_type(e),
+            )
+        )
+        raise InvalidQueryError(msg)
+    except:
+        logger.exception(
+            "Deleting token instances using query_filters %s failed.", query_filters
+        )
+    else:
+        logger.info("Deleted %s token objects" % (deleted_count))
+
+    # Print stats
+    logger.info("All token models expired after timestamp %s were deleted.", timestamp)

st2common/st2common/persistence/auth.py

@@ -87,6 +87,10 @@ def get(cls, value):
 
         return result
 
+    @classmethod
+    def delete_by_query(cls, *args, **query):
+        return cls._get_impl().delete_by_query(*args, **query)
+
 
 class ApiKey(Access):
     impl = MongoDBAccess(ApiKeyDB)

st2common/tests/unit/test_purge_token.py

@@ -0,0 +1,78 @@
+# Copyright 2022 The StackStorm Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import absolute_import
+from datetime import timedelta
+import bson
+
+from st2common import log as logging
+from st2common.garbage_collection.token import purge_tokens
+from st2common.models.db.auth import TokenDB
+from st2common.persistence.auth import Token
+from st2common.util import date as date_utils
+from st2tests.base import CleanDbTestCase
+
+LOG = logging.getLogger(__name__)
+
+
+class TestPurgeToken(CleanDbTestCase):
+    @classmethod
+    def setUpClass(cls):
+        CleanDbTestCase.setUpClass()
+        super(TestPurgeToken, cls).setUpClass()
+
+    def setUp(self):
+        super(TestPurgeToken, self).setUp()
+
+    def test_no_timestamp_doesnt_delete(self):
+        now = date_utils.get_datetime_utc_now()
+        TestPurgeToken._create_save_token(
+            expiry_timestamp=now - timedelta(days=20),
+        )
+
+        self.assertEqual(len(Token.get_all()), 1)
+        expected_msg = "Specify a valid timestamp"
+        self.assertRaisesRegexp(
+            ValueError,
+            expected_msg,
+            purge_tokens,
+            logger=LOG,
+            timestamp=None,
+        )
+        self.assertEqual(len(Token.get_all()), 1)
+
+    def test_purge(self):
+        now = date_utils.get_datetime_utc_now()
+        TestPurgeToken._create_save_token(
+            expiry_timestamp=now - timedelta(days=20),
+        )
+
+        TestPurgeToken._create_save_token(
+            expiry_timestamp=now - timedelta(days=5),
+        )
+
+        self.assertEqual(len(Token.get_all()), 2)
+        purge_tokens(logger=LOG, timestamp=now - timedelta(days=10))
+        self.assertEqual(len(Token.get_all()), 1)
+
+    @staticmethod
+    def _create_save_token(expiry_timestamp=None):
+        created = TokenDB(
+            id=str(bson.ObjectId()),
+            user="pony",
+            token=str(bson.ObjectId()),
+            expiry=expiry_timestamp,
+            metadata={"service": "action-runner"},
+        )
+        return Token.add_or_update(created)

st2reactor/st2reactor/garbage_collector/base.py

@@ -46,6 +46,7 @@
 )
 from st2common.garbage_collection.trigger_instances import purge_trigger_instances
 from st2common.garbage_collection.trace import purge_traces
+from st2common.garbage_collection.token import purge_tokens
 from st2common.garbage_collection.rule_enforcement import purge_rule_enforcements
 
 __all__ = ["GarbageCollectorService"]
@@ -76,6 +77,7 @@ def __init__(
         )
         self._trigger_instances_ttl = cfg.CONF.garbagecollector.trigger_instances_ttl
         self._traces_ttl = cfg.CONF.garbagecollector.traces_ttl
+        self._tokens_ttl = cfg.CONF.garbagecollector.tokens_ttl
         self._rule_enforcements_ttl = cfg.CONF.garbagecollector.rule_enforcements_ttl
         self._purge_inquiries = cfg.CONF.garbagecollector.purge_inquiries
         self._workflow_execution_max_idle = cfg.CONF.workflow_engine.gc_max_idle_sec
@@ -170,6 +172,11 @@ def _validate_ttl_values(self):
                 "Minimum possible TTL for traces_ttl in days is %s" % (MINIMUM_TTL_DAYS)
             )
 
+        if self._tokens_ttl and self._tokens_ttl < MINIMUM_TTL_DAYS:
+            raise ValueError(
+                "Minimum possible TTL for tokens_ttl in days is %s" % (MINIMUM_TTL_DAYS)
+            )
+
         if (
             self._rule_enforcements_ttl
             and self._rule_enforcements_ttl < MINIMUM_TTL_DAYS
@@ -232,6 +239,16 @@ def _perform_garbage_collection(self):
         else:
             LOG.debug(skip_message, obj_type)
 
+        obj_type = "token"
+
+        if self._tokens_ttl and self._tokens_ttl >= MINIMUM_TTL_DAYS:
+
+            LOG.info(proc_message, obj_type)
+            self._purge_tokens()
+            concurrency.sleep(self._sleep_delay)
+        else:
+            LOG.debug(skip_message, obj_type)
+
         obj_type = "rule enforcement"
 
         if (
@@ -457,6 +474,35 @@ def _purge_traces(self):
 
         return True
 
+    def _purge_tokens(self):
+        """
+        Purge token objects which match the criteria defined in the config.
+        """
+        utc_now = get_datetime_utc_now()
+        timestamp = utc_now - datetime.timedelta(days=self._tokens_ttl)
+
+        # Another sanity check to make sure we don't delete new objects
+        if timestamp > (utc_now - datetime.timedelta(days=MINIMUM_TTL_DAYS)):
+            raise ValueError(
+                "Calculated timestamp would violate the minimum TTL constraint"
+            )
+
+        timestamp_str = isotime.format(dt=timestamp)
+        LOG.info("Deleting token objects expired older than: %s" % (timestamp_str))
+
+        if timestamp >= utc_now:
+            raise ValueError(
+                f"Calculated timestamp ({timestamp}) is"
+                f" later than now in UTC ({utc_now})."
+            )
+
+        try:
+            purge_tokens(logger=LOG, timestamp=timestamp)
+        except Exception as e:
+            LOG.exception("Failed to delete token: %s" % (six.text_type(e)))
+
+        return True
+
     def _purge_rule_enforcements(self):
         """
         Purge rule enforcements which match the criteria defined in the config.

st2reactor/st2reactor/garbage_collector/config.py

@@ -101,6 +101,11 @@ def _register_garbage_collector_opts(ignore_errors=False):
             default=None,
             help="Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
+        cfg.IntOpt(
+            "tokens_ttl",
+            default=None,
+            help="Tokens that expired over this value (days) will be automatically deleted. Defaults to None (disabled).",
+        ),
         cfg.IntOpt(
             "traces_ttl",
             default=None,

st2tests/st2tests/config.py

@@ -489,6 +489,11 @@ def _register_garbage_collector_opts():
             default=None,
             help="Rule enforcements older than this value (days) will be automatically deleted. Defaults to None (disabled).",
         ),
+        cfg.IntOpt(
+            "tokens_ttl",
+            default=None,
+            help="Tokens that expired over this value (days) will be automatically deleted. Defaults to None (disabled).",
+        ),
         cfg.IntOpt(
             "traces_ttl",
             default=None,