Add pki-server <subsystem>-db-index-add/rebuild

The code that adds and rebuilds DS database search indexes in
SubsystemDBInitCLI has been moved into SubsystemDBIndexAddCLI
and SubsystemDBIndexRebuildCLI, respectively, such that they
can be used more independently.

Similarly, the PKISubsystem.init_database() has been split
into add_indexes() and rebuild_indexes().

The pki-server <subsystem>-db-index-add/rebuilds commands have
been added to execute these operations from command line.

The test for installing CA with existing DS has been updated
to use the new commands.

.github/workflows/ca-existing-ds-test.yml

@@ -221,52 +221,11 @@ jobs:
 
       - name: Add CA search indexes
         run: |
-          sed \
-              -e 's/{database}/userroot/g' \
-              base/ca/database/ds/index.ldif \
-              | tee index.ldif
-          docker exec ds ldapadd \
-              -H ldap://ds.example.com:3389 \
-              -D "cn=Directory Manager" \
-              -w Secret.123 \
-              -f $SHARED/index.ldif
+          docker exec pki pki-server ca-db-index-add -v
 
       - name: Rebuild CA search indexes
         run: |
-          # start rebuild task
-          sed \
-              -e 's/{database}/userroot/g' \
-              base/ca/database/ds/indextasks.ldif \
-              | tee indextasks.ldif
-          docker exec ds ldapadd \
-              -H ldap://ds.example.com:3389 \
-              -D "cn=Directory Manager" \
-              -w Secret.123 \
-              -f $SHARED/indextasks.ldif
-
-          # wait for task to complete
-          while true; do
-              sleep 1
-
-              docker exec ds ldapsearch \
-                  -H ldap://ds.example.com:3389 \
-                  -D "cn=Directory Manager" \
-                  -w Secret.123 \
-                  -b "cn=index1160589770, cn=index, cn=tasks, cn=config" \
-                  -LLL \
-                  nsTaskExitCode \
-                  | tee output
-
-              sed -n -e 's/nsTaskExitCode:\s*\(.*\)/\1/p' output > nsTaskExitCode
-              cat nsTaskExitCode
-
-              if [ -s nsTaskExitCode ]; then
-                  break
-              fi
-          done
-
-          echo "0" > expected
-          diff expected nsTaskExitCode
+          docker exec pki pki-server ca-db-index-rebuild -v
 
       - name: Add CA VLV indexes
         run: |

base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java

@@ -21,6 +21,7 @@
 import org.dogtagpki.cli.CLI;
 import org.dogtagpki.server.cli.SubsystemDBAccessCLI;
 import org.dogtagpki.server.cli.SubsystemDBEmptyCLI;
+import org.dogtagpki.server.cli.SubsystemDBIndexCLI;
 import org.dogtagpki.server.cli.SubsystemDBInfoCLI;
 import org.dogtagpki.server.cli.SubsystemDBInitCLI;
 import org.dogtagpki.server.cli.SubsystemDBRemoveCLI;
@@ -42,6 +43,7 @@ public CADBCLI(CLI parent) {
         addModule(new CADBUpgradeCLI(this));
 
         addModule(new SubsystemDBAccessCLI(this));
+        addModule(new SubsystemDBIndexCLI(this));
         addModule(new SubsystemDBReplicationCLI(this));
         addModule(new SubsystemDBVLVCLI(this));
     }

base/server/python/pki/server/cli/db.py

@@ -218,6 +218,7 @@ def __init__(self, parent):
         self.add_module(SubsystemDBUpgradeCLI(self))
 
         self.add_module(SubsystemDBAccessCLI(self))
+        self.add_module(SubsystemDBIndexCLI(self))
         self.add_module(SubsystemDBVLVCLI(self))
 
     @staticmethod
@@ -1045,6 +1046,176 @@ def execute(self, argv):
         subsystem.revoke_database_access(dn, as_current_user=as_current_user)
 
 
+class SubsystemDBIndexCLI(pki.cli.CLI):
+    '''
+    {subsystem} index management commands
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBIndexCLI, self).__init__(
+            'index',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.name.upper()))
+
+        self.parent = parent
+        self.add_module(SubsystemDBIndexAddCLI(self))
+        self.add_module(SubsystemDBIndexRebuildCLI(self))
+
+
+class SubsystemDBIndexAddCLI(pki.cli.CLI):
+    '''
+    Add {subsystem} indexes
+    '''
+
+    help = '''\
+        Usage: pki-server {subsystem}-db-index-add [OPTIONS]
+
+          -i, --instance <instance ID>       Instance ID (default: pki-tomcat)
+          -v, --verbose                      Run in verbose mode.
+              --debug                        Run in debug mode.
+              --help                         Show help message.
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBIndexAddCLI, self).__init__(
+            'add',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.parent.name.upper()))
+
+        self.parent = parent
+
+    def print_help(self):
+        print(textwrap.dedent(self.__class__.help).format(
+            subsystem=self.parent.parent.parent.name))
+
+    def execute(self, argv):
+        try:
+            opts, _ = getopt.gnu_getopt(argv, 'i:v', [
+                'instance=',
+                'verbose', 'debug', 'help'])
+
+        except getopt.GetoptError as e:
+            logger.error(e)
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        subsystem_name = self.parent.parent.parent.name
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o in ('-v', '--verbose'):
+                logging.getLogger().setLevel(logging.INFO)
+
+            elif o == '--debug':
+                logging.getLogger().setLevel(logging.DEBUG)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                logger.error('Invalid option: %s', o)
+                self.print_help()
+                sys.exit(1)
+
+        instance = pki.server.instance.PKIInstance(instance_name)
+
+        if not instance.exists():
+            logger.error('Invalid instance: %s', instance_name)
+            sys.exit(1)
+
+        instance.load()
+
+        subsystem = instance.get_subsystem(subsystem_name)
+
+        if not subsystem:
+            logger.error('No %s subsystem in instance %s.',
+                         subsystem_name.upper(), instance_name)
+            sys.exit(1)
+
+        subsystem.add_indexes()
+
+
+class SubsystemDBIndexRebuildCLI(pki.cli.CLI):
+    '''
+    Rebuild {subsystem} indexes
+    '''
+
+    help = '''\
+        Usage: pki-server {subsystem}-db-index-rebuild [OPTIONS]
+
+          -i, --instance <instance ID>       Instance ID (default: pki-tomcat)
+          -v, --verbose                      Run in verbose mode.
+              --debug                        Run in debug mode.
+              --help                         Show help message.
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBIndexRebuildCLI, self).__init__(
+            'rebuild',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.parent.name.upper()))
+
+        self.parent = parent
+
+    def print_help(self):
+        print(textwrap.dedent(self.__class__.help).format(
+            subsystem=self.parent.parent.parent.name))
+
+    def execute(self, argv):
+        try:
+            opts, _ = getopt.gnu_getopt(argv, 'i:v', [
+                'instance=',
+                'verbose', 'debug', 'help'])
+
+        except getopt.GetoptError as e:
+            logger.error(e)
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        subsystem_name = self.parent.parent.parent.name
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o in ('-v', '--verbose'):
+                logging.getLogger().setLevel(logging.INFO)
+
+            elif o == '--debug':
+                logging.getLogger().setLevel(logging.DEBUG)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                logger.error('Invalid option: %s', o)
+                self.print_help()
+                sys.exit(1)
+
+        instance = pki.server.instance.PKIInstance(instance_name)
+
+        if not instance.exists():
+            logger.error('Invalid instance: %s', instance_name)
+            sys.exit(1)
+
+        instance.load()
+
+        subsystem = instance.get_subsystem(subsystem_name)
+
+        if not subsystem:
+            logger.error('No %s subsystem in instance %s.',
+                         subsystem_name.upper(), instance_name)
+            sys.exit(1)
+
+        subsystem.rebuild_indexes()
+
+
 class SubsystemDBVLVCLI(pki.cli.CLI):
 
     def __init__(self, parent):

base/server/python/pki/server/deployment/__init__.py

@@ -1531,18 +1531,20 @@ def setup_database(self, subsystem, master_config):
 
         create_containers = not config.str2bool(self.mdict['pki_clone'])
 
-        # If the database is already replicated but not yet indexed, rebuild the indexes.
-
-        rebuild_indexes = config.str2bool(self.mdict['pki_clone']) and \
-            not config.str2bool(self.mdict['pki_clone_setup_replication']) and \
-            config.str2bool(self.mdict['pki_clone_reindex_data'])
-
         subsystem.init_database(
             setup_schema=setup_schema,
             create_database=create_database,
             create_base=create_base,
-            create_containers=create_containers,
-            rebuild_indexes=rebuild_indexes)
+            create_containers=create_containers)
+
+        # always create indexes
+        subsystem.add_indexes()
+
+        # if the database is already replicated but not yet indexed, rebuild the indexes
+        if config.str2bool(self.mdict['pki_clone']) and \
+                not config.str2bool(self.mdict['pki_clone_setup_replication']) and \
+                config.str2bool(self.mdict['pki_clone_reindex_data']):
+            subsystem.rebuild_indexes()
 
         if config.str2bool(self.mdict['pki_clone']) and \
                 config.str2bool(self.mdict['pki_clone_setup_replication']):

base/server/python/pki/server/subsystem.py

@@ -1121,7 +1121,6 @@ def init_database(
             create_database=False,
             create_base=False,
             create_containers=False,
-            rebuild_indexes=False,
             as_current_user=False):
 
         cmd = [self.name + '-db-init']
@@ -1138,9 +1137,6 @@ def init_database(
         if create_containers:
             cmd.append('--create-containers')
 
-        if rebuild_indexes:
-            cmd.append('--rebuild-indexes')
-
         if logger.isEnabledFor(logging.DEBUG):
             cmd.append('--debug')
 
@@ -1149,6 +1145,30 @@ def init_database(
 
         self.run(cmd, as_current_user=as_current_user)
 
+    def add_indexes(self):
+
+        cmd = [self.name + '-db-index-add']
+
+        if logger.isEnabledFor(logging.DEBUG):
+            cmd.append('--debug')
+
+        elif logger.isEnabledFor(logging.INFO):
+            cmd.append('--verbose')
+
+        self.run(cmd)
+
+    def rebuild_indexes(self):
+
+        cmd = [self.name + '-db-index-rebuild']
+
+        if logger.isEnabledFor(logging.DEBUG):
+            cmd.append('--debug')
+
+        elif logger.isEnabledFor(logging.INFO):
+            cmd.append('--verbose')
+
+        self.run(cmd)
+
     def empty_database(self, force=False, as_current_user=False):
 
         cmd = [self.name + '-db-empty']

base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java

@@ -35,6 +35,7 @@ public SubsystemDBCLI(CLI parent) {
         addModule(new SubsystemDBUpgradeCLI(this));
 
         addModule(new SubsystemDBAccessCLI(this));
+        addModule(new SubsystemDBIndexCLI(this));
         addModule(new SubsystemDBReplicationCLI(this));
         addModule(new SubsystemDBVLVCLI(this));
     }