Add pki-server <subsystem>-db-index-add/rebuild

.github/workflows/ca-existing-ds-test.yml

@@ -221,52 +221,11 @@ jobs:
 
       - name: Add CA search indexes
         run: |
-          sed \
-              -e 's/{database}/userroot/g' \
-              base/ca/database/ds/index.ldif \
-              | tee index.ldif
-          docker exec ds ldapadd \
-              -H ldap://ds.example.com:3389 \
-              -D "cn=Directory Manager" \
-              -w Secret.123 \
-              -f $SHARED/index.ldif
+          docker exec pki pki-server ca-db-index-add -v
 
       - name: Rebuild CA search indexes
         run: |
-          # start rebuild task
-          sed \
-              -e 's/{database}/userroot/g' \
-              base/ca/database/ds/indextasks.ldif \
-              | tee indextasks.ldif
-          docker exec ds ldapadd \
-              -H ldap://ds.example.com:3389 \
-              -D "cn=Directory Manager" \
-              -w Secret.123 \
-              -f $SHARED/indextasks.ldif
-
-          # wait for task to complete
-          while true; do
-              sleep 1
-
-              docker exec ds ldapsearch \
-                  -H ldap://ds.example.com:3389 \
-                  -D "cn=Directory Manager" \
-                  -w Secret.123 \
-                  -b "cn=index1160589770, cn=index, cn=tasks, cn=config" \
-                  -LLL \
-                  nsTaskExitCode \
-                  | tee output
-
-              sed -n -e 's/nsTaskExitCode:\s*\(.*\)/\1/p' output > nsTaskExitCode
-              cat nsTaskExitCode
-
-              if [ -s nsTaskExitCode ]; then
-                  break
-              fi
-          done
-
-          echo "0" > expected
-          diff expected nsTaskExitCode
+          docker exec pki pki-server ca-db-index-rebuild -v
 
       - name: Add CA VLV indexes
         run: |

base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java

@@ -21,6 +21,7 @@
 import org.dogtagpki.cli.CLI;
 import org.dogtagpki.server.cli.SubsystemDBAccessCLI;
 import org.dogtagpki.server.cli.SubsystemDBEmptyCLI;
+import org.dogtagpki.server.cli.SubsystemDBIndexCLI;
 import org.dogtagpki.server.cli.SubsystemDBInfoCLI;
 import org.dogtagpki.server.cli.SubsystemDBInitCLI;
 import org.dogtagpki.server.cli.SubsystemDBRemoveCLI;
@@ -42,6 +43,7 @@ public CADBCLI(CLI parent) {
         addModule(new CADBUpgradeCLI(this));
 
         addModule(new SubsystemDBAccessCLI(this));
+        addModule(new SubsystemDBIndexCLI(this));
         addModule(new SubsystemDBReplicationCLI(this));
         addModule(new SubsystemDBVLVCLI(this));
     }

base/server/python/pki/server/cli/db.py

@@ -218,6 +218,7 @@ def __init__(self, parent):
         self.add_module(SubsystemDBUpgradeCLI(self))
 
         self.add_module(SubsystemDBAccessCLI(self))
+        self.add_module(SubsystemDBIndexCLI(self))
         self.add_module(SubsystemDBVLVCLI(self))
 
     @staticmethod
@@ -1045,6 +1046,176 @@ def execute(self, argv):
         subsystem.revoke_database_access(dn, as_current_user=as_current_user)
 
 
+class SubsystemDBIndexCLI(pki.cli.CLI):
+    '''
+    {subsystem} index management commands
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBIndexCLI, self).__init__(
+            'index',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.name.upper()))
+
+        self.parent = parent
+        self.add_module(SubsystemDBIndexAddCLI(self))
+        self.add_module(SubsystemDBIndexRebuildCLI(self))
+
+
+class SubsystemDBIndexAddCLI(pki.cli.CLI):
+    '''
+    Add {subsystem} indexes
+    '''
+
+    help = '''\
+        Usage: pki-server {subsystem}-db-index-add [OPTIONS]
+
+          -i, --instance <instance ID>       Instance ID (default: pki-tomcat)
+          -v, --verbose                      Run in verbose mode.
+              --debug                        Run in debug mode.
+              --help                         Show help message.
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBIndexAddCLI, self).__init__(
+            'add',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.parent.name.upper()))
+
+        self.parent = parent
+
+    def print_help(self):
+        print(textwrap.dedent(self.__class__.help).format(
+            subsystem=self.parent.parent.parent.name))
+
+    def execute(self, argv):
+        try:
+            opts, _ = getopt.gnu_getopt(argv, 'i:v', [
+                'instance=',
+                'verbose', 'debug', 'help'])
+
+        except getopt.GetoptError as e:
+            logger.error(e)
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        subsystem_name = self.parent.parent.parent.name
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o in ('-v', '--verbose'):
+                logging.getLogger().setLevel(logging.INFO)
+
+            elif o == '--debug':
+                logging.getLogger().setLevel(logging.DEBUG)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                logger.error('Invalid option: %s', o)
+                self.print_help()
+                sys.exit(1)
+
+        instance = pki.server.instance.PKIInstance(instance_name)
+
+        if not instance.exists():
+            logger.error('Invalid instance: %s', instance_name)
+            sys.exit(1)
+
+        instance.load()
+
+        subsystem = instance.get_subsystem(subsystem_name)
+
+        if not subsystem:
+            logger.error('No %s subsystem in instance %s.',
+                         subsystem_name.upper(), instance_name)
+            sys.exit(1)
+
+        subsystem.add_indexes()
+
+
+class SubsystemDBIndexRebuildCLI(pki.cli.CLI):
+    '''
+    Rebuild {subsystem} indexes
+    '''
+
+    help = '''\
+        Usage: pki-server {subsystem}-db-index-rebuild [OPTIONS]
+
+          -i, --instance <instance ID>       Instance ID (default: pki-tomcat)
+          -v, --verbose                      Run in verbose mode.
+              --debug                        Run in debug mode.
+              --help                         Show help message.
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBIndexRebuildCLI, self).__init__(
+            'rebuild',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.parent.name.upper()))
+
+        self.parent = parent
+
+    def print_help(self):
+        print(textwrap.dedent(self.__class__.help).format(
+            subsystem=self.parent.parent.parent.name))
+
+    def execute(self, argv):
+        try:
+            opts, _ = getopt.gnu_getopt(argv, 'i:v', [
+                'instance=',
+                'verbose', 'debug', 'help'])
+
+        except getopt.GetoptError as e:
+            logger.error(e)
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        subsystem_name = self.parent.parent.parent.name
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o in ('-v', '--verbose'):
+                logging.getLogger().setLevel(logging.INFO)
+
+            elif o == '--debug':
+                logging.getLogger().setLevel(logging.DEBUG)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                logger.error('Invalid option: %s', o)
+                self.print_help()
+                sys.exit(1)
+
+        instance = pki.server.instance.PKIInstance(instance_name)
+
+        if not instance.exists():
+            logger.error('Invalid instance: %s', instance_name)
+            sys.exit(1)
+
+        instance.load()
+
+        subsystem = instance.get_subsystem(subsystem_name)
+
+        if not subsystem:
+            logger.error('No %s subsystem in instance %s.',
+                         subsystem_name.upper(), instance_name)
+            sys.exit(1)
+
+        subsystem.rebuild_indexes()
+
+
 class SubsystemDBVLVCLI(pki.cli.CLI):
 
     def __init__(self, parent):

base/server/python/pki/server/deployment/__init__.py

@@ -1531,18 +1531,20 @@ def setup_database(self, subsystem, master_config):
 
         create_containers = not config.str2bool(self.mdict['pki_clone'])
 
-        # If the database is already replicated but not yet indexed, rebuild the indexes.
-
-        rebuild_indexes = config.str2bool(self.mdict['pki_clone']) and \
-            not config.str2bool(self.mdict['pki_clone_setup_replication']) and \
-            config.str2bool(self.mdict['pki_clone_reindex_data'])
-
         subsystem.init_database(
             setup_schema=setup_schema,
             create_database=create_database,
             create_base=create_base,
-            create_containers=create_containers,
-            rebuild_indexes=rebuild_indexes)
+            create_containers=create_containers)
+
+        # always create indexes
+        subsystem.add_indexes()
+
+        # if the database is already replicated but not yet indexed, rebuild the indexes
+        if config.str2bool(self.mdict['pki_clone']) and \
+                not config.str2bool(self.mdict['pki_clone_setup_replication']) and \
+                config.str2bool(self.mdict['pki_clone_reindex_data']):
+            subsystem.rebuild_indexes()
 
         if config.str2bool(self.mdict['pki_clone']) and \
                 config.str2bool(self.mdict['pki_clone_setup_replication']):

base/server/python/pki/server/subsystem.py

@@ -1121,7 +1121,6 @@ def init_database(
             create_database=False,
             create_base=False,
             create_containers=False,
-            rebuild_indexes=False,
             as_current_user=False):
 
         cmd = [self.name + '-db-init']
@@ -1138,9 +1137,6 @@ def init_database(
         if create_containers:
             cmd.append('--create-containers')
 
-        if rebuild_indexes:
-            cmd.append('--rebuild-indexes')
-
         if logger.isEnabledFor(logging.DEBUG):
             cmd.append('--debug')
 
@@ -1149,6 +1145,30 @@ def init_database(
 
         self.run(cmd, as_current_user=as_current_user)
 
+    def add_indexes(self):
+
+        cmd = [self.name + '-db-index-add']
+
+        if logger.isEnabledFor(logging.DEBUG):
+            cmd.append('--debug')
+
+        elif logger.isEnabledFor(logging.INFO):
+            cmd.append('--verbose')
+
+        self.run(cmd)
+
+    def rebuild_indexes(self):
+
+        cmd = [self.name + '-db-index-rebuild']
+
+        if logger.isEnabledFor(logging.DEBUG):
+            cmd.append('--debug')
+
+        elif logger.isEnabledFor(logging.INFO):
+            cmd.append('--verbose')
+
+        self.run(cmd)
+
     def empty_database(self, force=False, as_current_user=False):
 
         cmd = [self.name + '-db-empty']

base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java

@@ -35,6 +35,7 @@ public SubsystemDBCLI(CLI parent) {
         addModule(new SubsystemDBUpgradeCLI(this));
 
         addModule(new SubsystemDBAccessCLI(this));
+        addModule(new SubsystemDBIndexCLI(this));
         addModule(new SubsystemDBReplicationCLI(this));
         addModule(new SubsystemDBVLVCLI(this));
     }