From f3368e72d7333b835a28e804728f16b646ddd9fd Mon Sep 17 00:00:00 2001 From: "Endi S. Dewata" Date: Wed, 29 Nov 2023 05:33:54 +0700 Subject: [PATCH] Add pki-server -db-index-add/rebuild The code that adds and rebuilds DS database search indexes in SubsystemDBInitCLI has been moved into SubsystemDBIndexAddCLI and SubsystemDBIndexRebuildCLI, respectively, such that they can be used more independently. Similarly, the PKISubsystem.init_database() has been split into add_indexes() and rebuild_indexes(). The pki-server -db-index-add/rebuild commands have been added to execute these operations from command line. The test for installing CA with existing DS has been updated to use the new commands. --- .github/workflows/ca-existing-ds-test.yml | 45 +---- .../org/dogtagpki/server/ca/cli/CADBCLI.java | 2 + base/server/python/pki/server/cli/db.py | 171 ++++++++++++++++++ .../python/pki/server/deployment/__init__.py | 18 +- base/server/python/pki/server/subsystem.py | 28 ++- .../dogtagpki/server/cli/SubsystemDBCLI.java | 1 + .../server/cli/SubsystemDBIndexAddCLI.java | 74 ++++++++ .../server/cli/SubsystemDBIndexCLI.java | 21 +++ .../cli/SubsystemDBIndexRebuildCLI.java | 74 ++++++++ .../server/cli/SubsystemDBInitCLI.java | 7 - 10 files changed, 379 insertions(+), 62 deletions(-) create mode 100644 base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexAddCLI.java create mode 100644 base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexCLI.java create mode 100644 base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexRebuildCLI.java diff --git a/.github/workflows/ca-existing-ds-test.yml b/.github/workflows/ca-existing-ds-test.yml index 8179cf5637b..aa32076dff1 100644 --- a/.github/workflows/ca-existing-ds-test.yml +++ b/.github/workflows/ca-existing-ds-test.yml @@ -221,52 +221,11 @@ jobs: - name: Add CA search indexes run: | - sed \ - -e 's/{database}/userroot/g' \ - base/ca/database/ds/index.ldif \ - | tee index.ldif - docker exec ds ldapadd \ - -H ldap://ds.example.com:3389 \ - -D "cn=Directory Manager" \ - -w Secret.123 \ - -f $SHARED/index.ldif + docker exec pki pki-server ca-db-index-add -v - name: Rebuild CA search indexes run: | - # start rebuild task - sed \ - -e 's/{database}/userroot/g' \ - base/ca/database/ds/indextasks.ldif \ - | tee indextasks.ldif - docker exec ds ldapadd \ - -H ldap://ds.example.com:3389 \ - -D "cn=Directory Manager" \ - -w Secret.123 \ - -f $SHARED/indextasks.ldif - - # wait for task to complete - while true; do - sleep 1 - - docker exec ds ldapsearch \ - -H ldap://ds.example.com:3389 \ - -D "cn=Directory Manager" \ - -w Secret.123 \ - -b "cn=index1160589770, cn=index, cn=tasks, cn=config" \ - -LLL \ - nsTaskExitCode \ - | tee output - - sed -n -e 's/nsTaskExitCode:\s*\(.*\)/\1/p' output > nsTaskExitCode - cat nsTaskExitCode - - if [ -s nsTaskExitCode ]; then - break - fi - done - - echo "0" > expected - diff expected nsTaskExitCode + docker exec pki pki-server ca-db-index-rebuild -v - name: Add CA VLV indexes run: | diff --git a/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java b/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java index 5274fed0f99..56b8d337507 100644 --- a/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java +++ b/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java @@ -21,6 +21,7 @@ import org.dogtagpki.cli.CLI; import org.dogtagpki.server.cli.SubsystemDBAccessCLI; import org.dogtagpki.server.cli.SubsystemDBEmptyCLI; +import org.dogtagpki.server.cli.SubsystemDBIndexCLI; import org.dogtagpki.server.cli.SubsystemDBInfoCLI; import org.dogtagpki.server.cli.SubsystemDBInitCLI; import org.dogtagpki.server.cli.SubsystemDBRemoveCLI; @@ -42,6 +43,7 @@ public CADBCLI(CLI parent) { addModule(new CADBUpgradeCLI(this)); addModule(new SubsystemDBAccessCLI(this)); + addModule(new SubsystemDBIndexCLI(this)); addModule(new SubsystemDBReplicationCLI(this)); addModule(new SubsystemDBVLVCLI(this)); } diff --git a/base/server/python/pki/server/cli/db.py b/base/server/python/pki/server/cli/db.py index 101661bb982..046c2155e31 100644 --- a/base/server/python/pki/server/cli/db.py +++ b/base/server/python/pki/server/cli/db.py @@ -218,6 +218,7 @@ def __init__(self, parent): self.add_module(SubsystemDBUpgradeCLI(self)) self.add_module(SubsystemDBAccessCLI(self)) + self.add_module(SubsystemDBIndexCLI(self)) self.add_module(SubsystemDBVLVCLI(self)) @staticmethod @@ -1045,6 +1046,176 @@ def execute(self, argv): subsystem.revoke_database_access(dn, as_current_user=as_current_user) +class SubsystemDBIndexCLI(pki.cli.CLI): + ''' + {subsystem} index management commands + ''' + + def __init__(self, parent): + super(SubsystemDBIndexCLI, self).__init__( + 'index', + inspect.cleandoc(self.__class__.__doc__).format( + subsystem=parent.parent.name.upper())) + + self.parent = parent + self.add_module(SubsystemDBIndexAddCLI(self)) + self.add_module(SubsystemDBIndexRebuildCLI(self)) + + +class SubsystemDBIndexAddCLI(pki.cli.CLI): + ''' + Add {subsystem} indexes + ''' + + help = '''\ + Usage: pki-server {subsystem}-db-index-add [OPTIONS] + + -i, --instance Instance ID (default: pki-tomcat) + -v, --verbose Run in verbose mode. + --debug Run in debug mode. + --help Show help message. + ''' + + def __init__(self, parent): + super(SubsystemDBIndexAddCLI, self).__init__( + 'add', + inspect.cleandoc(self.__class__.__doc__).format( + subsystem=parent.parent.parent.name.upper())) + + self.parent = parent + + def print_help(self): + print(textwrap.dedent(self.__class__.help).format( + subsystem=self.parent.parent.parent.name)) + + def execute(self, argv): + try: + opts, _ = getopt.gnu_getopt(argv, 'i:v', [ + 'instance=', + 'verbose', 'debug', 'help']) + + except getopt.GetoptError as e: + logger.error(e) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + subsystem_name = self.parent.parent.parent.name + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o in ('-v', '--verbose'): + logging.getLogger().setLevel(logging.INFO) + + elif o == '--debug': + logging.getLogger().setLevel(logging.DEBUG) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + logger.error('Invalid option: %s', o) + self.print_help() + sys.exit(1) + + instance = pki.server.instance.PKIInstance(instance_name) + + if not instance.exists(): + logger.error('Invalid instance: %s', instance_name) + sys.exit(1) + + instance.load() + + subsystem = instance.get_subsystem(subsystem_name) + + if not subsystem: + logger.error('No %s subsystem in instance %s.', + subsystem_name.upper(), instance_name) + sys.exit(1) + + subsystem.add_indexes() + + +class SubsystemDBIndexRebuildCLI(pki.cli.CLI): + ''' + Rebuild {subsystem} indexes + ''' + + help = '''\ + Usage: pki-server {subsystem}-db-index-rebuild [OPTIONS] + + -i, --instance Instance ID (default: pki-tomcat) + -v, --verbose Run in verbose mode. + --debug Run in debug mode. + --help Show help message. + ''' + + def __init__(self, parent): + super(SubsystemDBIndexRebuildCLI, self).__init__( + 'rebuild', + inspect.cleandoc(self.__class__.__doc__).format( + subsystem=parent.parent.parent.name.upper())) + + self.parent = parent + + def print_help(self): + print(textwrap.dedent(self.__class__.help).format( + subsystem=self.parent.parent.parent.name)) + + def execute(self, argv): + try: + opts, _ = getopt.gnu_getopt(argv, 'i:v', [ + 'instance=', + 'verbose', 'debug', 'help']) + + except getopt.GetoptError as e: + logger.error(e) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + subsystem_name = self.parent.parent.parent.name + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o in ('-v', '--verbose'): + logging.getLogger().setLevel(logging.INFO) + + elif o == '--debug': + logging.getLogger().setLevel(logging.DEBUG) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + logger.error('Invalid option: %s', o) + self.print_help() + sys.exit(1) + + instance = pki.server.instance.PKIInstance(instance_name) + + if not instance.exists(): + logger.error('Invalid instance: %s', instance_name) + sys.exit(1) + + instance.load() + + subsystem = instance.get_subsystem(subsystem_name) + + if not subsystem: + logger.error('No %s subsystem in instance %s.', + subsystem_name.upper(), instance_name) + sys.exit(1) + + subsystem.rebuild_indexes() + + class SubsystemDBVLVCLI(pki.cli.CLI): def __init__(self, parent): diff --git a/base/server/python/pki/server/deployment/__init__.py b/base/server/python/pki/server/deployment/__init__.py index 4ddd3b4e729..74648e5b990 100644 --- a/base/server/python/pki/server/deployment/__init__.py +++ b/base/server/python/pki/server/deployment/__init__.py @@ -1531,18 +1531,20 @@ def setup_database(self, subsystem, master_config): create_containers = not config.str2bool(self.mdict['pki_clone']) - # If the database is already replicated but not yet indexed, rebuild the indexes. - - rebuild_indexes = config.str2bool(self.mdict['pki_clone']) and \ - not config.str2bool(self.mdict['pki_clone_setup_replication']) and \ - config.str2bool(self.mdict['pki_clone_reindex_data']) - subsystem.init_database( setup_schema=setup_schema, create_database=create_database, create_base=create_base, - create_containers=create_containers, - rebuild_indexes=rebuild_indexes) + create_containers=create_containers) + + # always create indexes + subsystem.add_indexes() + + # if the database is already replicated but not yet indexed, rebuild the indexes + if config.str2bool(self.mdict['pki_clone']) and \ + not config.str2bool(self.mdict['pki_clone_setup_replication']) and \ + config.str2bool(self.mdict['pki_clone_reindex_data']): + subsystem.rebuild_indexes() if config.str2bool(self.mdict['pki_clone']) and \ config.str2bool(self.mdict['pki_clone_setup_replication']): diff --git a/base/server/python/pki/server/subsystem.py b/base/server/python/pki/server/subsystem.py index 576eb9dc1b2..3c62c91aeca 100644 --- a/base/server/python/pki/server/subsystem.py +++ b/base/server/python/pki/server/subsystem.py @@ -1121,7 +1121,6 @@ def init_database( create_database=False, create_base=False, create_containers=False, - rebuild_indexes=False, as_current_user=False): cmd = [self.name + '-db-init'] @@ -1138,9 +1137,6 @@ def init_database( if create_containers: cmd.append('--create-containers') - if rebuild_indexes: - cmd.append('--rebuild-indexes') - if logger.isEnabledFor(logging.DEBUG): cmd.append('--debug') @@ -1149,6 +1145,30 @@ def init_database( self.run(cmd, as_current_user=as_current_user) + def add_indexes(self): + + cmd = [self.name + '-db-index-add'] + + if logger.isEnabledFor(logging.DEBUG): + cmd.append('--debug') + + elif logger.isEnabledFor(logging.INFO): + cmd.append('--verbose') + + self.run(cmd) + + def rebuild_indexes(self): + + cmd = [self.name + '-db-index-rebuild'] + + if logger.isEnabledFor(logging.DEBUG): + cmd.append('--debug') + + elif logger.isEnabledFor(logging.INFO): + cmd.append('--verbose') + + self.run(cmd) + def empty_database(self, force=False, as_current_user=False): cmd = [self.name + '-db-empty'] diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java index 3ee2805c5be..248ce9c6db6 100644 --- a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java @@ -35,6 +35,7 @@ public SubsystemDBCLI(CLI parent) { addModule(new SubsystemDBUpgradeCLI(this)); addModule(new SubsystemDBAccessCLI(this)); + addModule(new SubsystemDBIndexCLI(this)); addModule(new SubsystemDBReplicationCLI(this)); addModule(new SubsystemDBVLVCLI(this)); } diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexAddCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexAddCLI.java new file mode 100644 index 00000000000..43099cdc5b8 --- /dev/null +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexAddCLI.java @@ -0,0 +1,74 @@ +// +// Copyright Red Hat, Inc. +// +// SPDX-License-Identifier: GPL-2.0-or-later +// +package org.dogtagpki.server.cli; + +import org.apache.commons.cli.CommandLine; +import org.dogtagpki.cli.CLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netscape.cms.servlet.csadmin.LDAPConfigurator; +import com.netscape.cmscore.apps.EngineConfig; +import com.netscape.cmscore.ldapconn.LDAPConfig; +import com.netscape.cmscore.ldapconn.LDAPConnectionConfig; +import com.netscape.cmscore.ldapconn.LdapAuthInfo; +import com.netscape.cmscore.ldapconn.LdapBoundConnection; +import com.netscape.cmscore.ldapconn.LdapConnInfo; +import com.netscape.cmscore.ldapconn.PKISocketConfig; +import com.netscape.cmscore.ldapconn.PKISocketFactory; +import com.netscape.cmsutil.password.PasswordStore; +import com.netscape.cmsutil.password.PasswordStoreConfig; + +/** + * @author Endi S. Dewata + */ +public class SubsystemDBIndexAddCLI extends SubsystemCLI { + + public static final Logger logger = LoggerFactory.getLogger(SubsystemDBIndexAddCLI.class); + + public SubsystemDBIndexAddCLI(CLI parent) { + super("add", "Add " + parent.parent.parent.getName().toUpperCase() + " indexes", parent); + } + + @Override + public void execute(CommandLine cmd) throws Exception { + + initializeTomcatJSS(); + String subsystem = parent.parent.parent.getName(); + EngineConfig cs = getEngineConfig(subsystem); + cs.load(); + + LDAPConfig ldapConfig = cs.getInternalDBConfig(); + String instanceId = cs.getInstanceID(); + + PasswordStoreConfig psc = cs.getPasswordStoreConfig(); + PasswordStore passwordStore = PasswordStore.create(psc); + + LDAPConnectionConfig connConfig = ldapConfig.getConnectionConfig(); + + LdapConnInfo connInfo = new LdapConnInfo(connConfig); + LdapAuthInfo authInfo = getAuthInfo(passwordStore, connInfo, ldapConfig); + + PKISocketConfig socketConfig = cs.getSocketConfig(); + + PKISocketFactory socketFactory = new PKISocketFactory(); + socketFactory.setSecure(connInfo.getSecure()); + if (authInfo.getAuthType() == LdapAuthInfo.LDAP_AUTHTYPE_SSLCLIENTAUTH) { + socketFactory.setClientCertNickname(authInfo.getClientCertNickname()); + } + socketFactory.init(socketConfig); + + LdapBoundConnection conn = new LdapBoundConnection(socketFactory, connInfo, authInfo); + LDAPConfigurator ldapConfigurator = new LDAPConfigurator(conn, ldapConfig, instanceId); + + try { + ldapConfigurator.createIndexes(subsystem); + + } finally { + conn.disconnect(); + } + } +} diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexCLI.java new file mode 100644 index 00000000000..60bfb1ffa1d --- /dev/null +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexCLI.java @@ -0,0 +1,21 @@ +// +// Copyright Red Hat, Inc. +// +// SPDX-License-Identifier: GPL-2.0-or-later +// +package org.dogtagpki.server.cli; + +import org.dogtagpki.cli.CLI; + +/** + * @author Endi S. Dewata + */ +public class SubsystemDBIndexCLI extends CLI { + + public SubsystemDBIndexCLI(CLI parent) { + super("index", parent.parent.name.toUpperCase() + " index management commands", parent); + + addModule(new SubsystemDBIndexAddCLI(this)); + addModule(new SubsystemDBIndexRebuildCLI(this)); + } +} diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexRebuildCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexRebuildCLI.java new file mode 100644 index 00000000000..8203a2ea236 --- /dev/null +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexRebuildCLI.java @@ -0,0 +1,74 @@ +// +// Copyright Red Hat, Inc. +// +// SPDX-License-Identifier: GPL-2.0-or-later +// +package org.dogtagpki.server.cli; + +import org.apache.commons.cli.CommandLine; +import org.dogtagpki.cli.CLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netscape.cms.servlet.csadmin.LDAPConfigurator; +import com.netscape.cmscore.apps.EngineConfig; +import com.netscape.cmscore.ldapconn.LDAPConfig; +import com.netscape.cmscore.ldapconn.LDAPConnectionConfig; +import com.netscape.cmscore.ldapconn.LdapAuthInfo; +import com.netscape.cmscore.ldapconn.LdapBoundConnection; +import com.netscape.cmscore.ldapconn.LdapConnInfo; +import com.netscape.cmscore.ldapconn.PKISocketConfig; +import com.netscape.cmscore.ldapconn.PKISocketFactory; +import com.netscape.cmsutil.password.PasswordStore; +import com.netscape.cmsutil.password.PasswordStoreConfig; + +/** + * @author Endi S. Dewata + */ +public class SubsystemDBIndexRebuildCLI extends SubsystemCLI { + + public static final Logger logger = LoggerFactory.getLogger(SubsystemDBIndexRebuildCLI.class); + + public SubsystemDBIndexRebuildCLI(CLI parent) { + super("rebuild", "Rebuild " + parent.parent.parent.getName().toUpperCase() + " indexes", parent); + } + + @Override + public void execute(CommandLine cmd) throws Exception { + + initializeTomcatJSS(); + String subsystem = parent.parent.parent.getName(); + EngineConfig cs = getEngineConfig(subsystem); + cs.load(); + + LDAPConfig ldapConfig = cs.getInternalDBConfig(); + String instanceId = cs.getInstanceID(); + + PasswordStoreConfig psc = cs.getPasswordStoreConfig(); + PasswordStore passwordStore = PasswordStore.create(psc); + + LDAPConnectionConfig connConfig = ldapConfig.getConnectionConfig(); + + LdapConnInfo connInfo = new LdapConnInfo(connConfig); + LdapAuthInfo authInfo = getAuthInfo(passwordStore, connInfo, ldapConfig); + + PKISocketConfig socketConfig = cs.getSocketConfig(); + + PKISocketFactory socketFactory = new PKISocketFactory(); + socketFactory.setSecure(connInfo.getSecure()); + if (authInfo.getAuthType() == LdapAuthInfo.LDAP_AUTHTYPE_SSLCLIENTAUTH) { + socketFactory.setClientCertNickname(authInfo.getClientCertNickname()); + } + socketFactory.init(socketConfig); + + LdapBoundConnection conn = new LdapBoundConnection(socketFactory, connInfo, authInfo); + LDAPConfigurator ldapConfigurator = new LDAPConfigurator(conn, ldapConfig, instanceId); + + try { + ldapConfigurator.rebuildIndexes(subsystem); + + } finally { + conn.disconnect(); + } + } +} diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBInitCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBInitCLI.java index 3589f8379ae..f98009bc2d2 100644 --- a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBInitCLI.java +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBInitCLI.java @@ -50,7 +50,6 @@ public void createOptions() { options.addOption(null, "create-database", false, "Create database"); options.addOption(null, "create-base", false, "Create base entry"); options.addOption(null, "create-containers", false, "Create container entries"); - options.addOption(null, "rebuild-indexes", false, "Rebuild indexes"); options.addOption("v", "verbose", false, "Run in verbose mode."); options.addOption(null, "debug", false, "Run in debug mode."); @@ -115,12 +114,6 @@ public void execute(CommandLine cmd) throws Exception { ldapConfigurator.setupACL(subsystem); } - ldapConfigurator.createIndexes(subsystem); - - if (cmd.hasOption("rebuild-indexes")) { - ldapConfigurator.rebuildIndexes(subsystem); - } - } finally { conn.disconnect(); }