diff --git a/.github/workflows/ca-existing-ds-test.yml b/.github/workflows/ca-existing-ds-test.yml index 8179cf5637b..aa32076dff1 100644 --- a/.github/workflows/ca-existing-ds-test.yml +++ b/.github/workflows/ca-existing-ds-test.yml @@ -221,52 +221,11 @@ jobs: - name: Add CA search indexes run: | - sed \ - -e 's/{database}/userroot/g' \ - base/ca/database/ds/index.ldif \ - | tee index.ldif - docker exec ds ldapadd \ - -H ldap://ds.example.com:3389 \ - -D "cn=Directory Manager" \ - -w Secret.123 \ - -f $SHARED/index.ldif + docker exec pki pki-server ca-db-index-add -v - name: Rebuild CA search indexes run: | - # start rebuild task - sed \ - -e 's/{database}/userroot/g' \ - base/ca/database/ds/indextasks.ldif \ - | tee indextasks.ldif - docker exec ds ldapadd \ - -H ldap://ds.example.com:3389 \ - -D "cn=Directory Manager" \ - -w Secret.123 \ - -f $SHARED/indextasks.ldif - - # wait for task to complete - while true; do - sleep 1 - - docker exec ds ldapsearch \ - -H ldap://ds.example.com:3389 \ - -D "cn=Directory Manager" \ - -w Secret.123 \ - -b "cn=index1160589770, cn=index, cn=tasks, cn=config" \ - -LLL \ - nsTaskExitCode \ - | tee output - - sed -n -e 's/nsTaskExitCode:\s*\(.*\)/\1/p' output > nsTaskExitCode - cat nsTaskExitCode - - if [ -s nsTaskExitCode ]; then - break - fi - done - - echo "0" > expected - diff expected nsTaskExitCode + docker exec pki pki-server ca-db-index-rebuild -v - name: Add CA VLV indexes run: | diff --git a/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java b/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java index 5274fed0f99..56b8d337507 100644 --- a/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java +++ b/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CADBCLI.java @@ -21,6 +21,7 @@ import org.dogtagpki.cli.CLI; import org.dogtagpki.server.cli.SubsystemDBAccessCLI; import org.dogtagpki.server.cli.SubsystemDBEmptyCLI; +import org.dogtagpki.server.cli.SubsystemDBIndexCLI; import org.dogtagpki.server.cli.SubsystemDBInfoCLI; import org.dogtagpki.server.cli.SubsystemDBInitCLI; import org.dogtagpki.server.cli.SubsystemDBRemoveCLI; @@ -42,6 +43,7 @@ public CADBCLI(CLI parent) { addModule(new CADBUpgradeCLI(this)); addModule(new SubsystemDBAccessCLI(this)); + addModule(new SubsystemDBIndexCLI(this)); addModule(new SubsystemDBReplicationCLI(this)); addModule(new SubsystemDBVLVCLI(this)); } diff --git a/base/server/python/pki/server/cli/db.py b/base/server/python/pki/server/cli/db.py index 101661bb982..046c2155e31 100644 --- a/base/server/python/pki/server/cli/db.py +++ b/base/server/python/pki/server/cli/db.py @@ -218,6 +218,7 @@ def __init__(self, parent): self.add_module(SubsystemDBUpgradeCLI(self)) self.add_module(SubsystemDBAccessCLI(self)) + self.add_module(SubsystemDBIndexCLI(self)) self.add_module(SubsystemDBVLVCLI(self)) @staticmethod @@ -1045,6 +1046,176 @@ def execute(self, argv): subsystem.revoke_database_access(dn, as_current_user=as_current_user) +class SubsystemDBIndexCLI(pki.cli.CLI): + ''' + {subsystem} index management commands + ''' + + def __init__(self, parent): + super(SubsystemDBIndexCLI, self).__init__( + 'index', + inspect.cleandoc(self.__class__.__doc__).format( + subsystem=parent.parent.name.upper())) + + self.parent = parent + self.add_module(SubsystemDBIndexAddCLI(self)) + self.add_module(SubsystemDBIndexRebuildCLI(self)) + + +class SubsystemDBIndexAddCLI(pki.cli.CLI): + ''' + Add {subsystem} indexes + ''' + + help = '''\ + Usage: pki-server {subsystem}-db-index-add [OPTIONS] + + -i, --instance Instance ID (default: pki-tomcat) + -v, --verbose Run in verbose mode. + --debug Run in debug mode. + --help Show help message. + ''' + + def __init__(self, parent): + super(SubsystemDBIndexAddCLI, self).__init__( + 'add', + inspect.cleandoc(self.__class__.__doc__).format( + subsystem=parent.parent.parent.name.upper())) + + self.parent = parent + + def print_help(self): + print(textwrap.dedent(self.__class__.help).format( + subsystem=self.parent.parent.parent.name)) + + def execute(self, argv): + try: + opts, _ = getopt.gnu_getopt(argv, 'i:v', [ + 'instance=', + 'verbose', 'debug', 'help']) + + except getopt.GetoptError as e: + logger.error(e) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + subsystem_name = self.parent.parent.parent.name + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o in ('-v', '--verbose'): + logging.getLogger().setLevel(logging.INFO) + + elif o == '--debug': + logging.getLogger().setLevel(logging.DEBUG) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + logger.error('Invalid option: %s', o) + self.print_help() + sys.exit(1) + + instance = pki.server.instance.PKIInstance(instance_name) + + if not instance.exists(): + logger.error('Invalid instance: %s', instance_name) + sys.exit(1) + + instance.load() + + subsystem = instance.get_subsystem(subsystem_name) + + if not subsystem: + logger.error('No %s subsystem in instance %s.', + subsystem_name.upper(), instance_name) + sys.exit(1) + + subsystem.add_indexes() + + +class SubsystemDBIndexRebuildCLI(pki.cli.CLI): + ''' + Rebuild {subsystem} indexes + ''' + + help = '''\ + Usage: pki-server {subsystem}-db-index-rebuild [OPTIONS] + + -i, --instance Instance ID (default: pki-tomcat) + -v, --verbose Run in verbose mode. + --debug Run in debug mode. + --help Show help message. + ''' + + def __init__(self, parent): + super(SubsystemDBIndexRebuildCLI, self).__init__( + 'rebuild', + inspect.cleandoc(self.__class__.__doc__).format( + subsystem=parent.parent.parent.name.upper())) + + self.parent = parent + + def print_help(self): + print(textwrap.dedent(self.__class__.help).format( + subsystem=self.parent.parent.parent.name)) + + def execute(self, argv): + try: + opts, _ = getopt.gnu_getopt(argv, 'i:v', [ + 'instance=', + 'verbose', 'debug', 'help']) + + except getopt.GetoptError as e: + logger.error(e) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + subsystem_name = self.parent.parent.parent.name + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o in ('-v', '--verbose'): + logging.getLogger().setLevel(logging.INFO) + + elif o == '--debug': + logging.getLogger().setLevel(logging.DEBUG) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + logger.error('Invalid option: %s', o) + self.print_help() + sys.exit(1) + + instance = pki.server.instance.PKIInstance(instance_name) + + if not instance.exists(): + logger.error('Invalid instance: %s', instance_name) + sys.exit(1) + + instance.load() + + subsystem = instance.get_subsystem(subsystem_name) + + if not subsystem: + logger.error('No %s subsystem in instance %s.', + subsystem_name.upper(), instance_name) + sys.exit(1) + + subsystem.rebuild_indexes() + + class SubsystemDBVLVCLI(pki.cli.CLI): def __init__(self, parent): diff --git a/base/server/python/pki/server/deployment/__init__.py b/base/server/python/pki/server/deployment/__init__.py index 4ddd3b4e729..74648e5b990 100644 --- a/base/server/python/pki/server/deployment/__init__.py +++ b/base/server/python/pki/server/deployment/__init__.py @@ -1531,18 +1531,20 @@ def setup_database(self, subsystem, master_config): create_containers = not config.str2bool(self.mdict['pki_clone']) - # If the database is already replicated but not yet indexed, rebuild the indexes. - - rebuild_indexes = config.str2bool(self.mdict['pki_clone']) and \ - not config.str2bool(self.mdict['pki_clone_setup_replication']) and \ - config.str2bool(self.mdict['pki_clone_reindex_data']) - subsystem.init_database( setup_schema=setup_schema, create_database=create_database, create_base=create_base, - create_containers=create_containers, - rebuild_indexes=rebuild_indexes) + create_containers=create_containers) + + # always create indexes + subsystem.add_indexes() + + # if the database is already replicated but not yet indexed, rebuild the indexes + if config.str2bool(self.mdict['pki_clone']) and \ + not config.str2bool(self.mdict['pki_clone_setup_replication']) and \ + config.str2bool(self.mdict['pki_clone_reindex_data']): + subsystem.rebuild_indexes() if config.str2bool(self.mdict['pki_clone']) and \ config.str2bool(self.mdict['pki_clone_setup_replication']): diff --git a/base/server/python/pki/server/subsystem.py b/base/server/python/pki/server/subsystem.py index 576eb9dc1b2..3c62c91aeca 100644 --- a/base/server/python/pki/server/subsystem.py +++ b/base/server/python/pki/server/subsystem.py @@ -1121,7 +1121,6 @@ def init_database( create_database=False, create_base=False, create_containers=False, - rebuild_indexes=False, as_current_user=False): cmd = [self.name + '-db-init'] @@ -1138,9 +1137,6 @@ def init_database( if create_containers: cmd.append('--create-containers') - if rebuild_indexes: - cmd.append('--rebuild-indexes') - if logger.isEnabledFor(logging.DEBUG): cmd.append('--debug') @@ -1149,6 +1145,30 @@ def init_database( self.run(cmd, as_current_user=as_current_user) + def add_indexes(self): + + cmd = [self.name + '-db-index-add'] + + if logger.isEnabledFor(logging.DEBUG): + cmd.append('--debug') + + elif logger.isEnabledFor(logging.INFO): + cmd.append('--verbose') + + self.run(cmd) + + def rebuild_indexes(self): + + cmd = [self.name + '-db-index-rebuild'] + + if logger.isEnabledFor(logging.DEBUG): + cmd.append('--debug') + + elif logger.isEnabledFor(logging.INFO): + cmd.append('--verbose') + + self.run(cmd) + def empty_database(self, force=False, as_current_user=False): cmd = [self.name + '-db-empty'] diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java index 3ee2805c5be..248ce9c6db6 100644 --- a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBCLI.java @@ -35,6 +35,7 @@ public SubsystemDBCLI(CLI parent) { addModule(new SubsystemDBUpgradeCLI(this)); addModule(new SubsystemDBAccessCLI(this)); + addModule(new SubsystemDBIndexCLI(this)); addModule(new SubsystemDBReplicationCLI(this)); addModule(new SubsystemDBVLVCLI(this)); } diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexAddCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexAddCLI.java new file mode 100644 index 00000000000..43099cdc5b8 --- /dev/null +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexAddCLI.java @@ -0,0 +1,74 @@ +// +// Copyright Red Hat, Inc. +// +// SPDX-License-Identifier: GPL-2.0-or-later +// +package org.dogtagpki.server.cli; + +import org.apache.commons.cli.CommandLine; +import org.dogtagpki.cli.CLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netscape.cms.servlet.csadmin.LDAPConfigurator; +import com.netscape.cmscore.apps.EngineConfig; +import com.netscape.cmscore.ldapconn.LDAPConfig; +import com.netscape.cmscore.ldapconn.LDAPConnectionConfig; +import com.netscape.cmscore.ldapconn.LdapAuthInfo; +import com.netscape.cmscore.ldapconn.LdapBoundConnection; +import com.netscape.cmscore.ldapconn.LdapConnInfo; +import com.netscape.cmscore.ldapconn.PKISocketConfig; +import com.netscape.cmscore.ldapconn.PKISocketFactory; +import com.netscape.cmsutil.password.PasswordStore; +import com.netscape.cmsutil.password.PasswordStoreConfig; + +/** + * @author Endi S. Dewata + */ +public class SubsystemDBIndexAddCLI extends SubsystemCLI { + + public static final Logger logger = LoggerFactory.getLogger(SubsystemDBIndexAddCLI.class); + + public SubsystemDBIndexAddCLI(CLI parent) { + super("add", "Add " + parent.parent.parent.getName().toUpperCase() + " indexes", parent); + } + + @Override + public void execute(CommandLine cmd) throws Exception { + + initializeTomcatJSS(); + String subsystem = parent.parent.parent.getName(); + EngineConfig cs = getEngineConfig(subsystem); + cs.load(); + + LDAPConfig ldapConfig = cs.getInternalDBConfig(); + String instanceId = cs.getInstanceID(); + + PasswordStoreConfig psc = cs.getPasswordStoreConfig(); + PasswordStore passwordStore = PasswordStore.create(psc); + + LDAPConnectionConfig connConfig = ldapConfig.getConnectionConfig(); + + LdapConnInfo connInfo = new LdapConnInfo(connConfig); + LdapAuthInfo authInfo = getAuthInfo(passwordStore, connInfo, ldapConfig); + + PKISocketConfig socketConfig = cs.getSocketConfig(); + + PKISocketFactory socketFactory = new PKISocketFactory(); + socketFactory.setSecure(connInfo.getSecure()); + if (authInfo.getAuthType() == LdapAuthInfo.LDAP_AUTHTYPE_SSLCLIENTAUTH) { + socketFactory.setClientCertNickname(authInfo.getClientCertNickname()); + } + socketFactory.init(socketConfig); + + LdapBoundConnection conn = new LdapBoundConnection(socketFactory, connInfo, authInfo); + LDAPConfigurator ldapConfigurator = new LDAPConfigurator(conn, ldapConfig, instanceId); + + try { + ldapConfigurator.createIndexes(subsystem); + + } finally { + conn.disconnect(); + } + } +} diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexCLI.java new file mode 100644 index 00000000000..60bfb1ffa1d --- /dev/null +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexCLI.java @@ -0,0 +1,21 @@ +// +// Copyright Red Hat, Inc. +// +// SPDX-License-Identifier: GPL-2.0-or-later +// +package org.dogtagpki.server.cli; + +import org.dogtagpki.cli.CLI; + +/** + * @author Endi S. Dewata + */ +public class SubsystemDBIndexCLI extends CLI { + + public SubsystemDBIndexCLI(CLI parent) { + super("index", parent.parent.name.toUpperCase() + " index management commands", parent); + + addModule(new SubsystemDBIndexAddCLI(this)); + addModule(new SubsystemDBIndexRebuildCLI(this)); + } +} diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexRebuildCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexRebuildCLI.java new file mode 100644 index 00000000000..8203a2ea236 --- /dev/null +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBIndexRebuildCLI.java @@ -0,0 +1,74 @@ +// +// Copyright Red Hat, Inc. +// +// SPDX-License-Identifier: GPL-2.0-or-later +// +package org.dogtagpki.server.cli; + +import org.apache.commons.cli.CommandLine; +import org.dogtagpki.cli.CLI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netscape.cms.servlet.csadmin.LDAPConfigurator; +import com.netscape.cmscore.apps.EngineConfig; +import com.netscape.cmscore.ldapconn.LDAPConfig; +import com.netscape.cmscore.ldapconn.LDAPConnectionConfig; +import com.netscape.cmscore.ldapconn.LdapAuthInfo; +import com.netscape.cmscore.ldapconn.LdapBoundConnection; +import com.netscape.cmscore.ldapconn.LdapConnInfo; +import com.netscape.cmscore.ldapconn.PKISocketConfig; +import com.netscape.cmscore.ldapconn.PKISocketFactory; +import com.netscape.cmsutil.password.PasswordStore; +import com.netscape.cmsutil.password.PasswordStoreConfig; + +/** + * @author Endi S. Dewata + */ +public class SubsystemDBIndexRebuildCLI extends SubsystemCLI { + + public static final Logger logger = LoggerFactory.getLogger(SubsystemDBIndexRebuildCLI.class); + + public SubsystemDBIndexRebuildCLI(CLI parent) { + super("rebuild", "Rebuild " + parent.parent.parent.getName().toUpperCase() + " indexes", parent); + } + + @Override + public void execute(CommandLine cmd) throws Exception { + + initializeTomcatJSS(); + String subsystem = parent.parent.parent.getName(); + EngineConfig cs = getEngineConfig(subsystem); + cs.load(); + + LDAPConfig ldapConfig = cs.getInternalDBConfig(); + String instanceId = cs.getInstanceID(); + + PasswordStoreConfig psc = cs.getPasswordStoreConfig(); + PasswordStore passwordStore = PasswordStore.create(psc); + + LDAPConnectionConfig connConfig = ldapConfig.getConnectionConfig(); + + LdapConnInfo connInfo = new LdapConnInfo(connConfig); + LdapAuthInfo authInfo = getAuthInfo(passwordStore, connInfo, ldapConfig); + + PKISocketConfig socketConfig = cs.getSocketConfig(); + + PKISocketFactory socketFactory = new PKISocketFactory(); + socketFactory.setSecure(connInfo.getSecure()); + if (authInfo.getAuthType() == LdapAuthInfo.LDAP_AUTHTYPE_SSLCLIENTAUTH) { + socketFactory.setClientCertNickname(authInfo.getClientCertNickname()); + } + socketFactory.init(socketConfig); + + LdapBoundConnection conn = new LdapBoundConnection(socketFactory, connInfo, authInfo); + LDAPConfigurator ldapConfigurator = new LDAPConfigurator(conn, ldapConfig, instanceId); + + try { + ldapConfigurator.rebuildIndexes(subsystem); + + } finally { + conn.disconnect(); + } + } +} diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBInitCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBInitCLI.java index 3589f8379ae..f98009bc2d2 100644 --- a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBInitCLI.java +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBInitCLI.java @@ -50,7 +50,6 @@ public void createOptions() { options.addOption(null, "create-database", false, "Create database"); options.addOption(null, "create-base", false, "Create base entry"); options.addOption(null, "create-containers", false, "Create container entries"); - options.addOption(null, "rebuild-indexes", false, "Rebuild indexes"); options.addOption("v", "verbose", false, "Run in verbose mode."); options.addOption(null, "debug", false, "Run in debug mode."); @@ -115,12 +114,6 @@ public void execute(CommandLine cmd) throws Exception { ldapConfigurator.setupACL(subsystem); } - ldapConfigurator.createIndexes(subsystem); - - if (cmd.hasOption("rebuild-indexes")) { - ldapConfigurator.rebuildIndexes(subsystem); - } - } finally { conn.disconnect(); }