From 4deda8c8358a0185e7d1fa1dd50b88f6836aba8c Mon Sep 17 00:00:00 2001 From: "Endi S. Dewata" Date: Tue, 12 Dec 2023 08:53:31 +0700 Subject: [PATCH] Add pki-server -db-repl-agmt-add The pki-server -db-repl-agmt-add has been added to create DS replication agreements prior to running pkispawn. The SubsystemDBReplicationSetupCLI has been modified to no longer create the replication agreements. The PKIDeployer.setup_database() has been modified to create the replication agreements in both the master and the replica. The test for CA cloning with replicated DS has been modified to use the new command. --- .../workflows/ca-clone-replicated-ds-test.yml | 42 +++-- base/server/python/pki/server/cli/db.py | 150 ++++++++++++++++++ .../python/pki/server/deployment/__init__.py | 83 ++++++++-- base/server/python/pki/server/subsystem.py | 44 +++++ .../cms/servlet/csadmin/LDAPConfigurator.java | 2 +- ...SubsystemDBReplicationAgreementAddCLI.java | 123 ++++++++++++++ .../SubsystemDBReplicationAgreementCLI.java | 1 + .../cli/SubsystemDBReplicationSetupCLI.java | 54 ------- 8 files changed, 411 insertions(+), 88 deletions(-) create mode 100644 base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationAgreementAddCLI.java diff --git a/.github/workflows/ca-clone-replicated-ds-test.yml b/.github/workflows/ca-clone-replicated-ds-test.yml index e20810c0797..cfbfe656ad3 100644 --- a/.github/workflows/ca-clone-replicated-ds-test.yml +++ b/.github/workflows/ca-clone-replicated-ds-test.yml @@ -194,18 +194,15 @@ jobs: - name: Create replication agreement on primary DS run: | - docker exec primaryds dsconf \ - -D "cn=Directory Manager" \ - -w Secret.123 \ - ldap://primaryds.example.com:3389 \ - repl-agmt create \ - --suffix=dc=ca,dc=pki,dc=example,dc=com \ - --host=secondaryds.example.com \ - --port=3389 \ - --conn-protocol=LDAP \ - --bind-dn="cn=Replication Manager,cn=config" \ - --bind-passwd=Secret.123 \ - --bind-method=SIMPLE \ + docker exec secondary pki-server ca-db-repl-agmt-add \ + --url ldap://primaryds.example.com:3389 \ + --bind-dn "cn=Directory Manager" \ + --bind-password Secret.123 \ + --replica-url ldap://secondaryds.example.com:3389 \ + --replica-bind-dn "cn=Replication Manager,cn=config" \ + --replica-bind-password Secret.123 \ + --suffix dc=ca,dc=pki,dc=example,dc=com \ + -v \ primaryds-to-secondaryds # check replication agreement @@ -219,18 +216,15 @@ jobs: - name: Create replication agreement on secondary DS run: | - docker exec secondaryds dsconf \ - -D "cn=Directory Manager" \ - -w Secret.123 \ - ldap://secondaryds.example.com:3389 \ - repl-agmt create \ - --suffix=dc=ca,dc=pki,dc=example,dc=com \ - --host=primaryds.example.com \ - --port=3389 \ - --conn-protocol=LDAP \ - --bind-dn="cn=Replication Manager,cn=config" \ - --bind-passwd=Secret.123 \ - --bind-method=SIMPLE \ + docker exec secondary pki-server ca-db-repl-agmt-add \ + --url ldap://secondaryds.example.com:3389 \ + --bind-dn "cn=Directory Manager" \ + --bind-password Secret.123 \ + --replica-url ldap://primaryds.example.com:3389 \ + --replica-bind-dn "cn=Replication Manager,cn=config" \ + --replica-bind-password Secret.123 \ + --suffix dc=ca,dc=pki,dc=example,dc=com \ + -v \ secondaryds-to-primaryds # check replication agreement diff --git a/base/server/python/pki/server/cli/db.py b/base/server/python/pki/server/cli/db.py index be126ebf098..96a83234ac6 100644 --- a/base/server/python/pki/server/cli/db.py +++ b/base/server/python/pki/server/cli/db.py @@ -1354,9 +1354,159 @@ def __init__(self, parent): subsystem=parent.parent.name.upper())) self.parent = parent + self.add_module(SubsystemDBReplicationAgreementAddCLI(self)) self.add_module(SubsystemDBReplicationAgreementInitCLI(self)) +class SubsystemDBReplicationAgreementAddCLI(pki.cli.CLI): + ''' + Add {subsystem} replication agreement + ''' + + help = '''\ + Usage: pki-server {subsystem}-db-repl-agmt-add [OPTIONS] + + -i, --instance Instance ID (default: pki-tomcat) + --url Database URL + --bind-dn Database bind DN + --bind-password Database bind password + --replica-url Replica URL + --replica-bind-dn Replica bind DN + --replica-bind-password Replica bind password + --replication-security Replication security: SSL, TLS, None + --suffix Database suffix + -v, --verbose Run in verbose mode. + --debug Run in debug mode. + --help Show help message. + ''' + + def __init__(self, parent): + super(SubsystemDBReplicationAgreementAddCLI, self).__init__( + 'add', + inspect.cleandoc(self.__class__.__doc__).format( + subsystem=parent.parent.parent.parent.name.upper())) + + self.parent = parent + + def print_help(self): + print(textwrap.dedent(self.__class__.help).format( + subsystem=self.parent.parent.parent.parent.name)) + + def execute(self, argv): + try: + opts, args = getopt.gnu_getopt(argv, 'i:v', [ + 'instance=', + 'url=', 'bind-dn=', 'bind-password=', + 'replica-url=', 'replica-bind-dn=', 'replica-bind-password=', + 'replication-security=', 'suffix=', + 'verbose', 'debug', 'help']) + + except getopt.GetoptError as e: + logger.error(e) + self.print_help() + sys.exit(1) + + instance_name = 'pki-tomcat' + subsystem_name = self.parent.parent.parent.parent.name + url = None + bind_dn = None + bind_password = None + replica_url = None + replica_bind_dn = None + replica_bind_password = None + replication_security = None + suffix = None + + for o, a in opts: + if o in ('-i', '--instance'): + instance_name = a + + elif o == '--url': + url = urllib.parse.urlparse(a) + + elif o == '--bind-dn': + bind_dn = a + + elif o == '--bind-password': + bind_password = a + + elif o == '--replica-url': + replica_url = a + + elif o == '--replica-bind-dn': + replica_bind_dn = a + + elif o == '--replica-bind-password': + replica_bind_password = a + + elif o == '--replication-security': + replication_security = a + + elif o == '--suffix': + suffix = a + + elif o in ('-v', '--verbose'): + logging.getLogger().setLevel(logging.INFO) + + elif o == '--debug': + logging.getLogger().setLevel(logging.DEBUG) + + elif o == '--help': + self.print_help() + sys.exit() + + else: + logger.error('Invalid option: %s', o) + self.print_help() + sys.exit(1) + + if len(args) < 1: + logger.error('Missing replication agreement name') + self.print_help() + sys.exit(1) + + name = args[0] + + instance = pki.server.instance.PKIInstance(instance_name) + + if not instance.exists(): + logger.error('Invalid instance: %s', instance_name) + sys.exit(1) + + instance.load() + + subsystem = instance.get_subsystem(subsystem_name) + + if not subsystem: + logger.error('No %s subsystem in instance %s.', + subsystem_name.upper(), instance_name) + sys.exit(1) + + ldap_config = {} + + if url.scheme == 'ldaps': + ldap_config['ldapconn.secureConn'] = 'true' + else: + ldap_config['ldapconn.secureConn'] = 'false' + + ldap_config['ldapconn.host'] = url.hostname + ldap_config['ldapconn.port'] = str(url.port) + + ldap_config['ldapauth.authtype'] = 'BasicAuth' + ldap_config['ldapauth.bindDN'] = bind_dn + ldap_config['ldapauth.bindPassword'] = bind_password + + ldap_config['basedn'] = suffix + + subsystem.add_replication_agreement( + name, + ldap_config, + replica_url, + replica_bind_dn, + replica_bind_password, + replication_security) + + class SubsystemDBReplicationAgreementInitCLI(pki.cli.CLI): ''' Initialize {subsystem} replication agreement diff --git a/base/server/python/pki/server/deployment/__init__.py b/base/server/python/pki/server/deployment/__init__.py index 7d4b0165870..d6a1f6edf69 100644 --- a/base/server/python/pki/server/deployment/__init__.py +++ b/base/server/python/pki/server/deployment/__init__.py @@ -1598,13 +1598,9 @@ def setup_database(self, subsystem, master_config): replica_replication_port=replica_replication_port, replication_security=replication_security) - logger.info('Initializing replication agreement') - - hostname = self.mdict['pki_hostname'] - agreement_name = 'masterAgreement1-%s-%s' % (hostname, self.instance.name) + # get master database config - # get master's database config - ldap_config = {} + master_ldap_config = {} for name in master_config['Properties']: match = re.match(r'internaldb\.(.*)$', name) @@ -1627,11 +1623,80 @@ def setup_database(self, subsystem, master_config): new_name = 'ldapauth.bindPassword' value = master_config['Properties'][name] - ldap_config[new_name] = value + + master_ldap_config[new_name] = value + + # get replica database config + + replica_ldap_config = {} + for name in subsystem.config: + + match = re.match(r'internaldb\.(.*)$', name) + + if not match: + continue + + new_name = match.group(1) # strip internaldb prefix + + if new_name.startswith('_'): # comments + continue + + elif new_name == 'ldapauth.bindPWPrompt': # replace + new_name = 'ldapauth.bindPassword' + value = self.instance.get_password('internaldb') + + else: + value = subsystem.config[name] + + replica_ldap_config[new_name] = value + + hostname = self.mdict['pki_hostname'] + master_agreement_name = 'masterAgreement1-%s-%s' % (hostname, self.instance.name) + replica_agreement_name = 'cloneAgreement1-%s-%s' % (hostname, self.instance.name) + + master_hostname = master_ldap_config['ldapconn.host'] + if not master_replication_port: + master_replication_port = master_ldap_config['ldapconn.port'] + master_url = 'ldap://%s:%s' % (master_hostname, master_replication_port) + + master_bind_dn = 'cn=Replication Manager %s,ou=csusers,cn=config' % master_agreement_name + master_bind_password = master_config['Properties']['internaldb.replication.password'] + + replica_hostname = replica_ldap_config['ldapconn.host'] + if not replica_replication_port: + replica_replication_port = ds_port + replica_url = 'ldap://%s:%s' % (replica_hostname, replica_replication_port) + + replica_bind_dn = 'cn=Replication Manager %s,ou=csusers,cn=config' % replica_agreement_name + replica_bind_password = self.instance.get_password('replicationdb') + + logger.info('Adding master replication agreement') + logger.info('- replica URL: %s', replica_url) + + subsystem.add_replication_agreement( + master_agreement_name, + master_ldap_config, + replica_url, + replica_bind_dn, + replica_bind_password, + replication_security) + + logger.info('Adding replica replication agreement') + logger.info('- master URL: %s', master_url) + + subsystem.add_replication_agreement( + replica_agreement_name, + replica_ldap_config, + master_url, + master_bind_dn, + master_bind_password, + replication_security) + + logger.info('Initializing replication agreement') subsystem.init_replication_agreement( - agreement_name, - ldap_config) + master_agreement_name, + master_ldap_config) # For security a PKI subsystem can be configured to use a database user # that only has a limited access to the database (instead of cn=Directory diff --git a/base/server/python/pki/server/subsystem.py b/base/server/python/pki/server/subsystem.py index ab1dfe14afa..f29bb55f6e6 100644 --- a/base/server/python/pki/server/subsystem.py +++ b/base/server/python/pki/server/subsystem.py @@ -1326,6 +1326,50 @@ def setup_replication( finally: shutil.rmtree(tmpdir) + def add_replication_agreement( + self, + name, + ldap_config, + replica_url, + replica_bind_dn, + replica_bind_password, + replication_security=None): + + tmpdir = tempfile.mkdtemp() + try: + ldap_config_file = os.path.join(tmpdir, 'ldap.conf') + pki.util.store_properties(ldap_config_file, ldap_config) + pki.util.chown(tmpdir, self.instance.uid, self.instance.gid) + + password_file = os.path.join(tmpdir, 'password.txt') + with open(password_file, 'w', encoding='utf-8') as f: + f.write(replica_bind_password) + pki.util.chown(password_file, self.instance.uid, self.instance.gid) + + cmd = [ + self.name + '-db-repl-agmt-add', + '--ldap-config', ldap_config_file, + '--replica-url', replica_url, + '--replica-bind-dn', replica_bind_dn, + '--replica-bind-password-file', password_file + ] + + if replication_security: + cmd.extend(['--replication-security', replication_security]) + + if logger.isEnabledFor(logging.DEBUG): + cmd.append('--debug') + + elif logger.isEnabledFor(logging.INFO): + cmd.append('--verbose') + + cmd.append(name) + + self.run(cmd) + + finally: + shutil.rmtree(tmpdir) + def init_replication_agreement( self, name, diff --git a/base/server/src/main/java/com/netscape/cms/servlet/csadmin/LDAPConfigurator.java b/base/server/src/main/java/com/netscape/cms/servlet/csadmin/LDAPConfigurator.java index 491994b934b..98dcba0d5cf 100644 --- a/base/server/src/main/java/com/netscape/cms/servlet/csadmin/LDAPConfigurator.java +++ b/base/server/src/main/java/com/netscape/cms/servlet/csadmin/LDAPConfigurator.java @@ -898,7 +898,6 @@ public void createReplicationAgreement( logger.debug("- nsDS5ReplicaHost: " + replicaHostname); logger.debug("- nsDS5ReplicaPort: " + replicaPort); logger.debug("- nsDS5ReplicaBindDN: " + replicaBindDN); - logger.debug("- nsDS5ReplicaTransportInfo: " + replicationSecurity); LDAPAttributeSet attrs = new LDAPAttributeSet(); attrs.add(new LDAPAttribute("objectclass", "top")); @@ -913,6 +912,7 @@ public void createReplicationAgreement( attrs.add(new LDAPAttribute("nsDS5ReplicaCredentials", replicaPassword)); if (replicationSecurity != null && !replicationSecurity.equalsIgnoreCase("None")) { + logger.debug("- nsDS5ReplicaTransportInfo: " + replicationSecurity); attrs.add(new LDAPAttribute("nsDS5ReplicaTransportInfo", replicationSecurity)); } diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationAgreementAddCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationAgreementAddCLI.java new file mode 100644 index 00000000000..218420b9c22 --- /dev/null +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationAgreementAddCLI.java @@ -0,0 +1,123 @@ +// +// Copyright Red Hat, Inc. +// +// SPDX-License-Identifier: GPL-2.0-or-later +// +package org.dogtagpki.server.cli; + +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.apache.commons.cli.CommandLine; +import org.dogtagpki.cli.CLI; +import org.dogtagpki.cli.CLIException; +import org.dogtagpki.util.logging.PKILogger; +import org.dogtagpki.util.logging.PKILogger.LogLevel; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.netscape.cms.servlet.csadmin.LDAPConfigurator; +import com.netscape.cmscore.apps.EngineConfig; +import com.netscape.cmscore.base.ConfigStorage; +import com.netscape.cmscore.base.FileConfigStorage; +import com.netscape.cmscore.ldapconn.LDAPConfig; +import com.netscape.cmscore.ldapconn.LdapBoundConnFactory; +import com.netscape.cmscore.ldapconn.PKISocketConfig; + +import netscape.ldap.LDAPConnection; +import netscape.ldap.LDAPUrl; + +/** + * @author Endi S. Dewata + */ +public class SubsystemDBReplicationAgreementAddCLI extends SubsystemCLI { + + public static Logger logger = LoggerFactory.getLogger(SubsystemDBReplicationAgreementAddCLI.class); + + public SubsystemDBReplicationAgreementAddCLI(CLI parent) { + super( + "add", + "Add " + parent.parent.parent.parent.getName().toUpperCase() + " replication agreement", + parent); + } + + @Override + public void createOptions() { + + options.addOption(null, "ldap-config", true, "LDAP configuration file"); + options.addOption(null, "replica-url", true, "Replica URL"); + options.addOption(null, "replica-bind-dn", true, "Replica bind DN"); + options.addOption(null, "replica-bind-password-file", true, "Replica bind password file"); + options.addOption(null, "replication-security", true, "Replication security: SSL, TLS, None"); + + options.addOption("v", "verbose", false, "Run in verbose mode."); + options.addOption(null, "debug", false, "Run in debug mode."); + options.addOption(null, "help", false, "Show help message."); + } + + @Override + public void execute(CommandLine cmd) throws Exception { + + String[] cmdArgs = cmd.getArgs(); + + if (cmdArgs.length < 1) { + throw new CLIException("Missing replication agreement name"); + } + + String agreementName = cmdArgs[0]; + + if (cmd.hasOption("debug")) { + PKILogger.setLevel(LogLevel.DEBUG); + + } else if (cmd.hasOption("verbose")) { + PKILogger.setLevel(LogLevel.INFO); + } + + String ldapConfigFile = cmd.getOptionValue("ldap-config"); + + if (ldapConfigFile == null) { + throw new Exception("Missing LDAP configuration file"); + } + + LDAPUrl replicaUrl = new LDAPUrl(cmd.getOptionValue("replica-url")); + String replicaBindDN = cmd.getOptionValue("replica-bind-dn"); + String replicaBindPasswordFile = cmd.getOptionValue("replica-bind-password-file"); + String replicationSecurity = cmd.getOptionValue("replication-security"); + + initializeTomcatJSS(); + String subsystem = parent.parent.parent.parent.getName(); + EngineConfig cs = getEngineConfig(subsystem); + cs.load(); + + PKISocketConfig socketConfig = cs.getSocketConfig(); + + logger.info("Loading {}", ldapConfigFile); + ConfigStorage masterConfigStorage = new FileConfigStorage(ldapConfigFile); + LDAPConfig ldapConfig = new LDAPConfig(masterConfigStorage); + ldapConfig.load(); + + String replicaHostname = replicaUrl.getHost(); + int replicaPort = replicaUrl.getPort(); + + String replicaBindPassword = Files.readAllLines(Paths.get(replicaBindPasswordFile)).get(0); + + LdapBoundConnFactory connFactory = new LdapBoundConnFactory("LDAPConfigurator"); + connFactory.init(socketConfig, ldapConfig); + LDAPConnection conn = connFactory.getConn(); + + try { + LDAPConfigurator configurator = new LDAPConfigurator(conn, ldapConfig); + + configurator.createReplicationAgreement( + agreementName, + replicaHostname, + replicaPort, + replicaBindDN, + replicaBindPassword, + replicationSecurity); + + } finally { + if (conn != null) conn.disconnect(); + } + } +} diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationAgreementCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationAgreementCLI.java index b2b6e67b945..409c24f809b 100644 --- a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationAgreementCLI.java +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationAgreementCLI.java @@ -18,6 +18,7 @@ public SubsystemDBReplicationAgreementCLI(CLI parent) { parent.parent.parent.name.toUpperCase() + " replication agreement management commands", parent); + addModule(new SubsystemDBReplicationAgreementAddCLI(this)); addModule(new SubsystemDBReplicationAgreementInitCLI(this)); } } diff --git a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationSetupCLI.java b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationSetupCLI.java index 199d5e7e830..96108a76589 100644 --- a/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationSetupCLI.java +++ b/base/server/src/main/java/org/dogtagpki/server/cli/SubsystemDBReplicationSetupCLI.java @@ -112,7 +112,6 @@ public void setupReplication( LDAPConfig ldapConfig = cs.getInternalDBConfig(); LDAPConnectionConfig replicaConnConfig = ldapConfig.getConnectionConfig(); - String replicaHostname = replicaConnConfig.getString("host", ""); String replicaPort = replicaConnConfig.getString("port", ""); if (replicaReplicationPort == null || replicaReplicationPort.equals("")) { @@ -128,7 +127,6 @@ public void setupReplication( try { LDAPConfig masterDBConfig = masterConfig.getSubStore("internaldb", LDAPConfig.class); LDAPConnectionConfig masterConnConfig = masterDBConfig.getConnectionConfig(); - String masterHostname = masterConnConfig.getString("host", ""); String masterPort = masterConnConfig.getString("port", ""); if (masterReplicationPort == null || masterReplicationPort.equals("")) { @@ -183,21 +181,6 @@ public void setupReplication( logger.info("New replica number range: " + beginReplicaNumber + "-" + endReplicaNumber); dbConfig.putString("beginReplicaNumber", Integer.toString(beginReplicaNumber)); - createReplicationAgreements( - masterConfigurator, - ldapConfigurator, - masterAgreementName, - replicaAgreementName, - replicationSecurity, - masterHostname, - replicaHostname, - Integer.parseInt(masterReplicationPort), - Integer.parseInt(replicaReplicationPort), - masterBindDN, - replicaBindDN, - masterReplicationPassword, - replicaReplicationPassword); - } finally { if (masterConn != null) masterConn.disconnect(); } @@ -252,41 +235,4 @@ public int enableReplication( return replicaID; } - - public void createReplicationAgreements( - LDAPConfigurator masterConfigurator, - LDAPConfigurator replicaConfigurator, - String masterAgreementName, - String replicaAgreementName, - String replicationSecurity, - String masterHostname, - String replicaHostname, - int masterReplicationPort, - int replicaReplicationPort, - String masterBindDN, - String replicaBindDN, - String masterReplicationPassword, - String replicaReplicationPassword) - throws Exception { - - logger.info("Creating replication agreement on " + masterHostname); - - masterConfigurator.createReplicationAgreement( - masterAgreementName, - replicaHostname, - replicaReplicationPort, - replicaBindDN, - replicaReplicationPassword, - replicationSecurity); - - logger.info("Creating replication agreement on " + replicaHostname); - - replicaConfigurator.createReplicationAgreement( - replicaAgreementName, - masterHostname, - masterReplicationPort, - masterBindDN, - masterReplicationPassword, - replicationSecurity); - } }