Add pki-server <subsystem>-db-repl-agmt-add

The pki-server <subsystem>-db-repl-agmt-add has been added to
create DS replication agreements prior to running pkispawn.

The SubsystemDBReplicationSetupCLI has been modified to no
longer create the replication agreements.

The PKIDeployer.setup_database() has been modified to create
the replication agreements in both the master and the replica.

The test for CA cloning with replicated DS has been modified
to use the new command.

.github/workflows/ca-clone-replicated-ds-test.yml

@@ -194,18 +194,15 @@ jobs:
 
       - name: Create replication agreement on primary DS
         run: |
-          docker exec primaryds dsconf \
-              -D "cn=Directory Manager" \
-              -w Secret.123 \
-              ldap://primaryds.example.com:3389 \
-              repl-agmt create \
-              --suffix=dc=ca,dc=pki,dc=example,dc=com \
-              --host=secondaryds.example.com \
-              --port=3389 \
-              --conn-protocol=LDAP \
-              --bind-dn="cn=Replication Manager,cn=config" \
-              --bind-passwd=Secret.123 \
-              --bind-method=SIMPLE \
+          docker exec secondary pki-server ca-db-repl-agmt-add \
+              --url ldap://primaryds.example.com:3389 \
+              --bind-dn "cn=Directory Manager" \
+              --bind-password Secret.123 \
+              --replica-url ldap://secondaryds.example.com:3389 \
+              --replica-bind-dn "cn=Replication Manager,cn=config" \
+              --replica-bind-password Secret.123 \
+              --suffix dc=ca,dc=pki,dc=example,dc=com \
+              -v \
               primaryds-to-secondaryds
 
           # check replication agreement
@@ -219,18 +216,15 @@ jobs:
 
       - name: Create replication agreement on secondary DS
         run: |
-          docker exec secondaryds dsconf \
-              -D "cn=Directory Manager" \
-              -w Secret.123 \
-              ldap://secondaryds.example.com:3389 \
-              repl-agmt create \
-              --suffix=dc=ca,dc=pki,dc=example,dc=com \
-              --host=primaryds.example.com \
-              --port=3389 \
-              --conn-protocol=LDAP \
-              --bind-dn="cn=Replication Manager,cn=config" \
-              --bind-passwd=Secret.123 \
-              --bind-method=SIMPLE \
+          docker exec secondary pki-server ca-db-repl-agmt-add \
+              --url ldap://secondaryds.example.com:3389 \
+              --bind-dn "cn=Directory Manager" \
+              --bind-password Secret.123 \
+              --replica-url ldap://primaryds.example.com:3389 \
+              --replica-bind-dn "cn=Replication Manager,cn=config" \
+              --replica-bind-password Secret.123 \
+              --suffix dc=ca,dc=pki,dc=example,dc=com \
+              -v \
               secondaryds-to-primaryds
 
           # check replication agreement

base/server/python/pki/server/cli/db.py

@@ -1354,9 +1354,159 @@ def __init__(self, parent):
                 subsystem=parent.parent.name.upper()))
 
         self.parent = parent
+        self.add_module(SubsystemDBReplicationAgreementAddCLI(self))
         self.add_module(SubsystemDBReplicationAgreementInitCLI(self))
 
 
+class SubsystemDBReplicationAgreementAddCLI(pki.cli.CLI):
+    '''
+    Add {subsystem} replication agreement
+    '''
+
+    help = '''\
+        Usage: pki-server {subsystem}-db-repl-agmt-add [OPTIONS] <name>
+
+          -i, --instance <instance ID>            Instance ID (default: pki-tomcat)
+              --url <URL>                         Database URL
+              --bind-dn <DN>                      Database bind DN
+              --bind-password <password>          Database bind password
+              --replica-url <URL>                 Replica URL
+              --replica-bind-dn <DN>              Replica bind DN
+              --replica-bind-password <password>  Replica bind password
+              --replication-security <value>      Replication security: SSL, TLS, None
+              --suffix <DN>                       Database suffix
+          -v, --verbose                           Run in verbose mode.
+              --debug                             Run in debug mode.
+              --help                              Show help message.
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBReplicationAgreementAddCLI, self).__init__(
+            'add',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.parent.parent.name.upper()))
+
+        self.parent = parent
+
+    def print_help(self):
+        print(textwrap.dedent(self.__class__.help).format(
+            subsystem=self.parent.parent.parent.parent.name))
+
+    def execute(self, argv):
+        try:
+            opts, args = getopt.gnu_getopt(argv, 'i:v', [
+                'instance=',
+                'url=', 'bind-dn=', 'bind-password=',
+                'replica-url=', 'replica-bind-dn=', 'replica-bind-password=',
+                'replication-security=', 'suffix=',
+                'verbose', 'debug', 'help'])
+
+        except getopt.GetoptError as e:
+            logger.error(e)
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        subsystem_name = self.parent.parent.parent.parent.name
+        url = None
+        bind_dn = None
+        bind_password = None
+        replica_url = None
+        replica_bind_dn = None
+        replica_bind_password = None
+        replication_security = None
+        suffix = None
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o == '--url':
+                url = urllib.parse.urlparse(a)
+
+            elif o == '--bind-dn':
+                bind_dn = a
+
+            elif o == '--bind-password':
+                bind_password = a
+
+            elif o == '--replica-url':
+                replica_url = a
+
+            elif o == '--replica-bind-dn':
+                replica_bind_dn = a
+
+            elif o == '--replica-bind-password':
+                replica_bind_password = a
+
+            elif o == '--replication-security':
+                replication_security = a
+
+            elif o == '--suffix':
+                suffix = a
+
+            elif o in ('-v', '--verbose'):
+                logging.getLogger().setLevel(logging.INFO)
+
+            elif o == '--debug':
+                logging.getLogger().setLevel(logging.DEBUG)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                logger.error('Invalid option: %s', o)
+                self.print_help()
+                sys.exit(1)
+
+        if len(args) < 1:
+            logger.error('Missing replication agreement name')
+            self.print_help()
+            sys.exit(1)
+
+        name = args[0]
+
+        instance = pki.server.instance.PKIInstance(instance_name)
+
+        if not instance.exists():
+            logger.error('Invalid instance: %s', instance_name)
+            sys.exit(1)
+
+        instance.load()
+
+        subsystem = instance.get_subsystem(subsystem_name)
+
+        if not subsystem:
+            logger.error('No %s subsystem in instance %s.',
+                         subsystem_name.upper(), instance_name)
+            sys.exit(1)
+
+        ldap_config = {}
+
+        if url.scheme == 'ldaps':
+            ldap_config['ldapconn.secureConn'] = 'true'
+        else:
+            ldap_config['ldapconn.secureConn'] = 'false'
+
+        ldap_config['ldapconn.host'] = url.hostname
+        ldap_config['ldapconn.port'] = str(url.port)
+
+        ldap_config['ldapauth.authtype'] = 'BasicAuth'
+        ldap_config['ldapauth.bindDN'] = bind_dn
+        ldap_config['ldapauth.bindPassword'] = bind_password
+
+        ldap_config['basedn'] = suffix
+
+        subsystem.add_replication_agreement(
+            name,
+            ldap_config,
+            replica_url,
+            replica_bind_dn,
+            replica_bind_password,
+            replication_security)
+
+
 class SubsystemDBReplicationAgreementInitCLI(pki.cli.CLI):
     '''
     Initialize {subsystem} replication agreement

base/server/python/pki/server/deployment/__init__.py

@@ -1598,13 +1598,9 @@ def setup_database(self, subsystem, master_config):
                 replica_replication_port=replica_replication_port,
                 replication_security=replication_security)
 
-            logger.info('Initializing replication agreement')
-
-            hostname = self.mdict['pki_hostname']
-            agreement_name = 'masterAgreement1-%s-%s' % (hostname, self.instance.name)
+            # get master database config
 
-            # get master's database config
-            ldap_config = {}
+            master_ldap_config = {}
             for name in master_config['Properties']:
 
                 match = re.match(r'internaldb\.(.*)$', name)
@@ -1627,11 +1623,80 @@ def setup_database(self, subsystem, master_config):
                     new_name = 'ldapauth.bindPassword'
 
                 value = master_config['Properties'][name]
-                ldap_config[new_name] = value
+
+                master_ldap_config[new_name] = value
+
+            # get replica database config
+
+            replica_ldap_config = {}
+            for name in subsystem.config:
+
+                match = re.match(r'internaldb\.(.*)$', name)
+
+                if not match:
+                    continue
+
+                new_name = match.group(1)  # strip internaldb prefix
+
+                if new_name.startswith('_'):  # comments
+                    continue
+
+                elif new_name == 'ldapauth.bindPWPrompt':  # replace
+                    new_name = 'ldapauth.bindPassword'
+                    value = self.instance.get_password('internaldb')
+
+                else:
+                    value = subsystem.config[name]
+
+                replica_ldap_config[new_name] = value
+
+            hostname = self.mdict['pki_hostname']
+            master_agreement_name = 'masterAgreement1-%s-%s' % (hostname, self.instance.name)
+            replica_agreement_name = 'cloneAgreement1-%s-%s' % (hostname, self.instance.name)
+
+            master_hostname = master_ldap_config['ldapconn.host']
+            if not master_replication_port:
+                master_replication_port = master_ldap_config['ldapconn.port']
+            master_url = 'ldap://%s:%s' % (master_hostname, master_replication_port)
+
+            master_bind_dn = 'cn=Replication Manager %s,ou=csusers,cn=config' % master_agreement_name
+            master_bind_password = master_config['Properties']['internaldb.replication.password']
+
+            replica_hostname = replica_ldap_config['ldapconn.host']
+            if not replica_replication_port:
+                replica_replication_port = ds_port
+            replica_url = 'ldap://%s:%s' % (replica_hostname, replica_replication_port)
+
+            replica_bind_dn = 'cn=Replication Manager %s,ou=csusers,cn=config' % replica_agreement_name
+            replica_bind_password = self.instance.get_password('replicationdb')
+
+            logger.info('Adding master replication agreement')
+            logger.info('- replica URL: %s', replica_url)
+
+            subsystem.add_replication_agreement(
+                master_agreement_name,
+                master_ldap_config,
+                replica_url,
+                replica_bind_dn,
+                replica_bind_password,
+                replication_security)
+
+            logger.info('Adding replica replication agreement')
+            logger.info('- master URL: %s', master_url)
+
+            subsystem.add_replication_agreement(
+                replica_agreement_name,
+                replica_ldap_config,
+                master_url,
+                master_bind_dn,
+                master_bind_password,
+                replication_security)
+
+            logger.info('Initializing replication agreement')
 
             subsystem.init_replication_agreement(
-                agreement_name,
-                ldap_config)
+                master_agreement_name,
+                master_ldap_config)
 
         # For security a PKI subsystem can be configured to use a database user
         # that only has a limited access to the database (instead of cn=Directory

base/server/python/pki/server/subsystem.py

@@ -1326,6 +1326,50 @@ def setup_replication(
         finally:
             shutil.rmtree(tmpdir)
 
+    def add_replication_agreement(
+            self,
+            name,
+            ldap_config,
+            replica_url,
+            replica_bind_dn,
+            replica_bind_password,
+            replication_security=None):
+
+        tmpdir = tempfile.mkdtemp()
+        try:
+            ldap_config_file = os.path.join(tmpdir, 'ldap.conf')
+            pki.util.store_properties(ldap_config_file, ldap_config)
+            pki.util.chown(tmpdir, self.instance.uid, self.instance.gid)
+
+            password_file = os.path.join(tmpdir, 'password.txt')
+            with open(password_file, 'w', encoding='utf-8') as f:
+                f.write(replica_bind_password)
+            pki.util.chown(password_file, self.instance.uid, self.instance.gid)
+
+            cmd = [
+                self.name + '-db-repl-agmt-add',
+                '--ldap-config', ldap_config_file,
+                '--replica-url', replica_url,
+                '--replica-bind-dn', replica_bind_dn,
+                '--replica-bind-password-file', password_file
+            ]
+
+            if replication_security:
+                cmd.extend(['--replication-security', replication_security])
+
+            if logger.isEnabledFor(logging.DEBUG):
+                cmd.append('--debug')
+
+            elif logger.isEnabledFor(logging.INFO):
+                cmd.append('--verbose')
+
+            cmd.append(name)
+
+            self.run(cmd)
+
+        finally:
+            shutil.rmtree(tmpdir)
+
     def init_replication_agreement(
             self,
             name,

base/server/src/main/java/com/netscape/cms/servlet/csadmin/LDAPConfigurator.java

@@ -898,7 +898,6 @@ public void createReplicationAgreement(
         logger.debug("- nsDS5ReplicaHost: " + replicaHostname);
         logger.debug("- nsDS5ReplicaPort: " + replicaPort);
         logger.debug("- nsDS5ReplicaBindDN: " + replicaBindDN);
-        logger.debug("- nsDS5ReplicaTransportInfo: " + replicationSecurity);
 
         LDAPAttributeSet attrs = new LDAPAttributeSet();
         attrs.add(new LDAPAttribute("objectclass", "top"));
@@ -913,6 +912,7 @@ public void createReplicationAgreement(
         attrs.add(new LDAPAttribute("nsDS5ReplicaCredentials", replicaPassword));
 
         if (replicationSecurity != null && !replicationSecurity.equalsIgnoreCase("None")) {
+            logger.debug("- nsDS5ReplicaTransportInfo: " + replicationSecurity);
             attrs.add(new LDAPAttribute("nsDS5ReplicaTransportInfo", replicationSecurity));
         }