Add pki-server <subsystem>-db-repl-agmt-init

The pki-server <subsystem>-db-repl-agmt-init has been added
to start initializing the replication agreement and wait
until it's complete.

The SubsystemDBReplicationSetupCLI has been modified to no
longer include initializing the replication agreement.

The test for CA cloning with replicated DS has been updated
to use the new command.

.github/workflows/ca-clone-replicated-ds-test.yml

@@ -103,6 +103,10 @@ jobs:
           docker exec secondary pki-server create
           docker exec secondary pki-server nss-create --no-password
 
+      - name: Create secondary CA subsystem
+        run: |
+          docker exec secondary pki-server ca-create -v
+
       - name: Import system certs and keys into secondary CA
         run: |
           docker exec secondary pki \
@@ -143,6 +147,22 @@ jobs:
               --bind-dn="cn=Replication Manager,cn=config" \
               --bind-passwd=Secret.123
 
+          # check replication manager
+          docker exec primaryds ldapsearch \
+              -H ldap://primaryds.example.com:3389 \
+              -D "cn=Directory Manager" \
+              -w Secret.123 \
+              -x \
+              -b "cn=Replication Manager,cn=config"
+
+          # check replica object
+          docker exec primaryds ldapsearch \
+              -H ldap://primaryds.example.com:3389 \
+              -D "cn=Directory Manager" \
+              -w Secret.123 \
+              -x \
+              -b "cn=replica,cn=dc\3Dca\2Cdc\3Dpki\2Cdc\3Dexample\2Cdc\3Dcom,cn=mapping tree,cn=config"
+
       - name: Enable replication on secondary DS
         run: |
           docker exec secondaryds dsconf \
@@ -156,7 +176,23 @@ jobs:
               --bind-dn="cn=Replication Manager,cn=config" \
               --bind-passwd=Secret.123
 
-      - name: Create replication manager on primary DS
+          # check replication manager
+          docker exec secondaryds ldapsearch \
+              -H ldap://secondaryds.example.com:3389 \
+              -D "cn=Directory Manager" \
+              -w Secret.123 \
+              -x \
+              -b "cn=Replication Manager,cn=config"
+
+          # check replica object
+          docker exec secondaryds ldapsearch \
+              -H ldap://secondaryds.example.com:3389 \
+              -D "cn=Directory Manager" \
+              -w Secret.123 \
+              -x \
+              -b "cn=replica,cn=dc\3Dca\2Cdc\3Dpki\2Cdc\3Dexample\2Cdc\3Dcom,cn=mapping tree,cn=config"
+
+      - name: Create replication agreement on primary DS
         run: |
           docker exec primaryds dsconf \
               -D "cn=Directory Manager" \
@@ -172,7 +208,16 @@ jobs:
               --bind-method=SIMPLE \
               primaryds-to-secondaryds
 
-      - name: Create replication manager on secondary DS
+          # check replication agreement
+          docker exec primaryds ldapsearch \
+              -H ldap://primaryds.example.com:3389 \
+              -D "cn=Directory Manager" \
+              -w Secret.123 \
+              -x \
+              -b "cn=primaryds-to-secondaryds,cn=replica,cn=dc\3Dca\2Cdc\3Dpki\2Cdc\3Dexample\2Cdc\3Dcom,cn=mapping tree,cn=config" \
+              dn
+
+      - name: Create replication agreement on secondary DS
         run: |
           docker exec secondaryds dsconf \
               -D "cn=Directory Manager" \
@@ -188,38 +233,24 @@ jobs:
               --bind-method=SIMPLE \
               secondaryds-to-primaryds
 
-      - name: Initializing replication agreement
-        run: |
-          # start initialization
-          docker exec primaryds dsconf \
+          # check replication agreement
+          docker exec secondaryds ldapsearch \
+              -H ldap://secondaryds.example.com:3389 \
               -D "cn=Directory Manager" \
               -w Secret.123 \
-              ldap://primaryds.example.com:3389 \
-              repl-agmt init \
-              --suffix=dc=ca,dc=pki,dc=example,dc=com \
-              primaryds-to-secondaryds
+              -x \
+              -b "cn=secondaryds-to-primaryds,cn=replica,cn=dc\3Dca\2Cdc\3Dpki\2Cdc\3Dexample\2Cdc\3Dcom,cn=mapping tree,cn=config" \
+              dn
 
-          # wait for initialization to complete
-          counter=0
-          while [[ "$counter" -lt 30 ]]; do
-              sleep 1
-
-              docker exec primaryds dsconf \
-                  -D "cn=Directory Manager" \
-                  -w Secret.123 \
-                  ldap://primaryds.example.com:3389 \
-                  repl-agmt init-status \
-                  --suffix=dc=ca,dc=pki,dc=example,dc=com \
-                  primaryds-to-secondaryds \
-                  > >(tee stdout) 2> >(tee stderr >&2) || true
-
-              STDOUT=$(cat stdout)
-              if [ "$STDOUT" = "Agreement successfully initialized." ]; then
-                  break
-              fi
-
-              counter=$((counter+1))
-          done
+      - name: Initializing replication agreement
+        run: |
+          docker exec secondary pki-server ca-db-repl-agmt-init \
+              --url ldap://primaryds.example.com:3389 \
+              --bind-dn "cn=Directory Manager" \
+              --bind-password Secret.123 \
+              --suffix dc=ca,dc=pki,dc=example,dc=com \
+              -v \
+              primaryds-to-secondaryds
 
       - name: Check entries in primary DS and secondary DS
         run: |

base/server/python/pki/server/cli/db.py

@@ -27,6 +27,7 @@
 import subprocess
 import sys
 import textwrap
+import urllib.parse
 
 import pki.cli
 import pki.server.instance
@@ -220,6 +221,7 @@ def __init__(self, parent):
 
         self.add_module(SubsystemDBAccessCLI(self))
         self.add_module(SubsystemDBIndexCLI(self))
+        self.add_module(SubsystemDBReplicationCLI(self))
         self.add_module(SubsystemDBVLVCLI(self))
 
     @staticmethod
@@ -1325,6 +1327,159 @@ def execute(self, argv):
         subsystem.rebuild_indexes()
 
 
+class SubsystemDBReplicationCLI(pki.cli.CLI):
+    '''
+    {subsystem} replication management commands
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBReplicationCLI, self).__init__(
+            'repl',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.name.upper()))
+
+        self.parent = parent
+        self.add_module(SubsystemDBReplicationAgreementCLI(self))
+
+
+class SubsystemDBReplicationAgreementCLI(pki.cli.CLI):
+    '''
+    {subsystem} replication agreement management commands
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBReplicationAgreementCLI, self).__init__(
+            'agmt',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.name.upper()))
+
+        self.parent = parent
+        self.add_module(SubsystemDBReplicationAgreementInitCLI(self))
+
+
+class SubsystemDBReplicationAgreementInitCLI(pki.cli.CLI):
+    '''
+    Initialize {subsystem} replication agreement
+    '''
+
+    help = '''\
+        Usage: pki-server {subsystem}-db-repl-agmt-init [OPTIONS] <name>
+
+          -i, --instance <instance ID>       Instance ID (default: pki-tomcat)
+              --url <URL>                    Database URL
+              --bind-dn <DN>                 Database bind DN
+              --bind-password <password>     Database bind password
+              --suffix <DN>                  Database suffix
+          -v, --verbose                      Run in verbose mode.
+              --debug                        Run in debug mode.
+              --help                         Show help message.
+    '''
+
+    def __init__(self, parent):
+        super(SubsystemDBReplicationAgreementInitCLI, self).__init__(
+            'init',
+            inspect.cleandoc(self.__class__.__doc__).format(
+                subsystem=parent.parent.parent.parent.name.upper()))
+
+        self.parent = parent
+
+    def print_help(self):
+        print(textwrap.dedent(self.__class__.help).format(
+            subsystem=self.parent.parent.parent.parent.name))
+
+    def execute(self, argv):
+        try:
+            opts, args = getopt.gnu_getopt(argv, 'i:v', [
+                'instance=',
+                'url=', 'bind-dn=', 'bind-password=', 'suffix=',
+                'verbose', 'debug', 'help'])
+
+        except getopt.GetoptError as e:
+            logger.error(e)
+            self.print_help()
+            sys.exit(1)
+
+        instance_name = 'pki-tomcat'
+        subsystem_name = self.parent.parent.parent.parent.name
+        url = None
+        bind_dn = None
+        bind_password = None
+        suffix = None
+
+        for o, a in opts:
+            if o in ('-i', '--instance'):
+                instance_name = a
+
+            elif o == '--url':
+                url = urllib.parse.urlparse(a)
+
+            elif o == '--bind-dn':
+                bind_dn = a
+
+            elif o == '--bind-password':
+                bind_password = a
+
+            elif o == '--suffix':
+                suffix = a
+
+            elif o in ('-v', '--verbose'):
+                logging.getLogger().setLevel(logging.INFO)
+
+            elif o == '--debug':
+                logging.getLogger().setLevel(logging.DEBUG)
+
+            elif o == '--help':
+                self.print_help()
+                sys.exit()
+
+            else:
+                logger.error('Invalid option: %s', o)
+                self.print_help()
+                sys.exit(1)
+
+        if len(args) < 1:
+            logger.error('Missing replication agreement name')
+            self.print_help()
+            sys.exit(1)
+
+        name = args[0]
+
+        instance = pki.server.instance.PKIInstance(instance_name)
+
+        if not instance.exists():
+            logger.error('Invalid instance: %s', instance_name)
+            sys.exit(1)
+
+        instance.load()
+
+        subsystem = instance.get_subsystem(subsystem_name)
+
+        if not subsystem:
+            logger.error('No %s subsystem in instance %s.',
+                         subsystem_name.upper(), instance_name)
+            sys.exit(1)
+
+        ldap_config = {}
+
+        if url.scheme == 'ldaps':
+            ldap_config['ldapconn.secureConn'] = 'true'
+        else:
+            ldap_config['ldapconn.secureConn'] = 'false'
+
+        ldap_config['ldapconn.host'] = url.hostname
+        ldap_config['ldapconn.port'] = str(url.port)
+
+        ldap_config['ldapauth.authtype'] = 'BasicAuth'
+        ldap_config['ldapauth.bindDN'] = bind_dn
+        ldap_config['ldapauth.bindPassword'] = bind_password
+
+        ldap_config['basedn'] = suffix
+
+        subsystem.init_replication_agreement(
+            name,
+            ldap_config)
+
+
 class SubsystemDBVLVCLI(pki.cli.CLI):
 
     def __init__(self, parent):

base/server/python/pki/server/deployment/__init__.py

@@ -1598,6 +1598,41 @@ def setup_database(self, subsystem, master_config):
                 replica_replication_port=replica_replication_port,
                 replication_security=replication_security)
 
+            logger.info('Initializing replication agreement')
+
+            hostname = self.mdict['pki_hostname']
+            agreement_name = 'masterAgreement1-%s-%s' % (hostname, self.instance.name)
+
+            # get master's database config
+            ldap_config = {}
+            for name in master_config['Properties']:
+
+                match = re.match(r'internaldb\.(.*)$', name)
+
+                if not match:
+                    continue
+
+                new_name = match.group(1)  # strip internaldb prefix
+
+                if new_name == 'replication.password':  # unused
+                    continue
+
+                elif new_name == 'ldapauth.bindPWPrompt':  # unused
+                    continue
+
+                elif new_name.startswith('_'):  # comments
+                    continue
+
+                elif new_name == 'ldapauth.password':  # rename
+                    new_name = 'ldapauth.bindPassword'
+
+                value = master_config['Properties'][name]
+                ldap_config[new_name] = value
+
+            subsystem.init_replication_agreement(
+                agreement_name,
+                ldap_config)
+
         # For security a PKI subsystem can be configured to use a database user
         # that only has a limited access to the database (instead of cn=Directory
         # Manager that has a full access to the database).

base/server/python/pki/server/subsystem.py

@@ -1326,6 +1326,35 @@ def setup_replication(
         finally:
             shutil.rmtree(tmpdir)
 
+    def init_replication_agreement(
+            self,
+            name,
+            ldap_config):
+
+        tmpdir = tempfile.mkdtemp()
+        try:
+            ldap_config_file = os.path.join(tmpdir, 'ldap.conf')
+            pki.util.store_properties(ldap_config_file, ldap_config)
+            pki.util.chown(tmpdir, self.instance.uid, self.instance.gid)
+
+            cmd = [self.name + '-db-repl-agmt-init']
+
+            if ldap_config_file:
+                cmd.extend(['--ldap-config', ldap_config_file])
+
+            if logger.isEnabledFor(logging.DEBUG):
+                cmd.append('--debug')
+
+            elif logger.isEnabledFor(logging.INFO):
+                cmd.append('--verbose')
+
+            cmd.append(name)
+
+            self.run(cmd)
+
+        finally:
+            shutil.rmtree(tmpdir)
+
     def find_vlv(self, as_current_user=False):
 
         cmd = [self.name + '-db-vlv-find']