diff --git a/gluu_install.py b/gluu_install.py index a5ffd2f88..e5703493f 100644 --- a/gluu_install.py +++ b/gluu_install.py @@ -25,8 +25,14 @@ parser.add_argument('-uninstall', help="Uninstall Gluu server and removes all files", action='store_true') parser.add_argument('--args', help="Arguments to be passed to setup.py") parser.add_argument('--keep-downloads', help="Keep downloaded files", action='store_true') + if '-a' in sys.argv: parser.add_argument('--jetty-version', help="Jetty verison. For example 11.0.6") + +if '-uninstall' not in sys.argv: + parser.add_argument('-maven-user', help="Maven username", required=True) + parser.add_argument('-maven-password', help="Maven password", required=True) + parser.add_argument('-n', help="No prompt", action='store_true') parser.add_argument('--no-setup', help="Do not launch setup", action='store_true') parser.add_argument('--dist-server-base', help="Download server", default='https://maven.gluu.org/maven') @@ -238,6 +244,14 @@ def check_installation(): sys.exit() + +passman = request.HTTPPasswordMgrWithDefaultRealm() +passman.add_password(None, maven_root, argsp.maven_user, argsp.maven_password) +authhandler = request.HTTPBasicAuthHandler(passman) +opener = request.build_opener(authhandler) +request.install_opener(opener) + + def download(url, target_fn): dst = os.path.join(app_dir, target_fn) pardir, fn = os.path.split(dst) @@ -259,7 +273,7 @@ def download(url, target_fn): def download_gcs(): if not os.path.exists(os.path.join(app_dir, 'gcs')): print("Downloading Spanner modules") - gcs_download_url = 'http://162.243.99.240/icrby8xcvbcv/spanner/gcs.tgz' + gcs_download_url = 'http://ox.gluu.org/icrby8xcvbcv/spanner/gcs.tgz' tmp_dir = '/tmp/' + os.urandom(5).hex() target_fn = os.path.join(tmp_dir, 'gcs.tgz') download(gcs_download_url, target_fn) diff --git a/schema/gluu_schema.json b/schema/gluu_schema.json index ba69396a9..33230534b 100644 --- a/schema/gluu_schema.json +++ b/schema/gluu_schema.json @@ -3220,6 +3220,7 @@ "primaryKeyValue" ], "oid": "oxAttribute", + "multivalued": true, "substr": "caseIgnoreSubstringsMatch", "syntax": "1.3.6.1.4.1.1466.115.121.1.15", "x_origin": "Gluu created attribute" @@ -3329,6 +3330,7 @@ "secondaryKeyValue" ], "oid": "oxAttribute", + "multivalued": true, "substr": "caseIgnoreSubstringsMatch", "syntax": "1.3.6.1.4.1.1466.115.121.1.15", "x_origin": "Gluu created attribute" @@ -3405,6 +3407,7 @@ "tertiaryKeyValue" ], "oid": "oxAttribute", + "multivalued": true, "substr": "caseIgnoreSubstringsMatch", "syntax": "1.3.6.1.4.1.1466.115.121.1.15", "x_origin": "Gluu created attribute" diff --git a/setup.py b/setup.py index 77295bc6e..c91795a23 100755 --- a/setup.py +++ b/setup.py @@ -42,6 +42,8 @@ from setup_app.messages import msg from setup_app.config import Config +sys.path.append(os.path.join(Config.distFolder, 'app/gcs')) + # set profile if argsp.profile == 'DISA-STIG' or os.path.exists(os.path.join(paths.INSTALL_DIR, 'disa-stig')): Config.profile = static.SetupProfiles.DISA_STIG @@ -170,6 +172,7 @@ if not Config.noPrompt and not GSA and not Config.installed_instance and not setup_loaded: propertiesUtils.promptForProperties() + if not (GSA or base.argsp.dummy): propertiesUtils.check_properties() @@ -194,7 +197,26 @@ rdbmInstaller.packageUtils = packageUtils + + if Config.installed_instance: + + exit_after_me = False + + if argsp.enable_script: + print("Enabling scripts {}".format(', '.join(argsp.enable_script))) + gluuInstaller.enable_scripts(argsp.enable_script) + exit_after_me = True + + if argsp.ox_authentication_mode or argsp.ox_trust_authentication_mode: + print("Setting Authentication Modes") + gluuInstaller.set_auth_modes() + exit_after_me = True + + if exit_after_me: + sys.exit() + + for installer in (openDjInstaller, couchbaseInstaller, httpdinstaller, oxauthInstaller, passportInstaller, scimInstaller, fidoInstaller, samlInstaller, oxdInstaller, @@ -333,10 +355,12 @@ def install_services(): def start_services(): for service in gluuProgress.services: - # we don't restart opendj - if service['object'].service_name in ('opendj', 'couchbase-server'): - continue + if service['app_type'] == static.AppType.SERVICE: + # we don't restart opendj + if service['object'].service_name in ('opendj', 'couchbase-server'): + continue + gluuProgress.progress(PostSetup.service_name, "Starting {}".format(service['name'].title())) time.sleep(2) service['object'].stop() diff --git a/setup_app/config.py b/setup_app/config.py index 9575dd673..71479a007 100644 --- a/setup_app/config.py +++ b/setup_app/config.py @@ -147,6 +147,7 @@ def progress(self, service_name, msg, incr=False): 'oxauthClient_4_inum': 'FF81-2D39', 'idp_attribute_resolver_ldap.search_filter': '(|(uid=$requestContext.principalName)(mail=$requestContext.principalName))', 'oxd_port': '8443', + 'server_time_zone': 'UTC' + time.strftime("%z"), } # java commands @@ -277,7 +278,8 @@ def progress(self, service_name, msg, incr=False): self.gluuScriptFiles = [ os.path.join(self.install_dir, 'static/scripts/logmanager.sh'), - os.path.join(self.install_dir, 'static/scripts/testBind.py') + os.path.join(self.install_dir, 'static/scripts/testBind.py'), + os.path.join(self.install_dir, 'static/scripts/jetty10CompatibleWar.py'), ] self.redhat_services = ['httpd', 'rsyslog'] diff --git a/setup_app/installers/gluu.py b/setup_app/installers/gluu.py index 69296410b..b7a960ed9 100644 --- a/setup_app/installers/gluu.py +++ b/setup_app/installers/gluu.py @@ -431,6 +431,12 @@ def post_install_tasks(self): self.chown(Config.gluuBaseFolder, Config.root_user, Config.gluu_group, recursive=True) self.chown(Config.oxBaseDataFolder, Config.root_user, Config.gluu_group, recursive=True) + #enable scripts + self.enable_scripts(base.argsp.enable_script) + + #set auth modes + self.set_auth_modes() + for sys_path in (Config.gluuOptFolder, Config.gluuBaseFolder, Config.oxBaseDataFolder): self.run([paths.cmd_chmod, '-R', 'u+rwX,g+rwX,o-rwX', sys_path]) @@ -441,6 +447,11 @@ def post_install_tasks(self): if Config.profile == static.SetupProfiles.DISA_STIG: self.disa_stig_post_install_tasks() + + if base.argsp.gluu_scan_cert: + self.generate_gluu_scan_api_keystore() + + def disa_stig_post_install_tasks(self): self.chown(Config.gluuOptFolder, Config.jetty_user, Config.gluu_group) @@ -461,3 +472,19 @@ def disa_stig_post_install_tasks(self): self.chown(jetty_absolute_dir.parent.as_posix(), Config.user_group, recursive=True) + def enable_scripts(self, inums): + if inums: + for inum in inums: + self.dbUtils.enable_script(inum) + + def set_auth_modes(self): + if base.argsp.ox_authentication_mode: + self.dbUtils.set_configuration('oxAuthenticationMode', base.argsp.ox_authentication_mode) + if base.argsp.ox_trust_authentication_mode: + self.dbUtils.set_configuration('oxTrustAuthenticationMode', base.argsp.ox_trust_authentication_mode) + + def generate_gluu_scan_api_keystore(self): + suffix = 'scan_api' + key_fn, csr_fn, crt_fn = self.gen_cert(suffix, 'changeit', user='jetty') + scan_api_keystore_fn = os.path.join(Config.certFolder, 'scanAKeystore.pcks12') + self.gen_keystore(suffix, scan_api_keystore_fn, 'changeit', key_fn, crt_fn, store_type='PKCS12') diff --git a/setup_app/installers/oxd.py b/setup_app/installers/oxd.py index 403c10a5b..a5dbcb5f9 100644 --- a/setup_app/installers/oxd.py +++ b/setup_app/installers/oxd.py @@ -32,6 +32,11 @@ def __init__(self): self.ks_type_bcfks = 'bcfks' self.ks_type_pkcs12 = 'pkcs12' + @property + def oxd_hostname(self): + oxd_hostname, oxd_port = self.parse_url(Config.oxd_server_https) + return oxd_hostname + def install(self): self.logIt("Installing {}".format(self.service_name.title()), pbar=self.service_name) self.run(['tar', '-zxf', Config.oxd_package, '--no-same-owner', '--strip-components=1', '-C', self.oxd_root]) @@ -69,6 +74,7 @@ def install(self): if not base.argsp.dummy: self.modify_config_yml() self.generate_keystore() + self.import_oxd_certificate() self.run([paths.cmd_chown, '-R', '{0}:{0}'.format(oxd_user), self.oxd_root]) @@ -89,22 +95,31 @@ def install(self): self.enable() - def modify_config_yml(self): - self.logIt("Configuring", pbar=self.service_name) + def get_yaml_config(self): yml_str = self.readFile(self.oxd_server_yml_fn) oxd_yaml = ruamel.yaml.load(yml_str, ruamel.yaml.RoundTripLoader) + return oxd_yaml + + def modify_config_yml(self): + self.logIt("Configuring", pbar=self.service_name) + + oxd_yaml = self.get_yaml_config() + addr_list = [Config.ip] + lo = '127.0.0.1' + if self.oxd_hostname == 'localhost': + addr_list.append(lo) if 'bind_ip_addresses' in oxd_yaml: - oxd_yaml['bind_ip_addresses'].append(Config.ip) + oxd_yaml['bind_ip_addresses'] += addr_list else: for i, k in enumerate(oxd_yaml): if k == 'storage': break else: i = 1 - addr_list = [Config.ip] - if Config.profile == SetupProfiles.DISA_STIG: - addr_list.append('127.0.0.1') + + if Config.profile == SetupProfiles.DISA_STIG and lo not in addr_list: + addr_list.append(lo) oxd_yaml.insert(i, 'bind_ip_addresses', addr_list) if Config.get('oxd_use_gluu_storage'): @@ -148,8 +163,8 @@ def modify_config_yml(self): admin_connectors['jceProvider'] = self.fips_provider['-providerclass'] admin_connectors['validateCerts'] = 'false' - oxd_yaml['crypt_provider_key_store_path']=self.oxd_jwks_keystore_fn - oxd_yaml['crypt_provider_key_store_password']=self.oxd_keystore_passw + oxd_yaml['crypt_provider_key_store_path'] = self.oxd_jwks_keystore_fn + oxd_yaml['crypt_provider_key_store_password'] = self.oxd_keystore_passw yml_str = ruamel.yaml.dump(oxd_yaml, Dumper=ruamel.yaml.RoundTripDumper) self.writeFile(self.oxd_server_yml_fn, yml_str) @@ -184,7 +199,7 @@ def generate_keystore(self): self.run(cmd_cert_gen) - else: + elif self.oxd_hostname != 'localhost': oxd_key_tmp = '{}/{}'.format(tempfile.gettempdir(),'oxd.key') oxd_crt_tmp = '{}/{}'.format(tempfile.gettempdir(),'oxd.crt') oxd_p12_tmp = '{}/{}'.format(tempfile.gettempdir(),'oxd.p12') @@ -223,54 +238,26 @@ def generate_keystore(self): for f in (oxd_key_tmp, oxd_crt_tmp, oxd_p12_tmp): self.run([paths.cmd_rm, '-f', f]) - self.run([paths.cmd_rm, '-f', os.path.join(self.oxd_root,'conf/oxd-server.keystore')]) + if os.path.exists(keystore_tmp): + self.run([paths.cmd_rm, '-f', os.path.join(self.oxd_root,'conf/oxd-server.keystore')]) + self.run(['cp', '-f', keystore_tmp, self.oxd_server_keystore_fn]) + self.run([paths.cmd_rm, '-f', keystore_tmp]) - self.run(['cp', '-f', keystore_tmp, self.oxd_server_keystore_fn]) self.run([paths.cmd_chown, 'jetty:jetty', self.oxd_server_keystore_fn]) - self.run([paths.cmd_rm, '-f', keystore_tmp]) - - self.import_oxd_certificate() - def installed(self): return os.path.exists(self.oxd_server_yml_fn) - def download_files(self, force=False): - oxd_url = Config.maven_root + '/maven/org/gluu/oxd-server/{0}/oxd-server-{0}-distribution.zip'.format(Config.oxVersion) - - self.logIt("Downloading {} and preparing package".format(os.path.basename(oxd_url))) - - oxd_zip_fn = os.path.join(Config.outputFolder, 'oxd-server.zip') - oxd_tgz_fn = os.path.join(Config.distGluuFolder, 'oxd-server.tgz') - tmp_dir = os.path.join('/tmp', os.urandom(5).hex()) - oxd_tmp_dir = os.path.join(tmp_dir, 'oxd-server') - - self.run([paths.cmd_mkdir, '-p', oxd_tmp_dir]) - self.download_file(oxd_url, oxd_zip_fn) - self.run([paths.cmd_unzip, '-qqo', oxd_zip_fn, '-d', oxd_tmp_dir]) - self.run([paths.cmd_mkdir, os.path.join(oxd_tmp_dir, 'data')]) - - service_file = 'oxd-server.init.d' if base.deb_sysd_clone else 'oxd-server.service' - service_url = 'https://raw.githubusercontent.com/GluuFederation/community-edition-package/master/package/systemd/oxd-server.service'.format(Config.oxVersion, service_file) - self.download_file(service_url, os.path.join(oxd_tmp_dir, service_file)) - - oxd_server_sh_url = 'https://raw.githubusercontent.com/GluuFederation/oxd/master/debian/oxd-server' - self.download_file(oxd_server_sh_url, os.path.join(oxd_tmp_dir, 'bin/oxd-server')) - - self.run(['tar', '-zcf', oxd_tgz_fn, 'oxd-server'], cwd=tmp_dir) - #self.run(['rm', '-r', '-f', tmp_dir]) - Config.oxd_package = oxd_tgz_fn - def import_oxd_certificate(self): - - oxd_hostname, oxd_port = self.parse_url(Config.oxd_server_https) - oxd_alias = 'oxd_' + oxd_hostname.replace('.','_') + oxd_yaml = self.get_yaml_config() + oxd_alias = 'oxd_' + self.oxd_hostname.replace('.','_') oxd_cert_fn = os.path.join(Config.outputFolder, '{}.pem'.format(oxd_alias)) # let's delete if alias exists self.delete_key(oxd_alias) - self.export_cert_from_store(Config.hostname, self.oxd_server_keystore_fn, self.oxd_keystore_passw, oxd_cert_fn) + store_alias = 'localhost' if self.oxd_hostname == 'localhost' else Config.hostname + self.export_cert_from_store(store_alias, oxd_yaml['server']['applicationConnectors'][0]['keyStorePath'], self.oxd_keystore_passw, oxd_cert_fn) self.import_cert_to_java_truststore(oxd_alias, oxd_cert_fn) diff --git a/setup_app/installers/oxtrust.py b/setup_app/installers/oxtrust.py index 7934a3476..04a1d8c0e 100644 --- a/setup_app/installers/oxtrust.py +++ b/setup_app/installers/oxtrust.py @@ -1,6 +1,7 @@ import os import glob import uuid +import json from setup_app import paths from setup_app.utils import base @@ -113,6 +114,12 @@ def render_import_templates(self): for tmp in (self.oxtrust_config_json, self.oxtrust_cache_refresh_json, self.oxtrust_import_person_json): self.renderTemplateInOut(tmp, self.templates_folder, self.output_folder) + if Config.rdbm_install: + oxtrust_config = base.readJsonFile(self.oxtrust_config_json, ordered=True) + oxtrust_config['personObjectClassTypes'] = [] + oxtrust_config['personObjectClassDisplayNames'] = [] + self.writeFile(self.oxtrust_config_json, json.dumps(oxtrust_config, indent=2)) + Config.templateRenderingDict['oxtrust_config_base64'] = self.generate_base64_ldap_file(self.oxtrust_config_json) Config.templateRenderingDict['oxtrust_cache_refresh_base64'] = self.generate_base64_ldap_file(self.oxtrust_cache_refresh_json) Config.templateRenderingDict['oxtrust_import_person_base64'] = self.generate_base64_ldap_file(self.oxtrust_import_person_json) diff --git a/setup_app/installers/rdbm.py b/setup_app/installers/rdbm.py index badf1c54a..ee72a1261 100644 --- a/setup_app/installers/rdbm.py +++ b/setup_app/installers/rdbm.py @@ -10,6 +10,7 @@ from setup_app.static import AppType, InstallOption from setup_app.config import Config from setup_app.utils import base +from setup_app.utils import ldif_utils from setup_app.static import InstallTypes from setup_app.installers.base import BaseInstaller from setup_app.utils.setup_utils import SetupUtils @@ -29,19 +30,31 @@ def __init__(self): self.qchar = '`' if Config.rdbm_type in ('mysql', 'spanner') else '"' self.output_dir = os.path.join(Config.outputFolder, Config.rdbm_type) - def install(self): - self.local_install() - schema_files = [] + def prepare(self): + self.schema_files = [] self.gluu_attributes = [] - for schema_fn in ('gluu_schema.json', 'custom_schema.json'): + json_schema_files = ['gluu_schema.json', 'custom_schema.json'] + + if Config.installSaml: + edu_person_fn = ldif_utils.schema2json( + os.path.join(Config.staticFolder, 'opendj/96-eduperson.ldif'), + os.path.join(Config.install_dir, 'schema') + ) + json_schema_files.append(edu_person_fn) + + for schema_fn in json_schema_files: schema_full_path = os.path.join(Config.install_dir, 'schema', schema_fn) - schema_files.append(schema_full_path) + self.schema_files.append(schema_full_path) schema_ = base.readJsonFile(schema_full_path) self.gluu_attributes += schema_.get('attributeTypes', []) - self.create_tables(schema_files) + + def install(self): + self.prepare() + self.local_install() + self.create_tables(self.schema_files) self.create_subtables() self.import_ldif() self.create_indexes() @@ -75,10 +88,11 @@ def local_install(self): if Config.rdbm_type == 'mysql': result, conn = self.dbUtils.mysqlconnection(log=False) if not result: + time.sleep(5) sql_cmd_list = [ - "CREATE DATABASE {};\n".format(Config.rdbm_db), - "CREATE USER '{}'@'localhost' IDENTIFIED BY '{}';\n".format(Config.rdbm_user, Config.rdbm_password), - "GRANT ALL PRIVILEGES ON {}.* TO '{}'@'localhost';\n".format(Config.rdbm_db, Config.rdbm_user), + "CREATE DATABASE {}".format(Config.rdbm_db), + "CREATE USER '{}'@'localhost' IDENTIFIED BY '{}'".format(Config.rdbm_user, Config.rdbm_password), + "GRANT ALL PRIVILEGES ON {}.* TO '{}'@'localhost'".format(Config.rdbm_db, Config.rdbm_user), ] for cmd in sql_cmd_list: self.run("echo \"{}\" | mysql".format(cmd), shell=True) @@ -135,6 +149,8 @@ def create_tables(self, schema_files): for schema_fn in schema_files: schema = base.readJsonFile(schema_fn) for obj in schema['objectClasses']: + if 'sql' in obj and Config.installSaml and obj['names'][0] == 'gluuPerson': + obj['sql']['includeObjectClass'].append('eduPerson') all_schema[obj['names'][0]] = obj for attr in schema['attributeTypes']: all_attribs[attr['names'][0]] = attr @@ -377,7 +393,6 @@ def import_ldif(self): def rdbmProperties(self): if Config.rdbm_type in ('sql', 'mysql'): - self.server_time_zone() Config.rdbm_password_enc = self.obscure(Config.rdbm_password) self.renderTemplateInOut(Config.gluuRDBMProperties, Config.templateFolder, Config.configFolder) @@ -392,9 +407,6 @@ def rdbmProperties(self): self.renderTemplateInOut(Config.gluuSpannerProperties, Config.templateFolder, Config.configFolder) - def server_time_zone(self): - Config.templateRenderingDict['server_time_zone'] = 'UTC' + time.strftime("%z") - def create_folders(self): self.createDirs(Config.static_rdbm_dir) diff --git a/setup_app/test_data_loader.py b/setup_app/test_data_loader.py index cb3fb0a48..ee3048e51 100644 --- a/setup_app/test_data_loader.py +++ b/setup_app/test_data_loader.py @@ -121,7 +121,6 @@ def load_test_data(self): rendered_text = self.fomatWithDict(template_text, self.merge_dicts(Config.__dict__, Config.templateRenderingDict)) Config.templateRenderingDict['config_oxauth_test_spanner'] = rendered_text else: - base.current_app.RDBMInstaller.server_time_zone() template_text = self.readFile(os.path.join(self.template_base, 'oxauth/server/config-oxauth-test-sql.properties.nrnd')) rendered_text = self.fomatWithDict(template_text, self.merge_dicts(Config.__dict__, Config.templateRenderingDict)) Config.templateRenderingDict['config_oxauth_test_sql'] = rendered_text diff --git a/setup_app/utils/arg_parser.py b/setup_app/utils/arg_parser.py index cecab0029..c37693937 100644 --- a/setup_app/utils/arg_parser.py +++ b/setup_app/utils/arg_parser.py @@ -81,6 +81,10 @@ def arg_parser(): parser.add_argument('--generate-oxd-certificate', help="Generate certificate for oxd based on hostname", action='store_true') parser.add_argument('--shell', help="Drop into interactive shell before starting installation", action='store_true') parser.add_argument('--no-progress', help="Use simple progress", action='store_true') + parser.add_argument('-enable-script', action='append', help="inum of script to enable", required=False) + parser.add_argument('-ox-authentication-mode', help="Sets oxAuthenticationMode") + parser.add_argument('-ox-trust-authentication-mode', help="Sets oxTrustAuthenticationMode") + parser.add_argument('--gluu-scan-cert', help="Creates Gluu Scan Api keystore", action='store_true') # spanner options parser.add_argument('-spanner-project', help="Spanner project name") diff --git a/setup_app/utils/crypto64.py b/setup_app/utils/crypto64.py index 9e408f8cb..35dceb19e 100644 --- a/setup_app/utils/crypto64.py +++ b/setup_app/utils/crypto64.py @@ -130,6 +130,7 @@ def gen_cert(self, suffix, password, user='root', cn=None, truststore_fn=None): self.delete_key(alias, truststore_fn) self.import_cert_to_java_truststore(alias, public_certificate) + return key, csr, public_certificate def delete_key(self, alias, truststore_fn=None): if not truststore_fn: diff --git a/setup_app/utils/db_utils.py b/setup_app/utils/db_utils.py index d7be358c0..1e84b674c 100644 --- a/setup_app/utils/db_utils.py +++ b/setup_app/utils/db_utils.py @@ -319,14 +319,14 @@ def set_oxTrustConfApplication(self, entries): n1ql = 'UPDATE `{}` USE KEYS "configuration_oxtrust" SET `oxTrustConfApplication`={}'.format(self.default_bucket, oxTrustConfApplication_js) self.cbm.exec_query(n1ql) - def enable_script(self, inum): + def enable_script(self, inum, enable=True): if not Config.loadData: return if self.moddb == BackendTypes.LDAP: ldap_operation_result = self.ldap_conn.modify( 'inum={},ou=scripts,o=gluu'.format(inum), - {"oxEnabled": [ldap3.MODIFY_REPLACE, 'true']} + {"oxEnabled": [ldap3.MODIFY_REPLACE, str(enable).lower()]} ) self.log_ldap_result(ldap_operation_result) @@ -340,10 +340,10 @@ def enable_script(self, inum): dn = 'inum={},ou=scripts,o=gluu'.format(inum) table = self.get_spanner_table_for_dn(dn) if table: - self.spanner.update_data(table=table, columns=['doc_id', 'oxEnabled'], values=[[inum, True]]) + self.spanner.update_data(table=table, columns=['doc_id', 'oxEnabled'], values=[[inum, enable]]) elif self.moddb == BackendTypes.COUCHBASE: - n1ql = 'UPDATE `{}` USE KEYS "scripts_{}" SET oxEnabled=true'.format(self.default_bucket, inum) + n1ql = 'UPDATE `{}` USE KEYS "scripts_{}" SET oxEnabled={}'.format(self.default_bucket, inum, str(enable).lower()) self.cbm.exec_query(n1ql) def enable_service(self, service): diff --git a/setup_app/utils/properties_utils.py b/setup_app/utils/properties_utils.py index b9ed8a4b7..fd53cdbce 100644 --- a/setup_app/utils/properties_utils.py +++ b/setup_app/utils/properties_utils.py @@ -201,6 +201,13 @@ def load_properties(self, prop_file, no_update=[]): if p.get('ldap_install') == '0': p['ldap_install'] = InstallTypes.NONE + if p.get('enable-script'): + base.argsp.enable_script = p['enable-script'].split() + + base.argsp.ox_authentication_mode = p.get('ox-authentication-mode') + base.argsp.ox_trust_authentication_mode = p.get('ox-trust-authentication-mode') + base.argsp.gluu_scan_cert = True if p.get('gluu-scan-cert','').lower() == 'true' else False + properties_list = list(p.keys()) if 'oxtrust_admin_password' not in p: diff --git a/static/rdbm/sql_data_types.json b/static/rdbm/sql_data_types.json index 853d1db28..d0264e770 100644 --- a/static/rdbm/sql_data_types.json +++ b/static/rdbm/sql_data_types.json @@ -788,5 +788,13 @@ "spanner": { "type": "ARRAY" } + }, + "oxRegistrationConfiguration": { + "mysql": { + "type": "TEXT" + }, + "spanner": { + "type": "STRING(MAX)" + } } } \ No newline at end of file diff --git a/static/scripts/jetty10CompatibleWar.py b/static/scripts/jetty10CompatibleWar.py new file mode 100755 index 000000000..a7ef993df --- /dev/null +++ b/static/scripts/jetty10CompatibleWar.py @@ -0,0 +1,68 @@ +#!/usr/bin/python3 + +import os +import sys +import shutil +import argparse +import tempfile +import xml.etree.ElementTree as ET + +parser = argparse.ArgumentParser(description="This script makes Jetty 9.x war file compatible with Jetty 10.0.x or 11.0.x") +parser.add_argument('-in-file', help="Download server", required=True) +parser.add_argument('-out-file', help="Download server", required=True) +parser.add_argument('-jetty-version', help="Jetty version", default="10.0.x") +argsp = parser.parse_args() + + +def war_for_jetty10(war_file, out_file, jetty_version): + + with tempfile.TemporaryDirectory() as tmp_dir: + + unpack_dir = os.path.join(tmp_dir, 'unpacked') + + print("Unpacking {} to {}".format(war_file, unpack_dir)) + try: + shutil.unpack_archive(war_file, unpack_dir, format='zip') + except Exception as e: + print("Error unpacking file {}: {}".format(war_file, e)) + return + + jetty_env_fn = os.path.join(unpack_dir, 'WEB-INF/jetty-env.xml') + + if not os.path.exists(jetty_env_fn): + print("Can't find {} exting.".format(jetty_env_fn)) + return + + print("Modifying {}".format(jetty_env_fn)) + + tree = ET.parse(jetty_env_fn) + root = tree.getroot() + + for new in root.findall("New"): + for arg in new.findall("Arg"): + for ref in arg.findall("Ref"): + if ref.attrib.get('id') == 'webAppCtx': + ref.set('refid', 'webAppCtx') + ref.attrib.pop('id') + + jetty_web_fn = os.path.join(unpack_dir, 'WEB-INF/jetty-web.xml') + if os.path.exists(jetty_web_fn): + os.remove(jetty_web_fn) + + jetty_version_string = '_'.join(jetty_version.split('.')[:2]) + + xml_header = '\n\n'.format(jetty_version_string) + with open(jetty_env_fn, 'wb') as f: + f.write(b'\n') + f.write(xml_header.encode()) + f.write(ET.tostring(root,method='xml')) + + tmp_war_fn = os.path.join(tmp_dir, '{}.war'.format(os.urandom(6).hex())) + print("Packing {}".format(tmp_war_fn)) + shutil.make_archive(tmp_war_fn, format='zip', root_dir=unpack_dir) + print("Renaming pack to", out_file) + shutil.move(tmp_war_fn+'.zip', out_file) + + +if __name__ == '__main__': + war_for_jetty10(argsp.in_file, argsp.out_file, argsp.jetty_version) diff --git a/tools/ldap_to_mysql/README.md b/tools/ldap_to_mysql/README.md new file mode 100644 index 000000000..f900cae87 --- /dev/null +++ b/tools/ldap_to_mysql/README.md @@ -0,0 +1,40 @@ +# Gluu OpenDJ to MySQL Migration + +This script migrates data from OpenDJ to MySQL. Please note that this tool is experimental. +Note!: Test this script on non-production server. + +# Prerequisites + +* Insptall `python3-ldap` package. + + On RHEL-8/CentOS-8-AppStream: `yum install python3-ldap` + + On Ubuntun: `apt install python3-ldap` + +* Install and Configure MySQL + + Install MySQL that upports a native JSON data type (See https://dev.mysql.com/doc/refman/5.7/en/json.html). + MySQL Server shipped with >=Ubuntu 20 and >=RHEL 8/CentOS-8-Appstream is fine. + Create a database, namely `gluudb`, and create + a user, namely `gluu`. User should have all previleges on created database. Sample MySQL commands + (If you installed MySQL on a seperate server, modify commands accordingly): + + ``` + CREATE DATABASE gluudb; + CREATE USER 'gluu'@'localhost' IDENTIFIED BY 'TopSecret'; + GRANT ALL PRIVILEGES ON gluudb.* TO 'gluu'@'localhost'; + ``` + +# Download CE Setup Fixes and Migration Script + + - Download files: + ``` + wget https://raw.githubusercontent.com/GluuFederation/community-edition-setup/version_4.4.0/setup_app/installers/rdbm.py -O /install/community-edition-setup/setup_app/installers/rdbm.py + wget https://raw.githubusercontent.com/GluuFederation/community-edition-setup/version_4.4.0/tools/ldap_to_mysql/ldap2mysql.py -O /install/community-edition-setup/ldap2mysql.py + ``` + + - Execute script: + ``` + cd /install/community-edition-setup/ + python3 ldap2mysql.py -rdbm-user=gluu -rdbm-password=TopSecret -rdbm-db=gluudb -rdbm-host=localhost + ``` diff --git a/static/scripts/ldap2mysql.py b/tools/ldap_to_mysql/ldap2mysql.py old mode 100755 new mode 100644 similarity index 97% rename from static/scripts/ldap2mysql.py rename to tools/ldap_to_mysql/ldap2mysql.py index 04f0539c5..7f6c1c1f3 --- a/static/scripts/ldap2mysql.py +++ b/tools/ldap_to_mysql/ldap2mysql.py @@ -47,6 +47,7 @@ from setup_app.utils.properties_utils import PropertiesUtils from setup_app.installers.gluu import GluuInstaller from setup_app.utils.ldif_utils import myLdifParser +from setup_app.installers.opendj import OpenDjInstaller from setup_app.installers.rdbm import RDBMInstaller from setup_app.pylib.ldif4.ldif import LDIFWriter @@ -61,6 +62,7 @@ collectProperties = CollectProperties() collectProperties.collect() Config.installed_instance = True +openDjInstaller = OpenDjInstaller() rdbmInstaller = RDBMInstaller() propertiesUtils = PropertiesUtils() @@ -150,13 +152,14 @@ current_custom_schema_fn ] +rdbmInstaller.prepare() rdbmInstaller.dbUtils.read_gluu_schema() for a in rdbm_config_params: if argsp_dict[a]: setattr(Config, a, argsp_dict[a]) -Config.wrends_install = static.InstallTypes.NONE +Config.ldap_install = static.InstallTypes.NONE Config.rdbm_install = static.InstallTypes.REMOTE Config.mappingLocations = { group: 'rdbm' for group in Config.couchbaseBucketDict }