From d8297498ee9af9cf24331fb14935837d02ea26d3 Mon Sep 17 00:00:00 2001 From: Marco Fargetta Date: Fri, 22 Dec 2023 17:40:52 +0100 Subject: [PATCH] Convert ca-cert-find server command to paged search --- .../cmscore/dbs/CertificateRepository.java | 28 ++++ .../org/dogtagpki/server/ca/CAServlet.java | 2 +- .../server/ca/cli/CACertFindCLI.java | 39 ++---- .../dogtagpki/server/ca/v2/CAInfoServlet.java | 2 +- .../dogtagpki/server/ca/v2/CertServlet.java | 2 +- .../netscape/certsrv/dbs/DBPagedSearch.java | 36 +++++ .../cmscore/dbs/CertRecordPagedList.java | 77 +++++++++++ .../com/netscape/cmscore/dbs/DBSSession.java | 17 +++ .../netscape/cmscore/dbs/LDAPPagedSearch.java | 130 ++++++++++++++++++ .../com/netscape/cmscore/dbs/LDAPSession.java | 20 ++- 10 files changed, 321 insertions(+), 32 deletions(-) create mode 100644 base/server/src/main/java/com/netscape/certsrv/dbs/DBPagedSearch.java create mode 100644 base/server/src/main/java/com/netscape/cmscore/dbs/CertRecordPagedList.java create mode 100644 base/server/src/main/java/com/netscape/cmscore/dbs/LDAPPagedSearch.java diff --git a/base/ca/src/main/java/com/netscape/cmscore/dbs/CertificateRepository.java b/base/ca/src/main/java/com/netscape/cmscore/dbs/CertificateRepository.java index 538dc9095f7..6d2db2e8388 100644 --- a/base/ca/src/main/java/com/netscape/cmscore/dbs/CertificateRepository.java +++ b/base/ca/src/main/java/com/netscape/cmscore/dbs/CertificateRepository.java @@ -35,6 +35,7 @@ import com.netscape.certsrv.base.EBaseException; import com.netscape.certsrv.base.MetaInfo; import com.netscape.certsrv.base.SessionContext; +import com.netscape.certsrv.dbs.DBPagedSearch; import com.netscape.certsrv.dbs.DBVirtualList; import com.netscape.certsrv.dbs.EDBRecordNotFoundException; import com.netscape.certsrv.dbs.Modification; @@ -1218,6 +1219,33 @@ public Enumeration findCertRecords(String filter) return e; } + /** + * Finds a list of certificate records that satisifies + * the filter. + * + * @param filter search filter + * @param attrs selected attribute + * @param sortKey key to use for sorting the returned elements + * @return a list of certificates + * @exception EBaseException failed to search + */ + public CertRecordPagedList findPagedCertRecords(String filter, + String[] attrs, String sortKey) + throws EBaseException { + + logger.debug("CertificateRepository.findCertRecordsInList()"); + + try (DBSSession session = dbSubsystem.createSession()) { + DBPagedSearch page = session.createPagedSearch( + mBaseDN, + filter, + attrs, + sortKey); + + return new CertRecordPagedList(page); + } + } + /** * Finds certificate records. Here is a list of filter * attribute can be used: diff --git a/base/ca/src/main/java/org/dogtagpki/server/ca/CAServlet.java b/base/ca/src/main/java/org/dogtagpki/server/ca/CAServlet.java index 19ee3e139e4..bcd86347a41 100644 --- a/base/ca/src/main/java/org/dogtagpki/server/ca/CAServlet.java +++ b/base/ca/src/main/java/org/dogtagpki/server/ca/CAServlet.java @@ -14,7 +14,7 @@ import javax.servlet.http.HttpServletResponse; /** - * @author Marco Fargetta + * @author Marco Fargetta {@literal } */ public class CAServlet extends HttpServlet { public static final long serialVersionUID = 1L; diff --git a/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CACertFindCLI.java b/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CACertFindCLI.java index dc75fc88342..cbc10fe2c73 100644 --- a/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CACertFindCLI.java +++ b/base/ca/src/main/java/org/dogtagpki/server/ca/cli/CACertFindCLI.java @@ -15,7 +15,6 @@ import org.dogtagpki.cli.CLI; import org.dogtagpki.cli.CommandCLI; import org.dogtagpki.jss.tomcat.TomcatJSS; -import org.dogtagpki.server.ca.CAConfig; import org.dogtagpki.server.ca.CAEngineConfig; import org.dogtagpki.util.logging.PKILogger; import org.dogtagpki.util.logging.PKILogger.LogLevel; @@ -30,7 +29,7 @@ import com.netscape.cmscore.base.ConfigStorage; import com.netscape.cmscore.base.FileConfigStorage; import com.netscape.cmscore.dbs.CertRecord; -import com.netscape.cmscore.dbs.CertRecordList; +import com.netscape.cmscore.dbs.CertRecordPagedList; import com.netscape.cmscore.dbs.CertificateRepository; import com.netscape.cmscore.dbs.DBSubsystem; import com.netscape.cmscore.ldapconn.LDAPConfig; @@ -82,8 +81,6 @@ public void execute(CommandLine cmd) throws Exception { String filter = builder.buildFilter(); logger.info("- filter: " + filter); - int size = 20; - String instanceDir = CMS.getInstanceDir(); TomcatJSS tomcatjss = TomcatJSS.getInstance(); @@ -115,48 +112,40 @@ public void execute(CommandLine cmd) throws Exception { dbSubsystem.setEngineConfig(cs); dbSubsystem.init(dbConfig, ldapConfig, socketConfig, passwordStore); - CAConfig caConfig = cs.getCAConfig(); - logger.info("Initializing cert repository"); - int increment = caConfig.getInteger(CertificateRepository.PROP_INCREMENT, 5); - logger.info("- increment: " + increment); - try { CertificateRepository certificateRepository = new CertificateRepository(secureRandom, dbSubsystem); certificateRepository.init(); - CertRecordList list = certificateRepository.findCertRecordsInList(filter, null, "serialno", size); - int total = list.getSize(); - - for (int i = 0; i < total; i++) { - - if (i > 0) { + CertRecordPagedList certPages = certificateRepository.findPagedCertRecords(filter, null, "serialno"); + boolean follow = false; + for (CertRecord cRec: certPages) { + CertId id = new CertId(cRec.getSerialNumber()); + X509Certificate cert = cRec.getCertificate(); + if(follow) { System.out.println(); + } else { + follow = true; } - - CertRecord record = list.getCertRecord(i); - CertId id = new CertId(record.getSerialNumber()); - X509Certificate cert = record.getCertificate(); - System.out.println(" Serial Number: " + id.toHexString()); System.out.println(" Subject DN: " + cert.getSubjectDN()); System.out.println(" Issuer DN: " + cert.getIssuerDN()); - System.out.println(" Status: " + record.getStatus()); + System.out.println(" Status: " + cRec.getStatus()); System.out.println(" Not Valid Before: " + cert.getNotBefore()); System.out.println(" Not Valid After: " + cert.getNotAfter()); - System.out.println(" Issued On: " + record.getCreateTime()); - System.out.println(" Issued By: " + record.getIssuedBy()); + System.out.println(" Issued On: " + cRec.getCreateTime()); + System.out.println(" Issued By: " + cRec.getIssuedBy()); - Date revokedOn = record.getRevokedOn(); + Date revokedOn = cRec.getRevokedOn(); if (revokedOn != null) { System.out.println(" Revoked On: " + revokedOn); } - String revokedBy = record.getRevokedBy(); + String revokedBy = cRec.getRevokedBy(); if (revokedBy != null) { System.out.println(" Revoked By: " + revokedBy); } diff --git a/base/ca/src/main/java/org/dogtagpki/server/ca/v2/CAInfoServlet.java b/base/ca/src/main/java/org/dogtagpki/server/ca/v2/CAInfoServlet.java index 548fbd78761..0425e19e2e0 100644 --- a/base/ca/src/main/java/org/dogtagpki/server/ca/v2/CAInfoServlet.java +++ b/base/ca/src/main/java/org/dogtagpki/server/ca/v2/CAInfoServlet.java @@ -19,7 +19,7 @@ import org.slf4j.LoggerFactory; /** - * @author Marco Fargetta + * @author Marco Fargetta {@literal } */ @WebServlet("/v2/info") public class CAInfoServlet extends CAServlet { diff --git a/base/ca/src/main/java/org/dogtagpki/server/ca/v2/CertServlet.java b/base/ca/src/main/java/org/dogtagpki/server/ca/v2/CertServlet.java index 54ff2ecf876..946ca8198e2 100644 --- a/base/ca/src/main/java/org/dogtagpki/server/ca/v2/CertServlet.java +++ b/base/ca/src/main/java/org/dogtagpki/server/ca/v2/CertServlet.java @@ -55,7 +55,7 @@ import com.netscape.cmscore.dbs.RevocationInfo; /** - * @author Marco Fargetta + * @author Marco Fargetta {@literal } */ @WebServlet("/v2/certs/*") public class CertServlet extends CAServlet { diff --git a/base/server/src/main/java/com/netscape/certsrv/dbs/DBPagedSearch.java b/base/server/src/main/java/com/netscape/certsrv/dbs/DBPagedSearch.java new file mode 100644 index 00000000000..55ddee8a6f5 --- /dev/null +++ b/base/server/src/main/java/com/netscape/certsrv/dbs/DBPagedSearch.java @@ -0,0 +1,36 @@ +// --- BEGIN COPYRIGHT BLOCK --- +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; version 2 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +// +// (C) 2023 Red Hat, Inc. +// All rights reserved. +// --- END COPYRIGHT BLOCK --- +package com.netscape.certsrv.dbs; + +import java.util.List; + +import com.netscape.certsrv.base.EBaseException; +import com.netscape.cmscore.dbs.CertRecord; + +/** + * A class represents a paged search. + * + * @author Marco Fargetta {@literal } + */ +public abstract class DBPagedSearch { + + public abstract List getPage() throws EBaseException; + + public abstract List getPage(int size) throws EBaseException; + +} diff --git a/base/server/src/main/java/com/netscape/cmscore/dbs/CertRecordPagedList.java b/base/server/src/main/java/com/netscape/cmscore/dbs/CertRecordPagedList.java new file mode 100644 index 00000000000..1303c81dfa7 --- /dev/null +++ b/base/server/src/main/java/com/netscape/cmscore/dbs/CertRecordPagedList.java @@ -0,0 +1,77 @@ +// --- BEGIN COPYRIGHT BLOCK --- +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; version 2 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +// +// (C) 2023 Red Hat, Inc. +// All rights reserved. +// --- END COPYRIGHT BLOCK --- +package com.netscape.cmscore.dbs; + +import java.util.Iterator; +import java.util.List; + +import com.netscape.certsrv.base.EBaseException; +import com.netscape.certsrv.dbs.DBPagedSearch; + +/** +* Contain all records in a page for a paged search. +* +* @author Marco Fargetta {@literal } +*/ +public class CertRecordPagedList implements Iterable { + public static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(CertRecordPagedList.class); + + private DBPagedSearch pages; + private Iterator pageEntries; + /** + * Constructs a request paged. + */ + public CertRecordPagedList(DBPagedSearch pages) { + this.pages = pages; + try { + pageEntries = pages.getPage().iterator(); + } catch (EBaseException e) { + throw new RuntimeException("CertRecordPagedList: Error to get a new page", e); + } + } + + @Override + public Iterator iterator() { + return new CertRecordPageIterator(); + } + + class CertRecordPageIterator implements Iterator { + + @Override + public boolean hasNext() { + if (!pageEntries.hasNext()) { + try { + List newPage = pages.getPage(); + pageEntries = newPage.iterator(); + } catch (EBaseException e) { + throw new RuntimeException("CertRecordPagedList: Error to get a new page", e); + } + } + return pageEntries.hasNext(); + } + + @Override + public CertRecord next() { + if (hasNext()) { + return pageEntries.next(); + } + return null; + } + + } +} diff --git a/base/server/src/main/java/com/netscape/cmscore/dbs/DBSSession.java b/base/server/src/main/java/com/netscape/cmscore/dbs/DBSSession.java index 69501234ffe..db8f72d1c4a 100644 --- a/base/server/src/main/java/com/netscape/cmscore/dbs/DBSSession.java +++ b/base/server/src/main/java/com/netscape/cmscore/dbs/DBSSession.java @@ -18,6 +18,7 @@ package com.netscape.cmscore.dbs; import com.netscape.certsrv.base.EBaseException; +import com.netscape.certsrv.dbs.DBPagedSearch; import com.netscape.certsrv.dbs.DBVirtualList; import com.netscape.certsrv.dbs.EDBException; import com.netscape.certsrv.dbs.IDBObj; @@ -353,6 +354,22 @@ public DBVirtualList createVirtualList( return null; } + /** + * Retrieves a paged search of objects. + * + * @param base starting point of the search + * @param filter search filter + * @param attrs selected attributes + * @param startFrom starting point + * @param sortKey key used to sort the list + * @return search results in virtual list + * @exception EBaseException failed to search + */ + public DBPagedSearch createPagedSearch(String base, String filter, String[] attrs, + String sortKey) throws EBaseException { + return null; + } + public void abandon(LDAPSearchResults results) throws EBaseException { } } diff --git a/base/server/src/main/java/com/netscape/cmscore/dbs/LDAPPagedSearch.java b/base/server/src/main/java/com/netscape/cmscore/dbs/LDAPPagedSearch.java new file mode 100644 index 00000000000..23842fa953d --- /dev/null +++ b/base/server/src/main/java/com/netscape/cmscore/dbs/LDAPPagedSearch.java @@ -0,0 +1,130 @@ +// --- BEGIN COPYRIGHT BLOCK --- +// This program is free software; you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation; version 2 of the License. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License along +// with this program; if not, write to the Free Software Foundation, Inc., +// 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +// +// (C) 2023 Red Hat, Inc. +// All rights reserved. +// --- END COPYRIGHT BLOCK --- +package com.netscape.cmscore.dbs; + +import java.util.ArrayList; +import java.util.List; + +import com.netscape.certsrv.base.EBaseException; +import com.netscape.certsrv.dbs.DBPagedSearch; +import com.netscape.certsrv.dbs.EDBException; +import com.netscape.certsrv.dbs.EDBNotAvailException; +import com.netscape.certsrv.dbs.IDBObj; +import com.netscape.cmscore.apps.CMS; + +import netscape.ldap.LDAPConnection; +import netscape.ldap.LDAPControl; +import netscape.ldap.LDAPException; +import netscape.ldap.LDAPSearchConstraints; +import netscape.ldap.LDAPSearchResults; +import netscape.ldap.LDAPSortKey; +import netscape.ldap.LDAPv3; +import netscape.ldap.controls.LDAPPagedResultsControl; +import netscape.ldap.controls.LDAPSortControl; + +/** + * Perform consecutive paged search until entries are available. + * + * @author Marco Fargetta {@literal } + */ +public class LDAPPagedSearch extends DBPagedSearch { + public static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LDAPPagedSearch.class); + + private DBRegistry registry; + private LDAPConnection conn = null; + private String base = null; + private String filter = null; + private String[] attrs = null; + private String sortKey = null; + private LDAPSearchResults res = null; + + public LDAPPagedSearch(DBRegistry registry, LDAPConnection conn, String base, String filter, String[] attrs, + String sortKey) throws EBaseException { + this.registry = registry; + this.base = base; + this.filter = filter; + this.attrs = attrs; + this.sortKey = sortKey; + try { + this.conn = (LDAPConnection) conn.clone(); + } catch (Exception e) { + throw new EBaseException(CMS.getUserMessage("CMS_BASE_CONN_FAILED", + e.toString()), e); + } + } + + @Override + public List getPage() + throws EBaseException { + return getPage(LDAPSession.MAX_PAGED_SEARCH_SIZE); + } + + @Override + public List getPage(int size) + throws EBaseException { + try { + logger.info("LDAPSession.continuousPagedSearch(): Searching {} for {}", base, filter); + + LDAPSearchConstraints cons = new LDAPSearchConstraints(); + LDAPPagedResultsControl pagecon; + String[] ldapattrs = null; + if (attrs != null) { + ldapattrs = registry.getLDAPAttributes(attrs); + } + + if(sortKey != null) { + LDAPSortKey sortOrder = new LDAPSortKey( sortKey ); + LDAPSortControl sortCtrl = new LDAPSortControl(sortOrder,true); + cons.setServerControls( sortCtrl ); + } + String ldapfilter = registry.getFilter(filter); + + byte[] cookie = null; + ArrayList entries = new ArrayList<>(); + if (res != null) { + for (LDAPControl c: res.getResponseControls()){ + if(c instanceof LDAPPagedResultsControl resC){ + cookie = resC.getCookie(); + } + } + if (cookie == null) { + conn.close(); + return entries; + } + } + if (res == null) { + pagecon = new LDAPPagedResultsControl(false, Math.min(size, LDAPSession.MAX_PAGED_SEARCH_SIZE)); + } else { + pagecon = new LDAPPagedResultsControl(false, Math.min(size, LDAPSession.MAX_PAGED_SEARCH_SIZE), cookie); + } + cons.setServerControls(pagecon); + res = conn.search(base, + LDAPv3.SCOPE_ONE, ldapfilter, ldapattrs, false, cons); + DBSearchResults sr = new DBSearchResults(registry, res); + while (sr.hasMoreElements()) { + entries.add((CertRecord) sr.nextElement()); + } + return entries; + } catch (LDAPException e) { + if (e.getLDAPResultCode() == LDAPException.UNAVAILABLE) + throw new EDBNotAvailException( + CMS.getUserMessage("CMS_DBS_INTERNAL_DIR_UNAVAILABLE"), e); + throw new EDBException("Unable to search LDAP record: " + e.getMessage(), e); + } + } +} diff --git a/base/server/src/main/java/com/netscape/cmscore/dbs/LDAPSession.java b/base/server/src/main/java/com/netscape/cmscore/dbs/LDAPSession.java index fd5f9a36a68..3a188ba6964 100644 --- a/base/server/src/main/java/com/netscape/cmscore/dbs/LDAPSession.java +++ b/base/server/src/main/java/com/netscape/cmscore/dbs/LDAPSession.java @@ -20,6 +20,7 @@ import java.util.Enumeration; import com.netscape.certsrv.base.EBaseException; +import com.netscape.certsrv.dbs.DBPagedSearch; import com.netscape.certsrv.dbs.DBVirtualList; import com.netscape.certsrv.dbs.EDBException; import com.netscape.certsrv.dbs.EDBNotAvailException; @@ -57,7 +58,9 @@ */ public class LDAPSession extends DBSSession { - public final static org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LDAPSession.class); + public static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(LDAPSession.class); + + public static final int MAX_PAGED_SEARCH_SIZE = 500; private DBSubsystem dbSubsystem; private LDAPConnection mConn = null; @@ -447,7 +450,7 @@ public DBSearchResults search(String base, String filter, String ldapfilter = dbSubsystem.getRegistry().getFilter(filter); logger.info("LDAPSession: Searching " + base + " for " + ldapfilter); - String ldapattrs[] = null; + String[] ldapattrs = null; if (attrs != null) { ldapattrs = dbSubsystem.getRegistry().getLDAPAttributes(attrs); } @@ -490,7 +493,7 @@ public DBSearchResults pagedSearch(String base, String filter, int start, int si LDAPPagedResultsControl pagecon; LDAPSearchResults res; int skipped = 0; - int pageSize = start > 0 ? Math.min(start, 500) : Math.min(size, 500); + int pageSize = start > 0 ? Math.min(start, MAX_PAGED_SEARCH_SIZE) : Math.min(size, MAX_PAGED_SEARCH_SIZE); byte[] cookie = null; pagecon = new LDAPPagedResultsControl(false, pageSize); @@ -505,7 +508,7 @@ public DBSearchResults pagedSearch(String base, String filter, int start, int si if(c instanceof LDAPPagedResultsControl resC){ cookie = resC.getCookie(); if(cookie!=null){ - pageSize = start - skipped > 0 ? Math.min((start - skipped) , 500) : Math.min(size, 500); + pageSize = start - skipped > 0 ? Math.min((start - skipped) , 500) : Math.min(size, MAX_PAGED_SEARCH_SIZE); pagecon = new LDAPPagedResultsControl(false, pageSize, cookie); } else { res = new LDAPSearchResults(); @@ -670,6 +673,15 @@ public DBVirtualList createVirtualList(String base, String } + @Override + public DBPagedSearch createPagedSearch(String base, String filter, + String attrs[], String sortKey) throws EBaseException { + logger.debug("LDAPSession: createPagedSearch({}, {})", base, filter); + + return new LDAPPagedSearch<>(dbSubsystem.getRegistry(),mConn, base, + filter, attrs, sortKey); + } + /** * Releases object to this interface. This allows us to * use memory more efficiently.