From 9f5a1a85cbbb42dc070edfc74e3cf961a89f98d5 Mon Sep 17 00:00:00 2001 From: David Han Date: Wed, 29 Jul 2026 00:07:18 -0500 Subject: [PATCH 1/4] Knox-3386: LDAP Proxy pages backends Adds a gateway.ldap.max.size.limit and max.time.limit configs to the LDAP Proxy. Thes configs are uses to set the LDAP server's maxSizeLimit and maxTimeLimit fields. The size limit indictes the maximum total number of search results for a request across all pages. The time limit is in milliseconds. Implements paging to the backend LDAP servers. This is controlled by a new "pageSize" config on the LDAP Proxy Backends. A bug was found during testing where the getUserGroups method will doubly retrieve the groups. This has been fixed. --- .github/workflows/build/gateway-site.xml | 13 ++ .../single-eku-no-mtls/gateway-site.xml | 13 ++ .../compose/single-eku/gateway-site.xml | 13 ++ .../config/impl/GatewayConfigImpl.java | 27 ++- .../services/ldap/KnoxLDAPServerManager.java | 11 +- .../gateway/services/ldap/LdapMessages.java | 12 +- .../ldap/backend/LdapProxyBackend.java | 155 +++++++++++------- .../src/main/resources/conf/gateway-site.xml | 12 ++ .../ldap/KnoxLDAPServerManagerTest.java | 20 +++ .../services/ldap/KnoxLDAPServiceTest.java | 2 + .../ldap/backend/LdapProxyBackendSslTest.java | 4 +- .../ldap/backend/LdapProxyBackendTest.java | 125 ++++++++++++-- .../resources/ldap-proxy-backend-test.ldif | 19 +++ .../knox/gateway/GatewayTestConfig.java | 10 ++ .../knox/gateway/config/GatewayConfig.java | 12 ++ knox-site/docs/service_ldap_server.md | 3 + 16 files changed, 366 insertions(+), 85 deletions(-) diff --git a/.github/workflows/build/gateway-site.xml b/.github/workflows/build/gateway-site.xml index 00698bebdb..002fadea5d 100644 --- a/.github/workflows/build/gateway-site.xml +++ b/.github/workflows/build/gateway-site.xml @@ -145,6 +145,14 @@ limitations under the License. gateway.ldap.base.dn dc=proxy,dc=org + + gateway.ldap.max.size.limit + 1000 + + + gateway.ldap.max.time.limit + 60000 + gateway.ldap.recursive.group.resolution true @@ -211,5 +219,10 @@ limitations under the License. gateway.ldap.interceptor.demoldap.groupMemberAttribute member + + + gateway.ldap.interceptor.demoldap.pageSize + 3 + diff --git a/.github/workflows/compose/single-eku-no-mtls/gateway-site.xml b/.github/workflows/compose/single-eku-no-mtls/gateway-site.xml index 94d613ff94..791a3b2a5d 100644 --- a/.github/workflows/compose/single-eku-no-mtls/gateway-site.xml +++ b/.github/workflows/compose/single-eku-no-mtls/gateway-site.xml @@ -157,6 +157,14 @@ limitations under the License. gateway.ldap.base.dn dc=proxy,dc=org + + gateway.ldap.max.size.limit + 1000 + + + gateway.ldap.max.time.limit + 60000 + gateway.ldap.recursive.group.resolution true @@ -203,5 +211,10 @@ limitations under the License. gateway.ldap.interceptor.demoldap.groupMemberAttribute member + + + gateway.ldap.interceptor.demoldap.pageSize + 3 + diff --git a/.github/workflows/compose/single-eku/gateway-site.xml b/.github/workflows/compose/single-eku/gateway-site.xml index 4272c7b739..76f488b85f 100644 --- a/.github/workflows/compose/single-eku/gateway-site.xml +++ b/.github/workflows/compose/single-eku/gateway-site.xml @@ -211,6 +211,14 @@ limitations under the License. gateway.ldap.base.dn dc=proxy,dc=org + + gateway.ldap.max.size.limit + 1000 + + + gateway.ldap.max.time.limit + 60000 + gateway.ldap.recursive.group.resolution true @@ -257,5 +265,10 @@ limitations under the License. gateway.ldap.interceptor.demoldap.groupMemberAttribute member + + + gateway.ldap.interceptor.demoldap.pageSize + 3 + diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java index 061518537d..d0e34d0fc2 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java @@ -392,6 +392,13 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig { public static final String STRICT_TRANSPORT_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".strict.transport.enabled"; public static final String STRICT_TRANSPORT_OPTION = GATEWAY_CONFIG_FILE_PREFIX + ".strict.transport.option"; + // Gateway LDAP Properties + public static final int DEFAULT_LDAP_PORT = 3890; + public static final String DEFAULT_LDAP_BASE_DN = "dc=proxy,dc=com"; + public static final int DEFAULT_LDAP_MAX_SIZE_LIMIT = 1000; + /* The default max time for LDAP search in milliseconds */ + public static final int DEFAULT_LDAP_MAX_TIME_LIMIT = 60 * 1000; + public GatewayConfigImpl() { init(); } @@ -1775,17 +1782,17 @@ public String getStrictTransportOption() { // LDAP Service Configuration @Override public boolean isLDAPEnabled() { - return Boolean.parseBoolean(get(LDAP_ENABLED, "false")); + return getBoolean(LDAP_ENABLED, false); } @Override public int getLDAPPort() { - return Integer.parseInt(get(LDAP_PORT, "3890")); + return getInt(LDAP_PORT, DEFAULT_LDAP_PORT); } @Override public String getLDAPBaseDN() { - return get(LDAP_BASE_DN, "dc=proxy,dc=com"); + return get(LDAP_BASE_DN, DEFAULT_LDAP_BASE_DN); } @Override @@ -1840,7 +1847,7 @@ public Map getLDAPInterceptorConfig(String interceptorName) { @Override public boolean isLDAPRecursiveGroupResolutionEnabled() { - return Boolean.parseBoolean(get(LDAP_RECURSIVE_GROUP_RESOLUTION, "false")); + return getBoolean(LDAP_RECURSIVE_GROUP_RESOLUTION, false); } @Override @@ -1865,7 +1872,7 @@ public String getLdapRolesLookupFilePath() { @Override public boolean isLDAPSSLEnabled() { - return Boolean.parseBoolean(get(LDAP_SSL_ENABLED, "false")); + return getBoolean(LDAP_SSL_ENABLED, false); } @Override @@ -1884,6 +1891,16 @@ public List getLDAPSSLEnabledCipherSuites() { return cipherSuites == null ? Collections.emptyList() : cipherSuites; } + @Override + public int getLDAPMaxSizeLimit() { + return getInt(LDAP_MAX_SIZE_LIMIT, DEFAULT_LDAP_MAX_SIZE_LIMIT); + } + + @Override + public int getLDAPMaxTimeLimit() { + return getInt(LDAP_MAX_TIME_LIMIT, DEFAULT_LDAP_MAX_TIME_LIMIT); + } + @Override public boolean getGroupUIServicesOnHomepage() { return getBoolean(KNOX_HOMEPAGE_GROUP_UI_SERVICES, DEFAULT_GROUP_UI_SERVICES); diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java index 3e42c2117c..1fbabf3204 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java @@ -72,7 +72,8 @@ public class KnoxLDAPServerManager { @VisibleForTesting DirectoryService directoryService; - private LdapServer ldapServer; + @VisibleForTesting + LdapServer ldapServer; private GatewayConfig gatewayConfig; private List interceptors; private boolean hasRolesLookupInterceptor; @@ -87,6 +88,8 @@ public class KnoxLDAPServerManager { private List sslEnabledCipherSuites; // Collection of DNs for the proxied backend LDAP servers private Set baseDns; + private int maxSizeLimit; + private int maxTimeLimit; KnoxLDAPServerManager(AliasService aliasService) { this(aliasService, null); @@ -137,6 +140,9 @@ public void initialize(GatewayConfig config) throws Exception { } workDir.mkdirs(); + + maxSizeLimit = config.getLDAPMaxSizeLimit(); + maxTimeLimit = config.getLDAPMaxTimeLimit(); } private void createInterceptors(GatewayConfig config) throws Exception { @@ -241,6 +247,9 @@ public void start() throws Exception { ldapServer.setTransports(transport); ldapServer.setDirectoryService(directoryService); + ldapServer.setMaxSizeLimit(maxSizeLimit); + ldapServer.setMaxTimeLimit(maxTimeLimit); + ldapServer.start(); LOG.ldapServiceStarted(port); diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java index 816fef0035..7900566577 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java @@ -101,6 +101,14 @@ public interface LdapMessages { text = "LDAP Search: {0} | {1}") void ldapSearch(String baseDn, String filter); + @Message(level = MessageLevel.DEBUG, + text = "LDAP Paged Search: {0} | {1}, page size {2}, page {3}") + void ldapPagedSearch(String baseDn, String filter, int pageSize, int pageNumber); + + @Message(level = MessageLevel.DEBUG, + text = "LDAP Paged Search Completed: {0} | {1}") + void ldapPagedSearchCompleted(String baseDn, String filter); + @Message(level = MessageLevel.ERROR, text = "LDAP Search failed: {0} | {1}, {2}") void ldapSearchFailed(String baseDn, String filter, @StackTrace(level = MessageLevel.DEBUG) Exception e); @@ -133,9 +141,9 @@ public interface LdapMessages { text = "Backend user not found: {0}") void ldapUserNull(String username); - @Message(level = MessageLevel.ERROR, + @Message(level = MessageLevel.DEBUG, text = "Failed to copy attribute: {0}") - void ldapAttributeCopyError(@StackTrace(level = MessageLevel.DEBUG) Exception e); + void ldapAttributeCopyError(@StackTrace(level = MessageLevel.TRACE) Exception e); @Message(level = MessageLevel.DEBUG, text = "LDAP authentication succeeded for user: {0}") void ldapAuthSucceeded(String user); diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java index 0c56168520..8246c4d36a 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java @@ -21,12 +21,20 @@ import org.apache.directory.api.ldap.model.cursor.CursorException; import org.apache.directory.api.ldap.model.cursor.EntryCursor; +import org.apache.directory.api.ldap.model.cursor.SearchCursor; import org.apache.directory.api.ldap.model.entry.Attribute; import org.apache.directory.api.ldap.model.entry.DefaultEntry; import org.apache.directory.api.ldap.model.entry.Entry; import org.apache.directory.api.ldap.model.entry.Value; import org.apache.directory.api.ldap.model.exception.LdapException; +import org.apache.directory.api.ldap.model.message.Response; +import org.apache.directory.api.ldap.model.message.SearchRequest; +import org.apache.directory.api.ldap.model.message.SearchRequestImpl; +import org.apache.directory.api.ldap.model.message.SearchResultDone; +import org.apache.directory.api.ldap.model.message.SearchResultEntry; import org.apache.directory.api.ldap.model.message.SearchScope; +import org.apache.directory.api.ldap.model.message.controls.PagedResults; +import org.apache.directory.api.ldap.model.message.controls.PagedResultsImpl; import org.apache.directory.api.ldap.model.name.Dn; import org.apache.directory.api.ldap.model.schema.SchemaManager; import org.apache.directory.ldap.client.api.DefaultLdapConnectionFactory; @@ -50,7 +58,6 @@ import java.security.cert.X509Certificate; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collection; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -96,6 +103,7 @@ public class LdapProxyBackend implements LdapBackend { private boolean useMemberOf; // Use memberOf attribute for group lookup (efficient for AD) private boolean recursiveGroupResolution; private int recursiveGroupResolutionMaxDepth; + private int pageSize; private final String proxyEntryGroupMembershipAttributeType = "memberOf"; @@ -184,6 +192,9 @@ public LdapProxyBackend(String name, Map config) { recursiveGroupResolution = Boolean.parseBoolean(config.getOrDefault("recursiveGroupResolution", "false")); recursiveGroupResolutionMaxDepth = Integer.parseInt(config.getOrDefault("recursiveGroupResolutionMaxDepth", "3")); + // Configure search parameters + pageSize = Integer.parseInt(config.getOrDefault("pageSize", "1000")); + // Configure secure transport (LDAPS) to the remote server. An ldaps:// URL enables it // by default; an explicit useSsl setting always wins. final boolean ldapsFromUrl = ldapUrl != null && ldapUrl.toLowerCase(Locale.ROOT).startsWith("ldaps://"); @@ -490,15 +501,14 @@ public List getUserGroups(String username, SchemaManager schemaManager) return List.of(); } - LdapConnection connection = null; - try { - connection = getConnection(); - List groups = getUserGroupsEntries(connection, user, createEntryCache(), createResolvedParentsCache()); - List cns = getCnsFromEntries(groups); - return cns; - } finally { - releaseConnection(connection); + List groups = new ArrayList<>(); + Attribute groupsAttribute = user.get(proxyEntryGroupMembershipAttributeType); + if (groupsAttribute != null) { + for (Value value : groupsAttribute) { + groups.add(new Dn(value.getString()).getRdn().getValue()); + } } + return groups; } @Override @@ -511,12 +521,10 @@ public List searchUsers(String filter, SchemaManager schemaManager) throw try { connection = getConnection(); String ldapFilter = "(" + remoteUserIdentifierAttribute + "=" + filter.trim() + ")"; - try (EntryCursor cursor = connection.search(remoteUserSearchBase, ldapFilter, SearchScope.SUBTREE, "*")) { - while (cursor.next()) { - Entry sourceEntry = cursor.get(); - addGroupMemberships(sourceEntry, connection, entryCache, resolvedParentsCache); - results.add(remoteSchemaConverter.convertRemoteEntryToProxyEntry(sourceEntry, schemaManager)); - } + List searchResults = performPagedSearch(connection, remoteUserSearchBase, ldapFilter, SearchScope.SUBTREE, "*"); + for (Entry sourceEntry : searchResults) { + addGroupMemberships(sourceEntry, connection, entryCache, resolvedParentsCache); + results.add(remoteSchemaConverter.convertRemoteEntryToProxyEntry(sourceEntry, schemaManager)); } return results; } finally { @@ -535,14 +543,10 @@ public List search(String searchBase, SearchScope searchScope, String fil try { connection = getConnection(); List results = new ArrayList<>(); - try (EntryCursor cursor = connection.search(remoteSearchBase, remoteFilter, searchScope, "*")) { - while (cursor.next()) { - Entry entry = cursor.get(); - addGroupMemberships(entry, connection, entryCache, resolvedParentsCache); - results.add(remoteSchemaConverter.convertRemoteEntryToProxyEntry(entry, schemaManager)); - } - } catch (LdapException e) { - LOG.ldapSearchFailed(remoteSearchBase, remoteFilter, e); + List searchResults = performPagedSearch(connection, remoteSearchBase, remoteFilter, searchScope, "*"); + for (Entry entry : searchResults) { + addGroupMemberships(entry, connection, entryCache, resolvedParentsCache); + results.add(remoteSchemaConverter.convertRemoteEntryToProxyEntry(entry, schemaManager)); } return results; } finally { @@ -721,21 +725,19 @@ private List resolveGroupsRecursive(LdapConnection connection, List searchResults = performPagedSearch(connection, remoteGroupSearchBase, filter, SearchScope.SUBTREE, "cn", "memberUid", "member", "uniqueMember"); + for (Entry parentGroup : searchResults) { + String parentDn = parentGroup.getDn().getNormName(); + + // Update cache for all groups found in this search + updateCache(entryCache, resolvedParentsCache, groupsToSearch, parentGroup); + + if (!allGroupDns.contains(parentDn)) { + allGroupDns.add(parentDn); + allGroups.add(parentGroup); + nextLevelGroups.add(parentGroup); + } else { + LOG.ldapRecursiveGroupSearchCycleDetected(entryName, parentDn); } } @@ -839,15 +841,67 @@ private List getUserGroupsInternal(LdapConnection connection, Dn... dns) String filter = buildMultipleGroupMemberFilter(dns); - try (EntryCursor cursor = connection.search(remoteGroupSearchBase, filter, SearchScope.SUBTREE, "cn")) { - while (cursor.next()) { - groups.add(cursor.get()); - } - } + groups.addAll(performPagedSearch(connection, remoteGroupSearchBase, filter, SearchScope.SUBTREE, "cn")); return groups; } + protected List performPagedSearch(LdapConnection connection, String baseDn, String filter, SearchScope scope, String... attributes ) throws LdapException, CursorException, IOException { + List results = new ArrayList<>(); + + // 1. Setup basic search parameters + SearchRequest searchRequest = new SearchRequestImpl(); + searchRequest.setBase(new Dn(baseDn)); + searchRequest.setFilter(filter); + searchRequest.setScope(scope); + searchRequest.addAttributes(attributes); + + // 2. Initialize the PagedResults control + PagedResults pagedControl = new PagedResultsImpl(); + pagedControl.setSize(pageSize); + searchRequest.addControl(pagedControl); + + byte[] cookie = null; + + // 3. Loop until no more pages remain + int pageNumber = 1; + do { + // Update cookie for the subsequent pages + if (cookie != null) { + pagedControl.setCookie(cookie); + } + + try (SearchCursor cursor = connection.search(searchRequest)) { + LOG.ldapPagedSearch(baseDn, filter, pageSize, pageNumber); + while (cursor.next()) { + Response response = cursor.get(); + + // Process matching entries + if (response instanceof SearchResultEntry) { + Entry entry = ((SearchResultEntry) response).getEntry(); + results.add(entry); + } + } + if (cursor.isDone()) { + SearchResultDone done = cursor.getSearchResultDone(); + PagedResults responseControl = (PagedResults) done.getControl(PagedResults.OID); + + if (responseControl != null) { + cookie = responseControl.getCookie(); + } else { + cookie = null; + } + } + pageNumber++; + } catch (LdapException e) { + LOG.ldapSearchFailed(baseDn, filter, e); + } + } while (cookie != null && cookie.length > 0); + LOG.ldapPagedSearchCompleted(baseDn, filter); + + return results; + } + private String buildMultipleGroupMemberFilter(Dn... dns) { StringBuilder filterBuilder = new StringBuilder(); filterBuilder.append("(|"); @@ -877,21 +931,6 @@ private String buildMultipleGroupMemberFilter(Dn... dns) { return filterBuilder.toString(); } - private List getCnsFromEntries(Collection entries) throws LdapException { - List cns = new ArrayList<>(); - for (Entry entry : entries) { - Attribute cnAttr = entry.get("cn"); - if (cnAttr != null) { - cns.add(cnAttr.getString()); - } else if (entry.getDn() != null && entry.getDn().getRdn() != null) { - // Fall back to the CN carried in the DN when the entry was fetched without the - // cn attribute, so resolved groups are not silently dropped from the result. - cns.add(entry.getDn().getRdn().getValue()); - } - } - return cns; - } - protected Map createEntryCache() { return new HashMap<>(); } diff --git a/gateway-server/src/main/resources/conf/gateway-site.xml b/gateway-server/src/main/resources/conf/gateway-site.xml index fda674c179..f549c413f7 100644 --- a/gateway-server/src/main/resources/conf/gateway-site.xml +++ b/gateway-server/src/main/resources/conf/gateway-site.xml @@ -56,6 +56,18 @@ limitations under the License. Base DN for LDAP entries in the proxy server. Default is dc=proxy,dc=com. + + gateway.ldap.max.size.limit + 1000 + Maximum number of entries returned by a search request. + + + + gateway.ldap.max.time.limit + 60000 + Maximum time for a search request in milliseconds. + + gateway.ldap.recursive.group.resolution false diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java index 327d701c30..9e0877f96c 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java @@ -465,6 +465,26 @@ public void testGetUserGroupsIgnoresBareRdnWhenRolesLookupInactive() throws Exce List.of("analysts"), groups); } + public void testStartSetsMaxSizeAndTime() throws Exception { + final int expectedMaxSize = 3158; + final int expectedMaxTime = 245000; + + GatewayConfig mockConfig = EasyMock.createNiceMock(GatewayConfig.class); + expect(mockConfig.getGatewayDataDir()).andReturn(tempWorkDir.getParent()).anyTimes(); + expect(mockConfig.getLDAPPort()).andReturn(port).anyTimes(); + expect(mockConfig.getLDAPBaseDN()).andReturn("dc=test,dc=com").anyTimes(); + expect(mockConfig.getLDAPInterceptorNames()).andReturn(List.of()).anyTimes(); + expect(mockConfig.getLDAPMaxSizeLimit()).andReturn(expectedMaxSize).anyTimes(); + expect(mockConfig.getLDAPMaxTimeLimit()).andReturn(expectedMaxTime).anyTimes(); + replay(mockConfig); + + serverManager.initialize(mockConfig); + serverManager.start(); + + assertEquals(expectedMaxSize, serverManager.ldapServer.getMaxSizeLimit()); + assertEquals(expectedMaxTime, serverManager.ldapServer.getMaxTimeLimit()); + } + @Test(expected = LdapException.class) public void testBindRequiredRejectsAnonymous() throws Exception { useBindPassword(BIND_PASSWORD); diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServiceTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServiceTest.java index ce681a5617..f32c036f84 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServiceTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServiceTest.java @@ -180,6 +180,8 @@ private void setupMockConfig(String backendType) throws Exception { expect(mockConfig.getLDAPBindUser()).andReturn(null).anyTimes(); expect(mockConfig.getLDAPInterceptorNames()).andReturn(List.of("testbackend")).atLeastOnce(); expect(mockConfig.getLDAPInterceptorConfig("testbackend")).andReturn(buildBackendConfig(backendType)).atLeastOnce(); + expect(mockConfig.getLDAPMaxSizeLimit()).andReturn(1000).atLeastOnce(); + expect(mockConfig.getLDAPMaxTimeLimit()).andReturn(60000).atLeastOnce(); replay(mockConfig); } diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendSslTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendSslTest.java index 96b8e5a8f3..3ae14d812f 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendSslTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendSslTest.java @@ -158,7 +158,8 @@ public void testGetUserOverLdaps() throws Exception { assertEquals("ldaptest1", entry.get("uid").getString()); validateMemberOf(entry, Set.of( "cn=group1,ou=groups,dc=hadoop,dc=apache,dc=org", - "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org")); + "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org", + "cn=group3,ou=groups,dc=hadoop,dc=apache,dc=org")); } @Test @@ -168,6 +169,7 @@ public void testGetUserGroupsOverLdaps() throws Exception { List groups = ldapProxyBackend.getUserGroups("ldaptest1", schemaManager); assertTrue(groups.contains("group1")); assertTrue(groups.contains("group2")); + assertTrue(groups.contains("group3")); } @Test diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendTest.java index 58432f1374..dec22d5fe9 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendTest.java @@ -25,6 +25,7 @@ import org.apache.directory.api.ldap.model.entry.Entry; import org.apache.directory.api.ldap.model.entry.Value; +import org.apache.directory.api.ldap.model.message.SearchRequest; import org.apache.directory.api.ldap.model.message.SearchScope; import org.apache.directory.api.ldap.model.name.Dn; import org.apache.directory.api.ldap.model.schema.SchemaManager; @@ -37,6 +38,8 @@ import org.apache.directory.server.core.factory.PartitionFactory; import org.apache.directory.server.core.partition.ldif.LdifPartition; import org.apache.directory.server.ldap.LdapServer; +import org.apache.directory.server.ldap.LdapSession; +import org.apache.directory.server.ldap.handlers.LdapRequestHandler; import org.apache.directory.server.protocol.shared.store.LdifFileLoader; import org.apache.directory.server.protocol.shared.transport.TcpTransport; import org.apache.knox.gateway.security.ldap.SimpleDirectoryService; @@ -47,6 +50,8 @@ import org.junit.Test; import java.io.File; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -57,12 +62,15 @@ import java.util.concurrent.atomic.AtomicInteger; public class LdapProxyBackendTest { + private static final int PAGE_SIZE = 2; + private static Map ldapBackendConfig; private static TcpTransport transport; private static DirectoryService directoryService; private static LdapServer ldapServer; private static SchemaManager schemaManager; + private static CapturingSearchRequestHandler capturingSearchRequestHandler; private LdapProxyBackend ldapProxyBackend; @@ -110,10 +118,19 @@ public static void setupBeforeClass() throws Exception { // Create and start the LDAP server ldapServer = new LdapServer(); + ldapServer.setTransports(transport); ldapServer.setDirectoryService(directoryService); + ldapServer.start(); + capturingSearchRequestHandler = new CapturingSearchRequestHandler(ldapServer.getSearchRequestHandler()); + ldapServer.setSearchHandlers( + capturingSearchRequestHandler, + ldapServer.getSearchResultEntryHandler(), + ldapServer.getSearchResultReferenceHandler(), + ldapServer.getSearchResultDoneHandler()); + // Setup common backend config values for tests ldapBackendConfig = Map.of( "baseDn", "dc=hadoop,dc=apache,dc=org", @@ -143,6 +160,7 @@ public static void tearDownAfterClass() throws Exception { @After public void tearDown() throws Exception { + capturingSearchRequestHandler.reset(); if (ldapProxyBackend != null) { ldapProxyBackend.close(); } @@ -157,7 +175,8 @@ public void testGetUserByDefaultUserSearchFilter() throws Exception { validateUserEntry(entry, "ldaptest1", "TestCn1", "ldaptest1@example.com", "Test user ldaptest1"); validateMemberOf(entry, Set.of( "cn=group1,ou=groups,dc=hadoop,dc=apache,dc=org", - "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org")); + "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org", + "cn=group3,ou=groups,dc=hadoop,dc=apache,dc=org")); } @Test @@ -177,7 +196,8 @@ public void testGetUserByUID() throws Exception { validateUserEntry(entry, "ldaptest1", "TestCn1", "ldaptest1@example.com", "Test user ldaptest1"); validateMemberOf(entry, Set.of( "cn=group1,ou=groups,dc=hadoop,dc=apache,dc=org", - "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org")); + "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org", + "cn=group3,ou=groups,dc=hadoop,dc=apache,dc=org")); } @Test @@ -189,7 +209,8 @@ public void testGetUserByCN() throws Exception { validateUserEntry(entry, "ldaptest1", "TestCn1", "ldaptest1@example.com", "Test user ldaptest1"); validateMemberOf(entry, Set.of( "cn=group1,ou=groups,dc=hadoop,dc=apache,dc=org", - "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org")); + "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org", + "cn=group3,ou=groups,dc=hadoop,dc=apache,dc=org")); } @Test @@ -211,7 +232,8 @@ public void testGetUserBySAMAccountName() throws Exception { assertEquals("TestSam1", entry.get("sAMAccountName").getString()); validateMemberOf(entry, Set.of( "cn=group1,ou=groups,dc=hadoop,dc=apache,dc=org", - "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org")); + "cn=group2,ou=groups,dc=hadoop,dc=apache,dc=org", + "cn=group3,ou=groups,dc=hadoop,dc=apache,dc=org")); } @Test @@ -230,8 +252,8 @@ public void testGetUserUseMemberOf() throws Exception { config.put("useMemberOf", "true"); ldapProxyBackend = new LdapProxyBackend("testbackend", config); - Entry entry = ldapProxyBackend.getUser("ldaptest2", schemaManager); - validateUserEntry(entry, "ldaptest2", "TestCn2", "ldaptest2@example.com", "Test user ldaptest2"); + Entry entry = ldapProxyBackend.getUser("ldapmemberof", schemaManager); + validateUserEntry(entry, "ldapmemberof", "TestMemberOf", "ldapmemberof@example.com", "Test user ldapmemberof"); validateMemberOf(entry, Set.of( "cn=groupMemberOf1,ou=groups,dc=hadoop,dc=apache,dc=org", "cn=groupMemberOf2,ou=groups,dc=hadoop,dc=apache,dc=org")); @@ -244,6 +266,21 @@ public void testGetUserGroups() throws Exception { List userGroups = ldapProxyBackend.getUserGroups("ldaptest1", schemaManager); assertTrue(userGroups.contains("group1")); assertTrue(userGroups.contains("group2")); + assertTrue(userGroups.contains("group3")); + } + + @Test + public void testGetUserGroupsPaging() throws Exception { + Map config = new HashMap<>(ldapBackendConfig); + config.put("pageSize", Integer.toString(PAGE_SIZE)); + ldapProxyBackend = new LdapProxyBackend("testbackend", config); + + List userGroups = ldapProxyBackend.getUserGroups("ldaptest1", schemaManager); + int matchingRequests = (int) capturingSearchRequestHandler.getRequests().stream() + .filter(request -> request.getBase().getName().equals("ou=groups,dc=hadoop,dc=apache,dc=org") && + request.getFilter().toString().contains("ldaptest1")) + .count(); + assertEquals(2, matchingRequests); } @Test @@ -268,7 +305,7 @@ public void testGetUserGroupsUseMemberOf() throws Exception { config.put("useMemberOf", "true"); ldapProxyBackend = new LdapProxyBackend("testbackend", config); - List userGroups = ldapProxyBackend.getUserGroups("ldaptest2", schemaManager); + List userGroups = ldapProxyBackend.getUserGroups("ldapmemberof", schemaManager); assertTrue(userGroups.contains("groupMemberOf1")); assertTrue(userGroups.contains("groupMemberOf2")); } @@ -323,13 +360,28 @@ public void testGetUserGroupsUseMemberOfRecursiveDepth2() throws Exception { @Test public void testSearchUsers() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateUserSearch("*", 3, Set.of("ldaptest1", "ldaptest2", "guest")); + validateUserSearch("*", 4, Set.of("ldaptest1", "ldaptest2", "ldapmemberof", "guest")); + } + + @Test + public void testSearchUsersWithPaging() throws Exception { + Map config = new HashMap<>(ldapBackendConfig); + config.put("pageSize", Integer.toString(PAGE_SIZE)); + ldapProxyBackend = new LdapProxyBackend("testbackend", config); + validateUserSearch("*", 4, Set.of("ldaptest1", "ldaptest2", "ldapmemberof", "guest")); + int matchingRequests = (int) capturingSearchRequestHandler.getRequests().stream() + .filter(request -> request.getBase().getName().equals("ou=people,dc=hadoop,dc=apache,dc=org") && + request.getFilter().toString().contains("uid=*")) + .count(); + assertEquals(2, matchingRequests); } + + @Test public void testSearchUsersPartial() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateUserSearch("ldap*", 2, Set.of("ldaptest1", "ldaptest2")); + validateUserSearch("ldap*", 3, Set.of("ldaptest1", "ldaptest2", "ldapmemberof")); } @Test @@ -343,7 +395,7 @@ public void testSearchUsersNoneFound() throws Exception { public void testSearchUsersByCn() throws Exception { Map config = createConfigWithUserAttr("cn"); ldapProxyBackend = new LdapProxyBackend("testbackend", config); - validateUserSearch("*", 4, Set.of("ldaptest1", "ldaptest2", "Guest", "TestCn3")); + validateUserSearch("*", 5, Set.of("ldaptest1", "ldaptest2", "ldapmemberof", "Guest", "TestCn3")); } @Test @@ -365,7 +417,7 @@ public void testSearchUsersNoneFoundByCn() throws Exception { public void testSearchUsersBySAMAccountName() throws Exception { Map config = createConfigWithUserAttr("sAMAccountName"); ldapProxyBackend = new LdapProxyBackend("testbackend", config); - validateUserSearch("*", 3, Set.of("ldaptest1", "ldaptest2", "TestSam3")); + validateUserSearch("*", 4, Set.of("ldaptest1", "ldaptest2", "ldapmemberof", "TestSam3")); } @Test @@ -451,7 +503,7 @@ public void testSearchRecursiveWithSharedGroups() throws Exception { @Test public void testSearchObjectClassInetOrgPerson() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateSearch("ou=people,dc=hadoop,dc=apache,dc=org", "(objectClass=inetOrgPerson)", 4, Set.of("ldaptest1", "ldaptest2", "guest", "TestCn3")); + validateSearch("ou=people,dc=hadoop,dc=apache,dc=org", "(objectClass=inetOrgPerson)", 5, Set.of("ldaptest1", "ldaptest2", "ldapmemberof", "guest", "TestCn3")); } @Test @@ -463,19 +515,32 @@ public void testSearchByUid() throws Exception { @Test public void testSearchByUidWildcard() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateSearch("ou=people,dc=hadoop,dc=apache,dc=org", "(uid=*)", 3, Set.of("ldaptest1", "ldaptest2", "guest")); + validateSearch("ou=people,dc=hadoop,dc=apache,dc=org", "(uid=*)", 4, Set.of("ldaptest1", "ldaptest2", "ldapmemberof", "guest")); + } + + @Test + public void testSearchWithPaging() throws Exception { + Map config = new HashMap<>(ldapBackendConfig); + config.put("pageSize", Integer.toString(PAGE_SIZE)); + ldapProxyBackend = new LdapProxyBackend("testbackend", config); + validateSearch("ou=people,dc=hadoop,dc=apache,dc=org", "(uid=*)", 4, Set.of("ldaptest1", "ldaptest2", "ldapmemberof", "guest")); + int matchingRequests = (int) capturingSearchRequestHandler.getRequests().stream() + .filter(request -> request.getBase().getName().equals("ou=people,dc=hadoop,dc=apache,dc=org") && + request.getFilter().toString().contains("uid=*")) + .count(); + assertEquals(2, matchingRequests); } @Test public void testSearchByUidSubstringWildcard() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateSearch("ou=people,dc=hadoop,dc=apache,dc=org", "(uid=ldap*)", 2, Set.of("ldaptest1", "ldaptest2")); + validateSearch("ou=people,dc=hadoop,dc=apache,dc=org", "(uid=ldap*)", 3, Set.of("ldaptest1", "ldaptest2", "ldapmemberof")); } @Test public void testSearchObjectClassGroupOfNames() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateSearch("ou=groups,dc=hadoop,dc=apache,dc=org", "(objectClass=groupOfNames)", 3, Set.of("group1", "group2", "nameddifferently")); + validateSearch("ou=groups,dc=hadoop,dc=apache,dc=org", "(objectClass=groupOfNames)", 4, Set.of("group1", "group2", "group3", "nameddifferently")); } @Test @@ -487,19 +552,19 @@ public void testSearchByCn() throws Exception { @Test public void testSearchByCnWildcard() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateSearch("ou=groups,dc=hadoop,dc=apache,dc=org", "(cn=*)", 3, Set.of("group1", "group2", "nameddifferently")); + validateSearch("ou=groups,dc=hadoop,dc=apache,dc=org", "(cn=*)", 4, Set.of("group1", "group2", "group3", "nameddifferently")); } @Test public void testSearchByCnWSubstringildcard() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateSearch("ou=groups,dc=hadoop,dc=apache,dc=org", "(cn=group*)", 2, Set.of("group1", "group2")); + validateSearch("ou=groups,dc=hadoop,dc=apache,dc=org", "(cn=group*)", 3, Set.of("group1", "group2", "group3")); } @Test public void testSearchByUidOrCnWildcard() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); - validateSearch("dc=hadoop,dc=apache,dc=org", "(|(uid=ldap*)(cn=group*))", 4, Set.of("ldaptest1", "ldaptest2", "group1", "group2")); + validateSearch("dc=hadoop,dc=apache,dc=org", "(|(uid=ldap*)(cn=group*))", 6, Set.of("ldaptest1", "ldaptest2", "ldapmemberof", "group1", "group2", "group3")); } @Test @@ -809,4 +874,28 @@ public Set get(Object key) { // For the second user, many groups should have been found in the cache. assertEquals("Expected " + expectedCacheHits + " cache hits for shared groups, but got " + cacheHits.get(), expectedCacheHits, cacheHits.get()); } + + private static class CapturingSearchRequestHandler extends LdapRequestHandler { + private final LdapRequestHandler delegate; + private final List requests = Collections.synchronizedList(new ArrayList<>()); + + CapturingSearchRequestHandler(LdapRequestHandler delegate) { + this.delegate = delegate; + } + + public void reset() { + requests.clear(); + } + + public List getRequests() { + return List.copyOf(requests); + } + + @Override + public void handle(LdapSession session, SearchRequest message) throws Exception { + requests.add(message); + System.out.println(message.toString()); + delegate.handle(session, message); + } + } } diff --git a/gateway-server/src/test/resources/ldap-proxy-backend-test.ldif b/gateway-server/src/test/resources/ldap-proxy-backend-test.ldif index 91fb4136c9..456105f986 100644 --- a/gateway-server/src/test/resources/ldap-proxy-backend-test.ldif +++ b/gateway-server/src/test/resources/ldap-proxy-backend-test.ldif @@ -58,6 +58,12 @@ objectclass:groupOfNames cn: group2 member: uid=ldaptest1,ou=people,dc=hadoop,dc=apache,dc=org +dn: cn=group3,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:groupOfNames +cn: group3 +member: uid=ldaptest1,ou=people,dc=hadoop,dc=apache,dc=org + dn: cn=nameddifferently,ou=groups,dc=hadoop,dc=apache,dc=org objectclass:top objectclass:groupOfNames @@ -89,6 +95,19 @@ sAMAccountName: TestSam2 userPassword: 12345 mail: ldaptest2@example.com description: Test user ldaptest2 + +dn: uid=ldapmemberof,ou=people,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass:person +objectclass:organizationalPerson +objectclass:inetOrgPerson +cn: TestMemberOf +sn: Ldap +uid: ldapmemberof +sAMAccountName: TestMemberOf +userPassword: 12345 +mail: ldapmemberof@example.com +description: Test user ldapmemberof memberOf: cn=groupMemberOf1,ou=groups,dc=hadoop,dc=apache,dc=org memberOf: cn=groupMemberOf2,ou=groups,dc=hadoop,dc=apache,dc=org diff --git a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java index a3b3d0d9a8..e2dbd08d51 100644 --- a/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java +++ b/gateway-spi-common/src/main/java/org/apache/knox/gateway/GatewayTestConfig.java @@ -1355,6 +1355,16 @@ public List getLDAPSSLEnabledCipherSuites() { return Collections.emptyList(); } + @Override + public int getLDAPMaxSizeLimit() { + return 0; + } + + @Override + public int getLDAPMaxTimeLimit() { + return 0; + } + @Override public boolean getGroupUIServicesOnHomepage() { return false; diff --git a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java index 6b36e729bf..4aec1fba43 100644 --- a/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java +++ b/gateway-spi/src/main/java/org/apache/knox/gateway/config/GatewayConfig.java @@ -155,6 +155,8 @@ public interface GatewayConfig { String LDAP_SSL_KEYSTORE_PATH = "gateway.ldap.ssl.keystore.path"; String LDAP_SSL_KEYSTORE_PASSWORD_ALIAS = "gateway.ldap.ssl.keystore.password.alias"; String LDAP_SSL_ENABLED_CIPHER_SUITES = "gateway.ldap.ssl.enabled.cipher.suites"; + String LDAP_MAX_SIZE_LIMIT = "gateway.ldap.max.size.limit"; + String LDAP_MAX_TIME_LIMIT = "gateway.ldap.max.time.limit"; /** * The location of the gateway configuration. @@ -1211,6 +1213,16 @@ public interface GatewayConfig { */ List getLDAPSSLEnabledCipherSuites(); + /** + * @return the maximum size limit for LDAP search + */ + int getLDAPMaxSizeLimit(); + + /** + * @return the maximum time limit for LDAP search in milliseconds + */ + int getLDAPMaxTimeLimit(); + /** * @return set of all property names in the configuration */ diff --git a/knox-site/docs/service_ldap_server.md b/knox-site/docs/service_ldap_server.md index 8d524395b6..f7aa82acb1 100644 --- a/knox-site/docs/service_ldap_server.md +++ b/knox-site/docs/service_ldap_server.md @@ -44,6 +44,8 @@ The service is configured in `gateway-site.xml`. | `gateway.ldap.roles.lookup.strategy` | N/A | The LDAP roles lookup strategy (`file` or `rest`). | | `gateway.ldap.roles.lookup.rest.api.endpoint` | N/A | The LDAP roles lookup REST API endpoint. | | `gateway.ldap.roles.lookup.file.path` | N/A | The LDAP roles lookup file path. | +| `gateway.ldap.roles.max.size.limit` | 1000 | The maximum size limit for search requests. | +| `gateway.ldap.roles.max.time.limit` | 60000 | The maximum time limit for search requests in milliseconds. | ### Bind Credentials @@ -198,6 +200,7 @@ The proxy backend delegates lookups to a remote LDAP or Active Directory server. | `gateway.ldap.interceptor..groupMemberAttribute` | `memberUid` | Attribute used for group membership (e.g., `member` for AD). | | `gateway.ldap.interceptor..useMemberOf` | `false` | If `true`, use the `memberOf` attribute for efficient group lookups. | | `gateway.ldap.interceptor..proxy.poolMaxActive` | `8` | Maximum number of active connections in the pool. | +| `gateway.ldap.interceptor..pageSize` | `1000` | Page size for search requests. | ## Active Directory (AD) Integration From 192151fae73baa88fc08b343861cf343e8595768 Mon Sep 17 00:00:00 2001 From: David Han Date: Wed, 29 Jul 2026 00:53:18 -0500 Subject: [PATCH 2/4] add test annotation @Test was missed when resolving conflicts. This commit adds it back in. --- .../knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java index 9e0877f96c..a5e7df664d 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManagerTest.java @@ -465,6 +465,7 @@ public void testGetUserGroupsIgnoresBareRdnWhenRolesLookupInactive() throws Exce List.of("analysts"), groups); } + @Test public void testStartSetsMaxSizeAndTime() throws Exception { final int expectedMaxSize = 3158; final int expectedMaxTime = 245000; From fa97adde8d7dfeb10d51db4f25c81c6f61849ace Mon Sep 17 00:00:00 2001 From: David Han Date: Thu, 30 Jul 2026 21:42:30 -0500 Subject: [PATCH 3/4] Add maxResultSetSize config to LdapProxyBackend and propagate LdapExceptions --- .../services/ldap/KnoxLDAPServerManager.java | 13 +++-- .../gateway/services/ldap/LdapMessages.java | 8 ++++ .../ldap/backend/LdapProxyBackend.java | 15 ++++-- .../ldap/backend/LdapProxyBackendTest.java | 47 ++++++++++++++++++- knox-site/docs/service_ldap_server.md | 5 +- 5 files changed, 79 insertions(+), 9 deletions(-) diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java index 1fbabf3204..e4445276e1 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java @@ -117,6 +117,9 @@ public void initialize(GatewayConfig config) throws Exception { this.baseDn = config.getLDAPBaseDN(); this.bindUser = config.getLDAPBindUser(); + maxSizeLimit = config.getLDAPMaxSizeLimit(); + maxTimeLimit = config.getLDAPMaxTimeLimit(); + // Secure (LDAPS) transport configuration. When enabled but no dedicated keystore is // configured, fall back to the gateway identity keystore so the embedded server can // reuse the gateway's own TLS material out of the box. @@ -140,9 +143,6 @@ public void initialize(GatewayConfig config) throws Exception { } workDir.mkdirs(); - - maxSizeLimit = config.getLDAPMaxSizeLimit(); - maxTimeLimit = config.getLDAPMaxTimeLimit(); } private void createInterceptors(GatewayConfig config) throws Exception { @@ -154,6 +154,13 @@ private void createInterceptors(GatewayConfig config) throws Exception { // Add common configuration interceptorConfig.put("baseDn", baseDn); + if (!interceptorConfig.containsKey("maxResultSetSize")) { + // Set the backend to return more results than the proxy's size limit. + // This will ensure that the proxy will return "Size limit exceeded" + if (maxSizeLimit != 0) { + interceptorConfig.put("maxResultSetSize", Integer.toString(maxSizeLimit + 1)); + } + } // Add common LDAP Proxy configurations to backends if ("backend".equalsIgnoreCase(interceptorConfig.get("interceptorType"))) { diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java index 7900566577..bb54bf56b8 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java @@ -73,6 +73,10 @@ public interface LdapMessages { text = "Creating LDAP interceptor: {0} (via {1})") void ldapInterceptorCreating(String interceptorName, String source); + @Message(level = MessageLevel.INFO, + text = "Configuring LDAP interceptor {0}: {1} = {2})") + void ldapInterceptorConfiguring(String interceptorNamee, String configName, String configValue); + @Message(level = MessageLevel.INFO, text = "Loading backend: {0} (via {1})") void ldapBackendLoading(String backendName, String source); @@ -105,6 +109,10 @@ public interface LdapMessages { text = "LDAP Paged Search: {0} | {1}, page size {2}, page {3}") void ldapPagedSearch(String baseDn, String filter, int pageSize, int pageNumber); + @Message(level = MessageLevel.ERROR, + text = "LDAP Paged Search Exceeded Max Result Set Size: {0} | {1}") + void ldapPagedSearchExceededMaxResultSetSize(int resultSetSize, int maxResultSetSize); + @Message(level = MessageLevel.DEBUG, text = "LDAP Paged Search Completed: {0} | {1}") void ldapPagedSearchCompleted(String baseDn, String filter); diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java index 8246c4d36a..32fd033b24 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackend.java @@ -104,6 +104,7 @@ public class LdapProxyBackend implements LdapBackend { private boolean recursiveGroupResolution; private int recursiveGroupResolutionMaxDepth; private int pageSize; + private int maxResultSetSize; private final String proxyEntryGroupMembershipAttributeType = "memberOf"; @@ -194,6 +195,8 @@ public LdapProxyBackend(String name, Map config) { // Configure search parameters pageSize = Integer.parseInt(config.getOrDefault("pageSize", "1000")); + maxResultSetSize = Integer.parseInt(config.getOrDefault("maxResultSetSize", "0")); // 0 means unlimited + LOG.ldapInterceptorConfiguring(name, "maxResultSetSize", Integer.toString(maxResultSetSize)); // Configure secure transport (LDAPS) to the remote server. An ldaps:// URL enables it // by default; an explicit useSsl setting always wins. @@ -893,11 +896,15 @@ protected List performPagedSearch(LdapConnection connection, String baseD } } pageNumber++; - } catch (LdapException e) { - LOG.ldapSearchFailed(baseDn, filter, e); } - } while (cookie != null && cookie.length > 0); - LOG.ldapPagedSearchCompleted(baseDn, filter); + } while (cookie != null && cookie.length > 0 && + (maxResultSetSize == 0 || results.size() < maxResultSetSize)); + + if (maxResultSetSize != 0 && results.size() >= maxResultSetSize) { + LOG.ldapPagedSearchExceededMaxResultSetSize(results.size(), maxResultSetSize); + } else { + LOG.ldapPagedSearchCompleted(baseDn, filter); + } return results; } diff --git a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendTest.java b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendTest.java index dec22d5fe9..7264527db8 100644 --- a/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendTest.java +++ b/gateway-server/src/test/java/org/apache/knox/gateway/services/ldap/backend/LdapProxyBackendTest.java @@ -276,6 +276,7 @@ public void testGetUserGroupsPaging() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", config); List userGroups = ldapProxyBackend.getUserGroups("ldaptest1", schemaManager); + assertEquals(3, userGroups.size()); int matchingRequests = (int) capturingSearchRequestHandler.getRequests().stream() .filter(request -> request.getBase().getName().equals("ou=groups,dc=hadoop,dc=apache,dc=org") && request.getFilter().toString().contains("ldaptest1")) @@ -283,6 +284,22 @@ public void testGetUserGroupsPaging() throws Exception { assertEquals(2, matchingRequests); } + @Test + public void testGetUserGroupsPagingExceedsMaxResultSetSize() throws Exception { + Map config = new HashMap<>(ldapBackendConfig); + config.put("pageSize", Integer.toString(PAGE_SIZE)); + config.put("maxResultSetSize", "1"); + ldapProxyBackend = new LdapProxyBackend("testbackend", config); + + List userGroups = ldapProxyBackend.getUserGroups("ldaptest1", schemaManager); + assertEquals(PAGE_SIZE, userGroups.size()); // only retrieve 1 page because that will exceed the maxResultSetSize + int matchingRequests = (int) capturingSearchRequestHandler.getRequests().stream() + .filter(request -> request.getBase().getName().equals("ou=groups,dc=hadoop,dc=apache,dc=org") && + request.getFilter().toString().contains("ldaptest1")) + .count(); + assertEquals(1, matchingRequests); + } + @Test public void testGetUserGroupsNoGroups() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); @@ -376,7 +393,21 @@ public void testSearchUsersWithPaging() throws Exception { assertEquals(2, matchingRequests); } + @Test + public void testSearchUsersWithPagingExceedsMaxResultSetSize() throws Exception { + Map config = new HashMap<>(ldapBackendConfig); + config.put("pageSize", Integer.toString(PAGE_SIZE)); + config.put("maxResultSetSize", "1"); + ldapProxyBackend = new LdapProxyBackend("testbackend", config); + List entries = ldapProxyBackend.searchUsers("*", schemaManager); + assertEquals(PAGE_SIZE, entries.size()); // only expect 1 page of results + int matchingRequests = (int) capturingSearchRequestHandler.getRequests().stream() + .filter(request -> request.getBase().getName().equals("ou=people,dc=hadoop,dc=apache,dc=org") && + request.getFilter().toString().contains("uid=*")) + .count(); + assertEquals(1, matchingRequests); + } @Test public void testSearchUsersPartial() throws Exception { @@ -531,6 +562,21 @@ public void testSearchWithPaging() throws Exception { assertEquals(2, matchingRequests); } + @Test + public void testSearchWithPagingExceedsMaxResultSize() throws Exception { + Map config = new HashMap<>(ldapBackendConfig); + config.put("pageSize", Integer.toString(PAGE_SIZE)); + config.put("maxResultSetSize", "1"); + ldapProxyBackend = new LdapProxyBackend("testbackend", config); + List entries = ldapProxyBackend.search("ou=people,dc=hadoop,dc=apache,dc=org", SearchScope.SUBTREE, "(uid=*)", schemaManager); + assertEquals(PAGE_SIZE, entries.size()); // only expect 1 page because that will exceed the maxResultSetSize + int matchingRequests = (int) capturingSearchRequestHandler.getRequests().stream() + .filter(request -> request.getBase().getName().equals("ou=people,dc=hadoop,dc=apache,dc=org") && + request.getFilter().toString().contains("uid=*")) + .count(); + assertEquals(1, matchingRequests); + } + @Test public void testSearchByUidSubstringWildcard() throws Exception { ldapProxyBackend = new LdapProxyBackend("testbackend", ldapBackendConfig); @@ -894,7 +940,6 @@ public List getRequests() { @Override public void handle(LdapSession session, SearchRequest message) throws Exception { requests.add(message); - System.out.println(message.toString()); delegate.handle(session, message); } } diff --git a/knox-site/docs/service_ldap_server.md b/knox-site/docs/service_ldap_server.md index f7aa82acb1..b71b9d3519 100644 --- a/knox-site/docs/service_ldap_server.md +++ b/knox-site/docs/service_ldap_server.md @@ -44,7 +44,7 @@ The service is configured in `gateway-site.xml`. | `gateway.ldap.roles.lookup.strategy` | N/A | The LDAP roles lookup strategy (`file` or `rest`). | | `gateway.ldap.roles.lookup.rest.api.endpoint` | N/A | The LDAP roles lookup REST API endpoint. | | `gateway.ldap.roles.lookup.file.path` | N/A | The LDAP roles lookup file path. | -| `gateway.ldap.roles.max.size.limit` | 1000 | The maximum size limit for search requests. | +| `gateway.ldap.roles.max.size.limit` | 1000 | The maximum size limit of the result set returned by search requests. | | `gateway.ldap.roles.max.time.limit` | 60000 | The maximum time limit for search requests in milliseconds. | ### Bind Credentials @@ -201,6 +201,9 @@ The proxy backend delegates lookups to a remote LDAP or Active Directory server. | `gateway.ldap.interceptor..useMemberOf` | `false` | If `true`, use the `memberOf` attribute for efficient group lookups. | | `gateway.ldap.interceptor..proxy.poolMaxActive` | `8` | Maximum number of active connections in the pool. | | `gateway.ldap.interceptor..pageSize` | `1000` | Page size for search requests. | +| `gateway.ldap.interceptor..maxResultSetSize` | `0` | Maximum number of results to return from a search, regardless of paging. 0 means unlimited. | + +NOTE: If this value is undefined and the interceptor was created by the KnoxLDAPServerManager, the KnoxLDAPServerManager will set the `gateway.ldap.interceptor..maxResultSetSize` value to be 1 greater than the proxy's `gateway.ldap.roles.max.time.limit` configuration. This will ensure that the proxy returns a "Size limit exceeded" result if the backend has more results than the proxy's limit. ## Active Directory (AD) Integration From 657e16282034eaea1536736cbc5b0fe31c92b9a6 Mon Sep 17 00:00:00 2001 From: David Han Date: Fri, 31 Jul 2026 07:06:57 -0500 Subject: [PATCH 4/4] fix documentation --- .../org/apache/knox/gateway/services/ldap/LdapMessages.java | 4 ++-- knox-site/docs/service_ldap_server.md | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java index bb54bf56b8..8683f46457 100644 --- a/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java +++ b/gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java @@ -74,8 +74,8 @@ public interface LdapMessages { void ldapInterceptorCreating(String interceptorName, String source); @Message(level = MessageLevel.INFO, - text = "Configuring LDAP interceptor {0}: {1} = {2})") - void ldapInterceptorConfiguring(String interceptorNamee, String configName, String configValue); + text = "Configuring LDAP interceptor {0}: {1} = {2}") + void ldapInterceptorConfiguring(String interceptorName, String configName, String configValue); @Message(level = MessageLevel.INFO, text = "Loading backend: {0} (via {1})") diff --git a/knox-site/docs/service_ldap_server.md b/knox-site/docs/service_ldap_server.md index b71b9d3519..347c40aa89 100644 --- a/knox-site/docs/service_ldap_server.md +++ b/knox-site/docs/service_ldap_server.md @@ -44,8 +44,8 @@ The service is configured in `gateway-site.xml`. | `gateway.ldap.roles.lookup.strategy` | N/A | The LDAP roles lookup strategy (`file` or `rest`). | | `gateway.ldap.roles.lookup.rest.api.endpoint` | N/A | The LDAP roles lookup REST API endpoint. | | `gateway.ldap.roles.lookup.file.path` | N/A | The LDAP roles lookup file path. | -| `gateway.ldap.roles.max.size.limit` | 1000 | The maximum size limit of the result set returned by search requests. | -| `gateway.ldap.roles.max.time.limit` | 60000 | The maximum time limit for search requests in milliseconds. | +| `gateway.ldap.max.size.limit` | 1000 | The maximum size limit of the result set returned by search requests. | +| `gateway.ldap.max.time.limit` | 60000 | The maximum time limit for search requests in milliseconds. | ### Bind Credentials @@ -203,7 +203,7 @@ The proxy backend delegates lookups to a remote LDAP or Active Directory server. | `gateway.ldap.interceptor..pageSize` | `1000` | Page size for search requests. | | `gateway.ldap.interceptor..maxResultSetSize` | `0` | Maximum number of results to return from a search, regardless of paging. 0 means unlimited. | -NOTE: If this value is undefined and the interceptor was created by the KnoxLDAPServerManager, the KnoxLDAPServerManager will set the `gateway.ldap.interceptor..maxResultSetSize` value to be 1 greater than the proxy's `gateway.ldap.roles.max.time.limit` configuration. This will ensure that the proxy returns a "Size limit exceeded" result if the backend has more results than the proxy's limit. +NOTE: If this value is undefined and the interceptor was created by the KnoxLDAPServerManager, the KnoxLDAPServerManager will set the `gateway.ldap.interceptor..maxResultSetSize` value to be 1 greater than the proxy's `gateway.ldap.max.size.limit` configuration. This will ensure that the proxy returns a "Size limit exceeded" result if the backend has more results than the proxy's limit. ## Active Directory (AD) Integration