KNOX-3386: LDAP Proxy pages backends

.github/workflows/build/gateway-site.xml

@@ -145,6 +145,14 @@ limitations under the License.
         <name>gateway.ldap.base.dn</name>
         <value>dc=proxy,dc=org</value>
     </property>
+    <property>
+        <name>gateway.ldap.max.size.limit</name>
+        <value>1000</value>
+    </property>
+    <property>
+        <name>gateway.ldap.max.time.limit</name>
+        <value>60000</value>
+    </property>
     <property>
         <name>gateway.ldap.recursive.group.resolution</name>
         <value>true</value>
@@ -211,5 +219,10 @@ limitations under the License.
         <name>gateway.ldap.interceptor.demoldap.groupMemberAttribute</name>
         <value>member</value>
     </property>
+    <!-- Set an unnaturally low page size to ensure that paging is used in tests -->
+    <property>
+        <name>gateway.ldap.interceptor.demoldap.pageSize</name>
+        <value>3</value>
+    </property>
 
 </configuration>

.github/workflows/compose/single-eku-no-mtls/gateway-site.xml

@@ -157,6 +157,14 @@ limitations under the License.
         <name>gateway.ldap.base.dn</name>
         <value>dc=proxy,dc=org</value>
     </property>
+    <property>
+        <name>gateway.ldap.max.size.limit</name>
+        <value>1000</value>
+    </property>
+    <property>
+        <name>gateway.ldap.max.time.limit</name>
+        <value>60000</value>
+    </property>
     <property>
         <name>gateway.ldap.recursive.group.resolution</name>
         <value>true</value>
@@ -203,5 +211,10 @@ limitations under the License.
         <name>gateway.ldap.interceptor.demoldap.groupMemberAttribute</name>
         <value>member</value>
     </property>
+    <!-- Set an unnaturally low page size to ensure that paging is used in tests -->
+    <property>
+        <name>gateway.ldap.interceptor.demoldap.pageSize</name>
+        <value>3</value>
+    </property>
 
 </configuration>

.github/workflows/compose/single-eku/gateway-site.xml

@@ -211,6 +211,14 @@ limitations under the License.
         <name>gateway.ldap.base.dn</name>
         <value>dc=proxy,dc=org</value>
     </property>
+    <property>
+        <name>gateway.ldap.max.size.limit</name>
+        <value>1000</value>
+    </property>
+    <property>
+        <name>gateway.ldap.max.time.limit</name>
+        <value>60000</value>
+    </property>
     <property>
         <name>gateway.ldap.recursive.group.resolution</name>
         <value>true</value>
@@ -257,5 +265,10 @@ limitations under the License.
         <name>gateway.ldap.interceptor.demoldap.groupMemberAttribute</name>
         <value>member</value>
     </property>
+    <!-- Set an unnaturally low page size to ensure that paging is used in tests -->
+    <property>
+        <name>gateway.ldap.interceptor.demoldap.pageSize</name>
+        <value>3</value>
+    </property>
 
 </configuration>

gateway-server/src/main/java/org/apache/knox/gateway/config/impl/GatewayConfigImpl.java

@@ -392,6 +392,13 @@ public class GatewayConfigImpl extends Configuration implements GatewayConfig {
   public static final String STRICT_TRANSPORT_ENABLED = GATEWAY_CONFIG_FILE_PREFIX + ".strict.transport.enabled";
   public static final String STRICT_TRANSPORT_OPTION = GATEWAY_CONFIG_FILE_PREFIX + ".strict.transport.option";
 
+  // Gateway LDAP Properties
+  public static final int DEFAULT_LDAP_PORT = 3890;
+  public static final String DEFAULT_LDAP_BASE_DN = "dc=proxy,dc=com";
+  public static final int DEFAULT_LDAP_MAX_SIZE_LIMIT = 1000;
+  /* The default max time for LDAP search in milliseconds */
+  public static final int DEFAULT_LDAP_MAX_TIME_LIMIT = 60 * 1000;
+
   public GatewayConfigImpl() {
     init();
   }
@@ -1775,17 +1782,17 @@ public String getStrictTransportOption() {
   // LDAP Service Configuration
   @Override
   public boolean isLDAPEnabled() {
-    return Boolean.parseBoolean(get(LDAP_ENABLED, "false"));
+    return getBoolean(LDAP_ENABLED, false);
   }
 
   @Override
   public int getLDAPPort() {
-    return Integer.parseInt(get(LDAP_PORT, "3890"));
+    return getInt(LDAP_PORT, DEFAULT_LDAP_PORT);
   }
 
   @Override
   public String getLDAPBaseDN() {
-    return get(LDAP_BASE_DN, "dc=proxy,dc=com");
+    return get(LDAP_BASE_DN, DEFAULT_LDAP_BASE_DN);
   }
 
   @Override
@@ -1840,7 +1847,7 @@ public Map<String, String> getLDAPInterceptorConfig(String interceptorName) {
 
   @Override
   public boolean isLDAPRecursiveGroupResolutionEnabled() {
-    return Boolean.parseBoolean(get(LDAP_RECURSIVE_GROUP_RESOLUTION, "false"));
+    return getBoolean(LDAP_RECURSIVE_GROUP_RESOLUTION, false);
   }
 
   @Override
@@ -1865,7 +1872,7 @@ public String getLdapRolesLookupFilePath() {
 
   @Override
   public boolean isLDAPSSLEnabled() {
-    return Boolean.parseBoolean(get(LDAP_SSL_ENABLED, "false"));
+    return getBoolean(LDAP_SSL_ENABLED, false);
   }
 
   @Override
@@ -1884,6 +1891,16 @@ public List<String> getLDAPSSLEnabledCipherSuites() {
     return cipherSuites == null ? Collections.emptyList() : cipherSuites;
   }
 
+  @Override
+  public int getLDAPMaxSizeLimit() {
+    return getInt(LDAP_MAX_SIZE_LIMIT, DEFAULT_LDAP_MAX_SIZE_LIMIT);
+  }
+
+  @Override
+  public int getLDAPMaxTimeLimit() {
+    return getInt(LDAP_MAX_TIME_LIMIT, DEFAULT_LDAP_MAX_TIME_LIMIT);
+  }
+
   @Override
   public boolean getGroupUIServicesOnHomepage() {
     return getBoolean(KNOX_HOMEPAGE_GROUP_UI_SERVICES, DEFAULT_GROUP_UI_SERVICES);

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/KnoxLDAPServerManager.java

@@ -72,7 +72,8 @@ public class KnoxLDAPServerManager {
 
     @VisibleForTesting
     DirectoryService directoryService;
-    private LdapServer ldapServer;
+    @VisibleForTesting
+    LdapServer ldapServer;
     private GatewayConfig gatewayConfig;
     private List<Interceptor> interceptors;
     private boolean hasRolesLookupInterceptor;
@@ -87,6 +88,8 @@ public class KnoxLDAPServerManager {
     private List<String> sslEnabledCipherSuites;
     // Collection of DNs for the proxied backend LDAP servers
     private Set<String> baseDns;
+    private int maxSizeLimit;
+    private int maxTimeLimit;
 
     KnoxLDAPServerManager(AliasService aliasService) {
         this(aliasService, null);
@@ -114,6 +117,9 @@ public void initialize(GatewayConfig config) throws Exception {
         this.baseDn = config.getLDAPBaseDN();
         this.bindUser = config.getLDAPBindUser();
 
+        maxSizeLimit = config.getLDAPMaxSizeLimit();
+        maxTimeLimit = config.getLDAPMaxTimeLimit();
+
         // Secure (LDAPS) transport configuration. When enabled but no dedicated keystore is
         // configured, fall back to the gateway identity keystore so the embedded server can
         // reuse the gateway's own TLS material out of the box.
@@ -148,6 +154,13 @@ private void createInterceptors(GatewayConfig config) throws Exception {
 
             // Add common configuration
             interceptorConfig.put("baseDn", baseDn);
+            if (!interceptorConfig.containsKey("maxResultSetSize")) {
+                // Set the backend to return more results than the proxy's size limit.
+                // This will ensure that the proxy will return "Size limit exceeded"
+                if (maxSizeLimit != 0) {
+                    interceptorConfig.put("maxResultSetSize", Integer.toString(maxSizeLimit + 1));
+                }
+            }
 
             // Add common LDAP Proxy configurations to backends
             if ("backend".equalsIgnoreCase(interceptorConfig.get("interceptorType"))) {
@@ -241,6 +254,9 @@ public void start() throws Exception {
         ldapServer.setTransports(transport);
         ldapServer.setDirectoryService(directoryService);
 
+        ldapServer.setMaxSizeLimit(maxSizeLimit);
+        ldapServer.setMaxTimeLimit(maxTimeLimit);
+
         ldapServer.start();
 
         LOG.ldapServiceStarted(port);

gateway-server/src/main/java/org/apache/knox/gateway/services/ldap/LdapMessages.java

@@ -73,6 +73,10 @@ public interface LdapMessages {
             text = "Creating LDAP interceptor: {0} (via {1})")
     void ldapInterceptorCreating(String interceptorName, String source);
 
+    @Message(level = MessageLevel.INFO,
+            text = "Configuring LDAP interceptor {0}: {1} = {2}")
+    void ldapInterceptorConfiguring(String interceptorName, String configName, String configValue);
+
     @Message(level = MessageLevel.INFO,
             text = "Loading backend: {0} (via {1})")
     void ldapBackendLoading(String backendName, String source);
@@ -101,6 +105,18 @@ public interface LdapMessages {
             text = "LDAP Search: {0} | {1}")
     void ldapSearch(String baseDn, String filter);
 
+    @Message(level = MessageLevel.DEBUG,
+            text = "LDAP Paged Search: {0} | {1}, page size {2}, page {3}")
+    void ldapPagedSearch(String baseDn, String filter, int pageSize, int pageNumber);
+
+    @Message(level = MessageLevel.ERROR,
+            text = "LDAP Paged Search Exceeded Max Result Set Size: {0} | {1}")
+    void ldapPagedSearchExceededMaxResultSetSize(int resultSetSize, int maxResultSetSize);
+
+    @Message(level = MessageLevel.DEBUG,
+            text = "LDAP Paged Search Completed: {0} | {1}")
+    void ldapPagedSearchCompleted(String baseDn, String filter);
+
     @Message(level = MessageLevel.ERROR,
             text = "LDAP Search failed: {0} | {1}, {2}")
     void ldapSearchFailed(String baseDn, String filter, @StackTrace(level = MessageLevel.DEBUG) Exception e);
@@ -133,9 +149,9 @@ public interface LdapMessages {
             text = "Backend user not found: {0}")
     void ldapUserNull(String username);
 
-    @Message(level = MessageLevel.ERROR,
+    @Message(level = MessageLevel.DEBUG,
             text = "Failed to copy attribute: {0}")
-    void ldapAttributeCopyError(@StackTrace(level = MessageLevel.DEBUG) Exception e);
+    void ldapAttributeCopyError(@StackTrace(level = MessageLevel.TRACE) Exception e);
 
     @Message(level = MessageLevel.DEBUG, text = "LDAP authentication succeeded for user: {0}")
     void ldapAuthSucceeded(String user);