diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/addresstranslation/AddressTranslator.java b/core/src/main/java/com/datastax/oss/driver/api/core/addresstranslation/AddressTranslator.java index 47ce62f1461..dbef705b196 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/addresstranslation/AddressTranslator.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/addresstranslation/AddressTranslator.java @@ -18,7 +18,9 @@ package com.datastax.oss.driver.api.core.addresstranslation; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.net.InetSocketAddress; +import java.util.UUID; /** * Translates IP addresses received from Cassandra nodes into locally queriable addresses. @@ -46,6 +48,32 @@ public interface AddressTranslator extends AutoCloseable { @NonNull InetSocketAddress translate(@NonNull InetSocketAddress address); + /** + * Translates an address reported by a Cassandra node into the address that the driver will use to + * connect, with additional node metadata for context. + * + *

This method is called during node discovery and allows implementations to use the node's + * host ID, datacenter, and rack to make translation decisions. For example, the client routes + * handler uses the host ID to look up the appropriate endpoint mapping. + * + *

The default implementation delegates to {@link #translate(InetSocketAddress)}, ignoring the + * additional parameters. This ensures backward compatibility with existing implementations. + * + * @param address the broadcast RPC address of the node + * @param hostId the unique identifier of the node (may be null for contact points) + * @param datacenter the datacenter of the node (may be null if not yet known) + * @param rack the rack of the node (may be null if not yet known) + * @return the translated address + */ + @NonNull + default InetSocketAddress translate( + @NonNull InetSocketAddress address, + @Nullable UUID hostId, + @Nullable String datacenter, + @Nullable String rack) { + return translate(address); + } + /** Called when the cluster that this translator is associated with closes. */ @Override void close(); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/ClientRoutesConfig.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/ClientRoutesConfig.java new file mode 100644 index 00000000000..1c6703253b2 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/ClientRoutesConfig.java @@ -0,0 +1,245 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.core.config; + +import com.datastax.oss.driver.api.core.session.SessionBuilder; +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import net.jcip.annotations.Immutable; + +/** + * Configuration for client routes, used in cloud private-endpoint deployments. + * + *

Client routes enable the driver to discover and connect to nodes through a load balancer (such + * as AWS PrivateLink, Azure Private Link, or GCP Private Service Connect) by reading endpoint + * mappings from the {@code system.client_routes} table. Each endpoint is identified by a connection + * ID and maps to specific node addresses. + * + *

This configuration is mutually exclusive with a user-provided {@link + * com.datastax.oss.driver.api.core.addresstranslation.AddressTranslator}. If client routes are + * configured, the driver will use its internal client routes handler for address translation. + * + *

Example usage: + * + * 

{@code
+ * ClientRoutesConfig config = ClientRoutesConfig.builder()
+ *     .addEndpoint(new ClientRoutesEndpoint(
+ *         UUID.fromString("12345678-1234-1234-1234-123456789012"),
+ *         "my-cluster-endpoint.example.com:9042"))
+ *     .withDnsCacheDuration(1000L)  // Cache DNS for 1 second (default: 500ms)
+ *     .build();
+ *
+ * CqlSession session = CqlSession.builder()
+ *     .withClientRoutesConfig(config)
+ *     .build();
+ * }
+ * + * @see SessionBuilder#withClientRoutesConfig(ClientRoutesConfig) + * @see ClientRoutesEndpoint + */ +@Immutable +public class ClientRoutesConfig { + + private static final String DEFAULT_TABLE_NAME = "system.client_routes"; + private static final long DEFAULT_DNS_CACHE_DURATION_MILLIS = 500L; + + private final List endpoints; + private final String tableName; + private final long dnsCacheDurationMillis; + + private ClientRoutesConfig( + List endpoints, String tableName, long dnsCacheDurationMillis) { + if (endpoints == null || endpoints.isEmpty()) { + throw new IllegalArgumentException("At least one endpoint must be specified"); + } + if (dnsCacheDurationMillis < 0) { + throw new IllegalArgumentException("DNS cache duration must be non-negative"); + } + this.endpoints = Collections.unmodifiableList(new ArrayList<>(endpoints)); + this.tableName = tableName; + this.dnsCacheDurationMillis = dnsCacheDurationMillis; + } + + /** + * Returns the list of configured endpoints. + * + * @return an immutable list of endpoints. + */ + @NonNull + public List getEndpoints() { + return endpoints; + } + + /** + * Returns the name of the system table to query for client routes. + * + * @return the table name, or null to use the default ({@code system.client_routes}). + */ + @Nullable + public String getTableName() { + return tableName; + } + + /** + * Returns the DNS cache duration in milliseconds. + * + *

This controls how long resolved DNS entries are cached before being re-resolved. A shorter + * duration is appropriate for dynamic environments where DNS mappings change frequently, while a + * longer duration can reduce DNS lookup overhead in stable environments. + * + * @return the DNS cache duration in milliseconds (default: 500ms). + */ + public long getDnsCacheDurationMillis() { + return dnsCacheDurationMillis; + } + + /** + * Creates a new builder for constructing a {@link ClientRoutesConfig}. + * + * @return a new builder instance. + */ + @NonNull + public static Builder builder() { + return new Builder(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ClientRoutesConfig)) { + return false; + } + ClientRoutesConfig that = (ClientRoutesConfig) o; + return dnsCacheDurationMillis == that.dnsCacheDurationMillis + && endpoints.equals(that.endpoints) + && Objects.equals(tableName, that.tableName); + } + + @Override + public int hashCode() { + return Objects.hash(endpoints, tableName, dnsCacheDurationMillis); + } + + @Override + public String toString() { + return "ClientRoutesConfig{" + + "endpoints=" + + endpoints + + ", tableName='" + + tableName + + '\'' + + ", dnsCacheDurationMillis=" + + dnsCacheDurationMillis + + '}'; + } + + /** Builder for {@link ClientRoutesConfig}. */ + public static class Builder { + private final List endpoints = new ArrayList<>(); + private String tableName = DEFAULT_TABLE_NAME; + private long dnsCacheDurationMillis = DEFAULT_DNS_CACHE_DURATION_MILLIS; + + /** + * Adds an endpoint to the configuration. + * + * @param endpoint the endpoint to add (must not be null). + * @return this builder. + */ + @NonNull + public Builder addEndpoint(@NonNull ClientRoutesEndpoint endpoint) { + this.endpoints.add(Objects.requireNonNull(endpoint, "endpoint must not be null")); + return this; + } + + /** + * Sets the endpoints for the configuration, replacing any previously added endpoints. + * + * @param endpoints the endpoints to set (must not be null or empty). + * @return this builder. + */ + @NonNull + public Builder withEndpoints(@NonNull List endpoints) { + Objects.requireNonNull(endpoints, "endpoints must not be null"); + if (endpoints.isEmpty()) { + throw new IllegalArgumentException("endpoints must not be empty"); + } + this.endpoints.clear(); + for (ClientRoutesEndpoint endpoint : endpoints) { + addEndpoint(endpoint); + } + return this; + } + + /** + * Sets the name of the system table to query for client routes. + * + *

This is primarily useful for testing. If not set, the driver will use the default table + * name from the configuration ({@code system.client_routes}). + * + * @param tableName the table name to use. + * @return this builder. + */ + @VisibleForTesting + @NonNull + public Builder withTableName(@Nullable String tableName) { + this.tableName = tableName; + return this; + } + + /** + * Sets the DNS cache duration in milliseconds. + * + *

This controls how long resolved DNS entries are cached before being re-resolved. A shorter + * duration is appropriate for dynamic environments where DNS mappings change frequently (e.g., + * during rolling updates), while a longer duration can reduce DNS lookup overhead in stable + * environments. + * + *

Default: 500ms + * + * @param durationMillis the cache duration in milliseconds (must be non-negative). + * @return this builder. + * @throws IllegalArgumentException if the duration is negative. + */ + @NonNull + public Builder withDnsCacheDuration(long durationMillis) { + if (durationMillis < 0) { + throw new IllegalArgumentException("DNS cache duration must be non-negative"); + } + this.dnsCacheDurationMillis = durationMillis; + return this; + } + + /** + * Builds the {@link ClientRoutesConfig} with the configured endpoints and table name. + * + * @return the new configuration instance. + * @throws IllegalArgumentException if no endpoints have been added. + */ + @NonNull + public ClientRoutesConfig build() { + return new ClientRoutesConfig(endpoints, tableName, dnsCacheDurationMillis); + } + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/ClientRoutesEndpoint.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/ClientRoutesEndpoint.java new file mode 100644 index 00000000000..58571b92ea5 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/ClientRoutesEndpoint.java @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.core.config; + +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.util.Objects; +import net.jcip.annotations.Immutable; + +/** + * Represents a client routes endpoint for PrivateLink-style deployments. + * + *

Each endpoint corresponds to a connection ID in the {@code system.client_routes} table, with + * an optional connection address that can be used as a seed host for initial connection. + */ +@Immutable +public class ClientRoutesEndpoint { + + private final String connectionId; + private final String connectionAddr; + + /** + * Creates a new endpoint with the given connection ID and no connection address. + * + * @param connectionId the connection ID (must not be null). + */ + public ClientRoutesEndpoint(@NonNull String connectionId) { + this(connectionId, null); + } + + /** + * Creates a new endpoint with the given connection ID and connection address. + * + * @param connectionId the connection ID (must not be null). + * @param connectionAddr the hostname of the seed contact point used for the initial connection + * (may be null). This is a plain hostname — not a {@code host:port} pair; the port + * is discovered from the {@code system.client_routes} table. + */ + public ClientRoutesEndpoint(@NonNull String connectionId, @Nullable String connectionAddr) { + this.connectionId = Objects.requireNonNull(connectionId, "connectionId must not be null"); + this.connectionAddr = connectionAddr; + } + + /** Returns the connection ID for this endpoint. */ + @NonNull + public String getConnectionId() { + return connectionId; + } + + /** + * Returns the hostname of the seed contact point for this endpoint, or null if not specified. + * + *

This is a plain hostname — not a {@code host:port} pair. The port for the initial + * connection is discovered from the {@code system.client_routes} table after the control + * connection is established. + * + *

When provided and no explicit contact points are given to the session builder, this hostname + * will be used as a seed for the initial connection. + */ + @Nullable + public String getConnectionAddr() { + return connectionAddr; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ClientRoutesEndpoint)) { + return false; + } + ClientRoutesEndpoint that = (ClientRoutesEndpoint) o; + return connectionId.equals(that.connectionId) + && Objects.equals(connectionAddr, that.connectionAddr); + } + + @Override + public int hashCode() { + return Objects.hash(connectionId, connectionAddr); + } + + @Override + public String toString() { + return "ClientRoutesEndpoint{" + + "connectionId=" + + connectionId + + ", connectionAddr='" + + connectionAddr + + '\'' + + '}'; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java index 9e0119903df..7052e8f34a8 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/DefaultDriverOption.java @@ -448,7 +448,6 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: {@link String} */ ADDRESS_TRANSLATOR_CLASS("advanced.address-translator.class"), - /** * The native protocol version to use. * @@ -1073,7 +1072,43 @@ public enum DefaultDriverOption implements DriverOption { *

Value-type: string */ LOAD_BALANCING_DEFAULT_LWT_REQUEST_ROUTING_METHOD( - "advanced.load-balancing-policy.default-lwt-request-routing-method"); + "advanced.load-balancing-policy.default-lwt-request-routing-method"), + + /** + * The list of client-routes endpoints for cloud private-endpoint (PrivateLink) deployments. + * + *

Each element is a HOCON object with the following fields: + * + *

    + *
  • {@code connection-id} (string, required) – opaque string that identifies the cloud + * private-endpoint connection in the {@code system.client_routes} table. + *
  • {@code connection-addr} (string, optional) – hostname of the seed contact point used for + * the initial connection. This is a plain hostname, not a {@code host:port} pair; + * the port is read from the {@code system.client_routes} table. + *
+ * + *

This option is read as a raw HOCON list-of-objects; it cannot be read via the flat {@link + * com.datastax.oss.driver.api.core.config.DriverExecutionProfile} typed getters. Parsing is + * performed directly from the underlying Typesafe {@code Config} object inside {@code + * DefaultDriverContext}. + * + *

Value type: list of HOCON objects + */ + CLIENT_ROUTES_ENDPOINTS("advanced.client-routes.endpoints"), + + /** + * The name of the system table to query for client-routes mappings. + * + *

Value type: {@link String} + */ + CLIENT_ROUTES_TABLE_NAME("advanced.client-routes.table-name"), + + /** + * How long resolved DNS entries are cached for client-routes endpoints. + * + *

Value-type: {@link java.time.Duration Duration} + */ + CLIENT_ROUTES_DNS_CACHE_DURATION("advanced.client-routes.dns-cache-duration"); private final String path; diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java index 28559ea8556..238a9e337e7 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/OptionsMap.java @@ -396,6 +396,10 @@ protected static void fillWithDriverDefaults(OptionsMap map) { map.put( TypedDriverOption.LOAD_BALANCING_DEFAULT_LWT_REQUEST_ROUTING_METHOD, "PRESERVE_REPLICA_ORDER"); + // CLIENT_ROUTES_ENDPOINTS is intentionally omitted: it is a list-of-objects (compound HOCON + // values) with no sensible scalar default, analogous to how CONFIG_RELOAD_INTERVAL is omitted. + map.put(TypedDriverOption.CLIENT_ROUTES_TABLE_NAME, "system.client_routes"); + map.put(TypedDriverOption.CLIENT_ROUTES_DNS_CACHE_DURATION, Duration.ofMillis(500)); } @Immutable diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java index 818468ee9d5..fad6e39d8ed 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/config/TypedDriverOption.java @@ -939,6 +939,22 @@ public String toString() { DefaultDriverOption.LOAD_BALANCING_DEFAULT_LWT_REQUEST_ROUTING_METHOD, GenericType.STRING); + /** + * The system table to query for client-routes endpoint mappings. + * + *

Note: {@link DefaultDriverOption#CLIENT_ROUTES_ENDPOINTS} intentionally has no typed + * equivalent here because its HOCON value is a list of compound objects (not a flat scalar type + * supported by the {@code DriverExecutionProfile} API); it is excluded from the {@code + * TypedDriverOptionTest} consistency check accordingly. + */ + public static final TypedDriverOption CLIENT_ROUTES_TABLE_NAME = + new TypedDriverOption<>(DefaultDriverOption.CLIENT_ROUTES_TABLE_NAME, GenericType.STRING); + + /** How long resolved DNS entries are cached for client-routes endpoints. */ + public static final TypedDriverOption CLIENT_ROUTES_DNS_CACHE_DURATION = + new TypedDriverOption<>( + DefaultDriverOption.CLIENT_ROUTES_DNS_CACHE_DURATION, GenericType.DURATION); + private static Iterable> introspectBuiltInValues() { try { ImmutableList.Builder> result = ImmutableList.builder(); diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/ProgrammaticArguments.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/ProgrammaticArguments.java index 4e08bd5434c..0373831da41 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/ProgrammaticArguments.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/ProgrammaticArguments.java @@ -18,6 +18,7 @@ package com.datastax.oss.driver.api.core.session; import com.datastax.oss.driver.api.core.auth.AuthProvider; +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; import com.datastax.oss.driver.api.core.loadbalancing.NodeDistanceEvaluator; import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.metadata.NodeStateListener; @@ -71,6 +72,7 @@ public static Builder builder() { private final String startupApplicationVersion; private final MutableCodecRegistry codecRegistry; private final Object metricRegistry; + private final ClientRoutesConfig clientRoutesConfig; private ProgrammaticArguments( @NonNull List> typeCodecs, @@ -88,7 +90,8 @@ private ProgrammaticArguments( @Nullable String startupApplicationName, @Nullable String startupApplicationVersion, @Nullable MutableCodecRegistry codecRegistry, - @Nullable Object metricRegistry) { + @Nullable Object metricRegistry, + @Nullable ClientRoutesConfig clientRoutesConfig) { this.typeCodecs = typeCodecs; this.nodeStateListener = nodeStateListener; @@ -106,6 +109,7 @@ private ProgrammaticArguments( this.startupApplicationVersion = startupApplicationVersion; this.codecRegistry = codecRegistry; this.metricRegistry = metricRegistry; + this.clientRoutesConfig = clientRoutesConfig; } @NonNull @@ -190,6 +194,11 @@ public Object getMetricRegistry() { return metricRegistry; } + @Nullable + public ClientRoutesConfig getClientRoutesConfig() { + return clientRoutesConfig; + } + public static class Builder { private final ImmutableList.Builder> typeCodecsBuilder = ImmutableList.builder(); @@ -210,6 +219,7 @@ public static class Builder { private String startupApplicationVersion; private MutableCodecRegistry codecRegistry; private Object metricRegistry; + private ClientRoutesConfig clientRoutesConfig; @NonNull public Builder addTypeCodecs(@NonNull TypeCodec... typeCodecs) { @@ -410,6 +420,12 @@ public Builder withMetricRegistry(@Nullable Object metricRegistry) { return this; } + @NonNull + public Builder withClientRoutesConfig(@Nullable ClientRoutesConfig clientRoutesConfig) { + this.clientRoutesConfig = clientRoutesConfig; + return this; + } + @NonNull public ProgrammaticArguments build() { return new ProgrammaticArguments( @@ -428,7 +444,8 @@ public ProgrammaticArguments build() { startupApplicationName, startupApplicationVersion, codecRegistry, - metricRegistry); + metricRegistry, + clientRoutesConfig); } } } diff --git a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java index 9402c77229f..7312beab62b 100644 --- a/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java +++ b/core/src/main/java/com/datastax/oss/driver/api/core/session/SessionBuilder.java @@ -28,6 +28,7 @@ import com.datastax.oss.driver.api.core.auth.AuthProvider; import com.datastax.oss.driver.api.core.auth.PlainTextAuthProviderBase; import com.datastax.oss.driver.api.core.auth.ProgrammaticPlainTextAuthProvider; +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; @@ -98,6 +99,7 @@ public abstract class SessionBuilder { protected Set programmaticContactPoints = new HashSet<>(); protected CqlIdentifier keyspace; protected Callable cloudConfigInputStream; + protected ClientRoutesConfig clientRoutesConfig; protected ProgrammaticArguments.Builder programmaticArgumentsBuilder = ProgrammaticArguments.builder(); @@ -735,6 +737,43 @@ public SelfT withCloudProxyAddress(@Nullable InetSocketAddress cloudProxyAddress return self; } + /** + * Configures this session to use client routes for cloud private-endpoint deployments. + * + *

Client routes enable the driver to discover and connect to nodes through a load balancer + * (such as AWS PrivateLink, Azure Private Link, or GCP Private Service Connect) by reading + * endpoint mappings from the {@code system.client_routes} table. Each endpoint is identified by a + * connection ID and maps to specific node addresses. + * + *

This configuration is mutually exclusive with a user-provided {@link + * com.datastax.oss.driver.api.core.addresstranslation.AddressTranslator}. If both are specified, + * an {@link IllegalStateException} is thrown when the session is built. + * + *

Example usage: + * + * 

{@code
+   * ClientRoutesConfig config = ClientRoutesConfig.builder()
+   *     .addEndpoint(new ClientRoutesEndpoint(
+   *         UUID.fromString("12345678-1234-1234-1234-123456789012"),
+   *         "my-cluster-endpoint.example.com:9042"))
+   *     .build();
+   *
+   * CqlSession session = CqlSession.builder()
+   *     .withClientRoutesConfig(config)
+   *     .build();
+   * }
+ * + * @param clientRoutesConfig the client routes configuration to use, or {@code null} to disable + * client routes. + * @see ClientRoutesConfig + */ + @NonNull + public SelfT withClientRoutesConfig(@Nullable ClientRoutesConfig clientRoutesConfig) { + this.clientRoutesConfig = clientRoutesConfig; + this.programmaticArgumentsBuilder.withClientRoutesConfig(clientRoutesConfig); + return self; + } + /** * A unique identifier for the created session. * @@ -829,6 +868,7 @@ public CompletionStage buildAsync() { CompletableFutures.propagateCancellation(wrapStage, buildStage); return wrapStage; } + /** * Convenience method to call {@link #buildAsync()} and block on the result. * @@ -896,6 +936,38 @@ protected final CompletionStage buildDefaultSessionAsync() { withSslEngineFactory(cloudConfig.getSslEngineFactory()); withCloudProxyAddress(cloudConfig.getProxyAddress()); programmaticArguments = programmaticArgumentsBuilder.build(); + + // Check for mutual exclusivity with client routes (programmatic or file-based) + boolean fileBasedClientRoutes = + defaultConfig.isDefined(DefaultDriverOption.CLIENT_ROUTES_ENDPOINTS); + if (clientRoutesConfig != null || fileBasedClientRoutes) { + throw new IllegalStateException( + "Both a secure connect bundle and client routes configuration were provided. " + + "They are mutually exclusive. Please use either a secure connect bundle OR client routes configuration, but not both."); + } + } + + // Determine whether client routes are active (programmatic or config-file). + boolean fileBasedClientRoutes = + defaultConfig.isDefined(DefaultDriverOption.CLIENT_ROUTES_ENDPOINTS); + + // Handle client routes mutual exclusivity with a custom AddressTranslator. + if (clientRoutesConfig != null || fileBasedClientRoutes) { + if (defaultConfig.isDefined(DefaultDriverOption.ADDRESS_TRANSLATOR_CLASS)) { + String translatorClass = + defaultConfig.getString(DefaultDriverOption.ADDRESS_TRANSLATOR_CLASS); + // PassThroughAddressTranslator is the default and is compatible with client routes. + if (!"PassThroughAddressTranslator".equals(translatorClass) + && !"com.datastax.oss.driver.internal.core.addresstranslation.PassThroughAddressTranslator" + .equals(translatorClass)) { + throw new IllegalStateException( + String.format( + "Both client routes configuration and a custom AddressTranslator ('%s') were " + + "provided. They are mutually exclusive. Please use either client routes " + + "OR a custom AddressTranslator, but not both.", + translatorClass)); + } + } } boolean resolveAddresses = diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java index 4fb9782f566..a96e6a6a5cc 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/addresstranslation/FixedHostNameAddressTranslator.java @@ -30,8 +30,10 @@ * using its native transport port. * *

The translator can be used for scenarios when all nodes are behind some kind of proxy, and it - * is not tailored for one concrete use case. One can use this, for example, for AWS PrivateLink as - * all nodes would be exposed to consumer - behind one hostname pointing to AWS Endpoint. + * is not tailored for one concrete use case. One can use this, for example, for cloud private + * endpoint services (such as AWS PrivateLink, Azure Private Link, or GCP Private Service Connect) + * where all nodes are exposed to the consumer behind one hostname pointing to a single load + * balancer endpoint. */ public class FixedHostNameAddressTranslator implements AddressTranslator { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java index 04809b400d9..7f3f2610f71 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/channel/ProtocolInitHandler.java @@ -65,6 +65,7 @@ import io.netty.channel.ChannelHandlerContext; import io.netty.channel.ChannelPipeline; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -170,10 +171,14 @@ private class InitRequest extends ChannelHandlerRequest { private Message request; private Authenticator authenticator; private ByteBuffer authResponseToken; + // Mutable copy of the event types to register; may be narrowed if the server rejects + // unsupported types (e.g. CLIENT_ROUTES_CHANGE on older ScyllaDB versions). + private final List registerEventTypes; InitRequest(ChannelHandlerContext ctx) { super(ctx, timeoutMillis); this.step = querySupportedOptions ? Step.OPTIONS : Step.STARTUP; + this.registerEventTypes = new ArrayList<>(options.eventTypes); } @Override @@ -200,7 +205,7 @@ Message getRequest() { case AUTH_RESPONSE: return request = new AuthResponse(authResponseToken); case REGISTER: - return request = new Register(options.eventTypes); + return request = new Register(registerEventTypes); default: throw new AssertionError("unhandled step: " + step); } @@ -323,7 +328,7 @@ void onResponse(Message response) { if (options.keyspace != null) { step = Step.SET_KEYSPACE; send(); - } else if (!options.eventTypes.isEmpty()) { + } else if (!registerEventTypes.isEmpty()) { step = Step.REGISTER; send(); } else { @@ -331,7 +336,7 @@ void onResponse(Message response) { } } } else if (step == Step.SET_KEYSPACE && response instanceof SetKeyspace) { - if (!options.eventTypes.isEmpty()) { + if (!registerEventTypes.isEmpty()) { step = Step.REGISTER; send(); } else { @@ -359,6 +364,34 @@ void onResponse(Message response) { } else if (step == Step.SET_KEYSPACE && error.code == ProtocolConstants.ErrorCode.INVALID) { fail(new InvalidKeyspaceException(error.message)); + } else if (step == Step.REGISTER + && error.code == ErrorCode.PROTOCOL_ERROR + && error.message.contains(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE)) { + // The server rejected CLIENT_ROUTES_CHANGE as an unknown event type. + // + // This is expected on ScyllaDB versions that do not implement the client_routes feature + // at the protocol level (Enterprise < 2026.1; not yet available on OSS). The driver + // registers for this event only when ClientRoutesConfig is set, so this branch fires + // on any pre-2026.1 Enterprise cluster that the user has (mis)configured with client + // routes. + // + // Behavior: strip CLIENT_ROUTES_CHANGE and retry REGISTER with the remaining event + // types (SCHEMA_CHANGE, STATUS_CHANGE, TOPOLOGY_CHANGE). The session connects + // successfully; client routes table queries may still work if the server exposes a + // compatible table, but live push-updates via this event will be absent. + LOG.warn( + "[{}] Server does not support {} event (requires ScyllaDB Enterprise ≥ 2026.1);" + + " retrying REGISTER without it — live client-route updates will be disabled", + logPrefix, + ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); + registerEventTypes.remove(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); + if (registerEventTypes.isEmpty()) { + // No event types left to register — proceed without event registration + setConnectSuccess(); + } else { + // Retry REGISTER with the remaining event types + send(); + } } else { failOnUnexpected(error); } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/CachingDnsResolver.java b/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/CachingDnsResolver.java new file mode 100644 index 00000000000..b51c6fd4fcd --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/CachingDnsResolver.java @@ -0,0 +1,198 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.clientroutes; + +import com.datastax.oss.driver.shaded.guava.common.annotations.VisibleForTesting; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Semaphore; +import java.util.concurrent.atomic.AtomicInteger; +import net.jcip.annotations.ThreadSafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ThreadSafe +public class CachingDnsResolver implements DnsResolver { + private static final Logger LOG = LoggerFactory.getLogger(CachingDnsResolver.class); + + private final long cacheDurationNanos; + /** + * Tracks one {@link Semaphore} per hostname that is currently being resolved, together with a + * reference count of how many threads are using it. The waiter count is incremented atomically + * inside {@link ConcurrentHashMap#compute} before any thread blocks, and decremented (also inside + * {@code compute}) after {@link Semaphore#release()}. Because {@code compute} holds the map's + * internal bucket lock for that key, the decrement-and-conditional-remove is one indivisible + * operation — closing the race that a plain {@code remove(key, value)} leaves open. + */ + private final ConcurrentHashMap semaphores = new ConcurrentHashMap<>(); + + private final ConcurrentHashMap cache = new ConcurrentHashMap<>(); + private final ConcurrentHashMap lastKnownGood = new ConcurrentHashMap<>(); + private final ThrowingFunction resolverFn; + + public CachingDnsResolver(long cacheDurationMillis) { + this(cacheDurationMillis, InetAddress::getByName); + } + + CachingDnsResolver(long cacheDurationMillis, ThrowingFunction resolverFn) { + this.cacheDurationNanos = cacheDurationMillis * 1_000_000L; + this.resolverFn = resolverFn; + } + + @Override + @NonNull + public InetAddress resolve(@NonNull String hostname) throws UnknownHostException { + // Fast path: unlocked read — avoids semaphore overhead on a warm cache. + CacheEntry entry = cachedEntry(hostname); + if (entry != null) { + return entry.address; + } + + // Atomically get-or-create the entry and increment the waiter count in one compute() call. + // This ensures the count is already > 0 before any other thread can observe the entry, + // preventing a premature removal by a concurrently finishing thread. + SemaphoreEntry semEntry = + semaphores.compute( + hostname, + (k, existing) -> { + if (existing == null) { + existing = new SemaphoreEntry(); + } + existing.waiters.incrementAndGet(); + return existing; + }); + + Semaphore semaphore = semEntry.semaphore; + if (!semaphore.tryAcquire()) { + // Another thread is already resolving this hostname. Block until it finishes, + // then re-check the cache (the other thread will have populated it). + try { + semaphore.acquire(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + // Decrement the waiter count we incremented above before bailing out. + semaphores.compute( + hostname, (k, v) -> (v != null && v.waiters.decrementAndGet() == 0) ? null : v); + throw new UnknownHostException( + "Interrupted while waiting for DNS resolution of " + hostname); + } + } + try { + // Contended path: the resolver that held the semaphore just finished — cache hit expected. + entry = cachedEntry(hostname); + if (entry != null) { + return entry.address; + } + // Cache still empty (e.g. the other thread failed); fall through to resolve ourselves. + return doResolve(hostname); + } finally { + semaphore.release(); + // Atomically decrement the waiter count and remove the map entry when it reaches zero. + // Using compute() here is essential: it holds the bucket lock while decrementing *and* + // deciding whether to remove, so no other thread can slip in between the two operations. + semaphores.compute( + hostname, (k, v) -> (v != null && v.waiters.decrementAndGet() == 0) ? null : v); + } + } + + /** Returns a non-expired {@link CacheEntry} for {@code hostname}, or {@code null}. */ + @Nullable + private CacheEntry cachedEntry(String hostname) { + CacheEntry entry = cache.get(hostname); + return (entry != null && System.nanoTime() < entry.expiryNanos) ? entry : null; + } + + /** + * Performs a real DNS lookup, stores the result in the cache and {@code lastKnownGood}, and + * returns the resolved address. Falls back to the last known good address on failure. + */ + private InetAddress doResolve(String hostname) throws UnknownHostException { + InetAddress address; + try { + address = resolverFn.apply(hostname); + } catch (UnknownHostException e) { + InetAddress fallback = lastKnownGood.get(hostname); + if (fallback != null) { + LOG.warn( + "DNS resolution failed for {}, using last known good address {}", hostname, fallback); + return fallback; + } + throw e; + } + cache.put(hostname, new CacheEntry(address, System.nanoTime() + cacheDurationNanos)); + lastKnownGood.put(hostname, address); + return address; + } + + @Override + public void evict(@NonNull String hostname) { + cache.remove(hostname); + lastKnownGood.remove(hostname); + } + + @Override + public void clearCache() { + cache.clear(); + // Semaphore entries are self-cleaning: the waiter count reaches zero and the entry is removed + // atomically inside compute() at the end of every resolution. Any entry that remains here + // belongs to a resolution currently in flight and must not be discarded. + // + // lastKnownGood is intentionally NOT cleared here. clearCache() is only called at session + // close; it is never called during a route-map refresh (CLIENT_ROUTES_CHANGE events or + // control-connection reconnects). Retaining last-known-good means the fallback address + // survives a close/reopen cycle and remains available if the first re-resolution fails. + } + + static class CacheEntry { + final InetAddress address; + final long expiryNanos; + + CacheEntry(InetAddress address, long expiryNanos) { + this.address = address; + this.expiryNanos = expiryNanos; + } + } + + @FunctionalInterface + interface ThrowingFunction { + R apply(T t) throws UnknownHostException; + } + + /** + * Wraps a {@link Semaphore} with a reference count ({@code waiters}) that tracks how many threads + * are currently using this entry. The count is managed exclusively through {@link + * ConcurrentHashMap#compute}, which serialises increments and decrements on the same key. + */ + static class SemaphoreEntry { + final Semaphore semaphore = new Semaphore(1); + final AtomicInteger waiters = new AtomicInteger(0); + } + + /** + * Returns the number of hostnames that currently have an active semaphore entry (i.e. have at + * least one resolution in flight). Visible for testing only — callers must not depend on the + * internal synchronisation mechanism or the map type. + */ + @VisibleForTesting + int semaphoreCount() { + return semaphores.size(); + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/ClientRouteInfo.java b/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/ClientRouteInfo.java new file mode 100644 index 00000000000..4cc9c773fd7 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/ClientRouteInfo.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2025 ScyllaDB + * + * Modified by ScyllaDB + */ +package com.datastax.oss.driver.internal.core.clientroutes; + +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.util.Objects; +import java.util.UUID; +import net.jcip.annotations.Immutable; + +/** + * Represents a single row from the system.client_routes table. + * + *

Each row maps a connection_id + host_id pair to an address that must be DNS-resolved before + * use. + */ +@Immutable +public class ClientRouteInfo { + + private final String connectionId; + private final UUID hostId; + private final String address; + private final Integer nativeTransportPort; + private final Integer nativeTransportPortSsl; + + public ClientRouteInfo( + @NonNull String connectionId, + @NonNull UUID hostId, + @NonNull String address, + @Nullable Integer nativeTransportPort, + @Nullable Integer nativeTransportPortSsl) { + this.connectionId = Objects.requireNonNull(connectionId, "connectionId must not be null"); + this.hostId = Objects.requireNonNull(hostId, "hostId must not be null"); + this.address = Objects.requireNonNull(address, "address must not be null"); + this.nativeTransportPort = nativeTransportPort; + this.nativeTransportPortSsl = nativeTransportPortSsl; + } + + @NonNull + public String getConnectionId() { + return connectionId; + } + + @NonNull + public UUID getHostId() { + return hostId; + } + + @NonNull + public String getAddress() { + return address; + } + + @Nullable + public Integer getNativeTransportPort() { + return nativeTransportPort; + } + + @Nullable + public Integer getNativeTransportPortSsl() { + return nativeTransportPortSsl; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof ClientRouteInfo)) { + return false; + } + ClientRouteInfo that = (ClientRouteInfo) o; + return connectionId.equals(that.connectionId) + && hostId.equals(that.hostId) + && address.equals(that.address) + && Objects.equals(nativeTransportPort, that.nativeTransportPort) + && Objects.equals(nativeTransportPortSsl, that.nativeTransportPortSsl); + } + + @Override + public int hashCode() { + return Objects.hash(connectionId, hostId, address, nativeTransportPort, nativeTransportPortSsl); + } + + @Override + public String toString() { + return "ClientRouteInfo{" + + "connectionId=" + + connectionId + + ", hostId=" + + hostId + + ", address='" + + address + + '\'' + + ", nativeTransportPort=" + + nativeTransportPort + + ", nativeTransportPortSsl=" + + nativeTransportPortSsl + + '}'; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/DnsResolver.java b/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/DnsResolver.java new file mode 100644 index 00000000000..0316c9364cf --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/DnsResolver.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.clientroutes; + +import edu.umd.cs.findbugs.annotations.NonNull; +import java.net.InetAddress; +import java.net.UnknownHostException; + +/** + * Interface for DNS resolution with caching support. + * + *

Implementations should provide caching to avoid DNS storms during application startup and + * should handle resolution failures gracefully (e.g., by returning the last known good IP). + * + *

This interface is part of the internal API and is not intended for public use. + */ +public interface DnsResolver { + + /** + * Resolves a hostname to an IP address, potentially using a cache. + * + *

Implementations should: + * + *

    + *
  • Cache successful resolutions for a configurable duration (default: 500ms per + * Description.md) + *
  • Return cached results when available and not expired + *
  • On failure, return the last known good IP if available + *
  • Limit concurrency to avoid DNS storms (default: 1 concurrent resolution per hostname) + *
+ * + * @param hostname the hostname to resolve (must not be null) + * @return the resolved IP address + * @throws UnknownHostException if the hostname cannot be resolved and no cached value is + * available + */ + @NonNull + InetAddress resolve(@NonNull String hostname) throws UnknownHostException; + + /** + * Evicts a single hostname from the DNS cache, forcing a fresh lookup on the next {@link + * #resolve} call. The last-known-good address for the hostname is also removed so that a stale + * fallback is not returned if the new lookup fails. + * + *

This is called when a route's hostname changes (detected during a merge), so that the next + * connection attempt to that host resolves the updated DNS name rather than the cached one. + * + * @param hostname the hostname whose cache entry should be invalidated + */ + void evict(@NonNull String hostname); + + /** + * Clears all cached DNS entries, but intentionally retains last-known-good addresses so that the + * fallback mechanism continues to work after a cache flush. + * + *

This method is called exactly once, by {@code ClientRoutesHandler.close()}, when the session + * is shut down. It is not called during route-map refreshes (triggered by {@code + * CLIENT_ROUTES_CHANGE} events or control-connection reconnects); those refreshes re-query the + * routes table but let the TTL-based DNS cache expire naturally. + * + *

This method is not intended for manual or administrative use outside of session close. + */ + void clearCache(); +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/ResolvedClientRoute.java b/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/ResolvedClientRoute.java new file mode 100644 index 00000000000..517ce3722ff --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/clientroutes/ResolvedClientRoute.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.clientroutes; + +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.util.Objects; +import java.util.UUID; +import net.jcip.annotations.Immutable; + +@Immutable +public class ResolvedClientRoute { + + private final UUID hostId; + private final String hostname; + private final Integer nativeTransportPort; + private final Integer nativeTransportPortSsl; + + public ResolvedClientRoute( + @NonNull UUID hostId, + @NonNull String hostname, + @Nullable Integer nativeTransportPort, + @Nullable Integer nativeTransportPortSsl) { + this.hostId = Objects.requireNonNull(hostId, "hostId must not be null"); + this.hostname = Objects.requireNonNull(hostname, "hostname must not be null"); + this.nativeTransportPort = nativeTransportPort; + this.nativeTransportPortSsl = nativeTransportPortSsl; + } + + @NonNull + public UUID getHostId() { + return hostId; + } + + @NonNull + public String getHostname() { + return hostname; + } + + @Nullable + public Integer getNativeTransportPort() { + return nativeTransportPort; + } + + @Nullable + public Integer getNativeTransportPortSsl() { + return nativeTransportPortSsl; + } + + @Override + public String toString() { + return "ResolvedClientRoute{" + + "hostId=" + + hostId + + ", hostname='" + + hostname + + '\'' + + ", nativeTransportPort=" + + nativeTransportPort + + ", nativeTransportPortSsl=" + + nativeTransportPortSsl + + '}'; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/config/typesafe/TypesafeDriverConfig.java b/core/src/main/java/com/datastax/oss/driver/internal/core/config/typesafe/TypesafeDriverConfig.java index e1d8c779f2c..2337bf82bc2 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/config/typesafe/TypesafeDriverConfig.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/config/typesafe/TypesafeDriverConfig.java @@ -31,6 +31,7 @@ import com.typesafe.config.ConfigValue; import com.typesafe.config.ConfigValueFactory; import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; import java.net.URL; import java.util.Map; import java.util.Optional; @@ -165,6 +166,24 @@ public DriverExecutionProfile getProfile(@NonNull String profileName) { return profiles; } + /** + * Returns the raw Typesafe {@link Config} backing the given profile, or {@code null} if the + * profile is not a Typesafe-based profile. + * + *

This method is public so that other internal components in different packages (e.g., {@code + * DefaultDriverContext}) can perform Typesafe-specific operations such as {@code getConfigList()} + * that are not available through the {@link DriverExecutionProfile} API. It is intentionally + * restricted to the {@code internal} package hierarchy and should not be considered part of the + * public driver API. + */ + @Nullable + public static Config getRawConfig(@NonNull DriverExecutionProfile profile) { + if (profile instanceof TypesafeDriverExecutionProfile) { + return ((TypesafeDriverExecutionProfile) profile).getEffectiveOptions(); + } + return null; + } + /** * Replace the given options, only if the original values came from {@code * reference.conf}: if the option was set explicitly in {@code application.conf}, then the diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java index cc725994d7c..ad2b76e190d 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/DefaultDriverContext.java @@ -28,6 +28,8 @@ import com.datastax.oss.driver.api.core.ProtocolVersion; import com.datastax.oss.driver.api.core.addresstranslation.AddressTranslator; import com.datastax.oss.driver.api.core.auth.AuthProvider; +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; import com.datastax.oss.driver.api.core.config.DefaultDriverOption; import com.datastax.oss.driver.api.core.config.DriverConfig; import com.datastax.oss.driver.api.core.config.DriverConfigLoader; @@ -55,7 +57,9 @@ import com.datastax.oss.driver.internal.core.channel.ChannelFactory; import com.datastax.oss.driver.internal.core.channel.DefaultWriteCoalescer; import com.datastax.oss.driver.internal.core.channel.WriteCoalescer; +import com.datastax.oss.driver.internal.core.config.typesafe.TypesafeDriverConfig; import com.datastax.oss.driver.internal.core.control.ControlConnection; +import com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.CloudTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.DefaultTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.LoadBalancingPolicyWrapper; @@ -102,6 +106,9 @@ import com.datastax.oss.protocol.internal.ProtocolV5ClientCodecs; import com.datastax.oss.protocol.internal.ProtocolV6ClientCodecs; import com.datastax.oss.protocol.internal.SegmentCodec; +import com.typesafe.config.Config; +import com.typesafe.config.ConfigList; +import com.typesafe.config.ConfigObject; import edu.umd.cs.findbugs.annotations.NonNull; import edu.umd.cs.findbugs.annotations.Nullable; import io.netty.buffer.ByteBuf; @@ -197,6 +204,7 @@ public class DefaultDriverContext implements InternalDriverContext { "loadBalancingPolicyWrapper", this::buildLoadBalancingPolicyWrapper, cycleDetector); private final LazyReference controlConnectionRef = new LazyReference<>("controlConnection", this::buildControlConnection, cycleDetector); + private final LazyReference clientRoutesHandlerRef; private final LazyReference requestProcessorRegistryRef = new LazyReference<>( "requestProcessorRegistry", this::buildRequestProcessorRegistry, cycleDetector); @@ -239,6 +247,7 @@ public class DefaultDriverContext implements InternalDriverContext { private final Map nodeDistanceEvaluatorsFromBuilder; private final ClassLoader classLoader; private final InetSocketAddress cloudProxyAddress; + private final ClientRoutesConfig clientRoutesConfigFromBuilder; private final LazyReference requestLogFormatterRef = new LazyReference<>("requestLogFormatter", this::buildRequestLogFormatter, cycleDetector); private final UUID startupClientId; @@ -294,6 +303,12 @@ public DefaultDriverContext( this.nodeDistanceEvaluatorsFromBuilder = programmaticArguments.getNodeDistanceEvaluators(); this.classLoader = programmaticArguments.getClassLoader(); this.cloudProxyAddress = programmaticArguments.getCloudProxyAddress(); + this.clientRoutesConfigFromBuilder = programmaticArguments.getClientRoutesConfig(); + this.clientRoutesHandlerRef = + new LazyReference<>( + "clientRoutesHandler", + () -> buildClientRoutesHandler(clientRoutesConfigFromBuilder), + cycleDetector); this.startupClientId = programmaticArguments.getStartupClientId(); this.startupApplicationName = programmaticArguments.getStartupApplicationName(); this.startupApplicationVersion = programmaticArguments.getStartupApplicationVersion(); @@ -418,6 +433,123 @@ protected AddressTranslator buildAddressTranslator() { DefaultDriverOption.ADDRESS_TRANSLATOR_CLASS))); } + protected ClientRoutesTopologyMonitor buildClientRoutesHandler( + ClientRoutesConfig clientRoutesConfigFromBuilder) { + ClientRoutesConfig configFromFile = buildClientRoutesConfigFromFile(); + if (clientRoutesConfigFromBuilder != null) { + if (configFromFile != null) { + LOG.warn( + "[{}] Both programmatic ClientRoutesConfig and '{}' were provided. " + + "The programmatic configuration takes precedence.", + getSessionName(), + DefaultDriverOption.CLIENT_ROUTES_ENDPOINTS.getPath()); + } + return new ClientRoutesTopologyMonitor(this, clientRoutesConfigFromBuilder); + } + if (configFromFile != null) { + return new ClientRoutesTopologyMonitor(this, configFromFile); + } + + return null; + } + + /** + * Reads the {@code advanced.client-routes} section from the HOCON configuration and builds a + * {@link ClientRoutesConfig}, or returns {@code null} if {@code endpoints} is not defined. + * + *

The {@code endpoints} value is a HOCON list of objects; it cannot be read through the flat + * {@link com.datastax.oss.driver.api.core.config.DriverExecutionProfile} typed API, so we access + * the underlying Typesafe {@link Config} directly via {@link TypesafeDriverConfig#getRawConfig}. + */ + @Nullable + // Package-private to allow unit-testing without a live cluster. + ClientRoutesConfig buildClientRoutesConfigFromFile() { + DriverExecutionProfile defaultProfile = config.getDefaultProfile(); + if (!defaultProfile.isDefined(DefaultDriverOption.CLIENT_ROUTES_ENDPOINTS)) { + return null; + } + + Config rawConfig = TypesafeDriverConfig.getRawConfig(defaultProfile); + if (rawConfig == null) { + // Non-Typesafe config backend: endpoints option is marked as defined but we cannot parse + // the compound-object list. Warn and skip. + LOG.warn( + "[{}] '{}' is defined but the underlying config implementation is not Typesafe-based; " + + "config-file client routes cannot be parsed. Use the programmatic API instead.", + getSessionName(), + DefaultDriverOption.CLIENT_ROUTES_ENDPOINTS.getPath()); + return null; + } + + String endpointsPath = DefaultDriverOption.CLIENT_ROUTES_ENDPOINTS.getPath(); + ConfigList endpointsList; + try { + endpointsList = rawConfig.getList(endpointsPath); + } catch (Exception e) { + throw new IllegalArgumentException( + String.format( + "Failed to read '%s' from configuration: %s. " + + "Expected a list of objects, each with a required 'connection-id' field " + + "and an optional 'connection-addr' field (plain hostname, no port), for example: " + + "%s = [{ connection-id = \"\", connection-addr = \"host.example.com\" }]", + endpointsPath, e.getMessage(), endpointsPath), + e); + } + + if (endpointsList.isEmpty()) { + throw new IllegalArgumentException( + String.format( + "'%s' is set but contains no entries; at least one endpoint is required. " + + "Each entry must be a HOCON object with a 'connection-id' field (UUID string) " + + "and an optional 'connection-addr' field (plain hostname, no port), for example: " + + "%s = [{ connection-id = \"\", connection-addr = \"host.example.com\" }]", + endpointsPath, endpointsPath)); + } + + ClientRoutesConfig.Builder builder = ClientRoutesConfig.builder(); + + for (int i = 0; i < endpointsList.size(); i++) { + if (!(endpointsList.get(i) instanceof ConfigObject)) { + throw new IllegalArgumentException( + String.format( + "'%s[%d]' must be a HOCON object but got a %s. " + + "Each entry must have a required 'connection-id' field (UUID string) " + + "and an optional 'connection-addr' field (plain hostname, no port), for example: " + + "{ connection-id = \"\", connection-addr = \"host.example.com\" }. " + + "Got: %s", + endpointsPath, i, endpointsList.get(i).valueType(), endpointsList.get(i))); + } + Config entry = ((ConfigObject) endpointsList.get(i)).toConfig(); + + if (!entry.hasPath("connection-id")) { + throw new IllegalArgumentException( + String.format( + "'%s[%d]' is missing the required 'connection-id' field. " + + "Each entry must be a HOCON object with a 'connection-id' field (UUID string) " + + "and an optional 'connection-addr' field (plain hostname, no port), for example: " + + "{ connection-id = \"\", connection-addr = \"host.example.com\" }", + endpointsPath, i)); + } + String connectionId = entry.getString("connection-id"); + String connectionAddr = + entry.hasPath("connection-addr") ? entry.getString("connection-addr") : null; + builder.addEndpoint(new ClientRoutesEndpoint(connectionId, connectionAddr)); + } + + // Read scalar options through the standard profile API + if (defaultProfile.isDefined(DefaultDriverOption.CLIENT_ROUTES_TABLE_NAME)) { + builder.withTableName(defaultProfile.getString(DefaultDriverOption.CLIENT_ROUTES_TABLE_NAME)); + } + if (defaultProfile.isDefined(DefaultDriverOption.CLIENT_ROUTES_DNS_CACHE_DURATION)) { + builder.withDnsCacheDuration( + defaultProfile + .getDuration(DefaultDriverOption.CLIENT_ROUTES_DNS_CACHE_DURATION) + .toMillis()); + } + + return builder.build(); + } + protected Optional buildSslEngineFactory(SslEngineFactory factoryFromBuilder) { return (factoryFromBuilder != null) ? Optional.of(factoryFromBuilder) @@ -492,10 +624,13 @@ protected ChannelFactory buildChannelFactory() { } protected TopologyMonitor buildTopologyMonitor() { - if (cloudProxyAddress == null) { - return new DefaultTopologyMonitor(this); + if (cloudProxyAddress != null) { + return new CloudTopologyMonitor(this, cloudProxyAddress); + } + if (clientRoutesConfigFromBuilder != null) { + return new ClientRoutesTopologyMonitor(this, clientRoutesConfigFromBuilder); } - return new CloudTopologyMonitor(this, cloudProxyAddress); + return new DefaultTopologyMonitor(this); } protected MetadataManager buildMetadataManager() { @@ -905,6 +1040,12 @@ public ControlConnection getControlConnection() { return controlConnectionRef.get(); } + @Nullable + @Override + public ClientRoutesTopologyMonitor getClientRoutesHandler() { + return clientRoutesHandlerRef.get(); + } + @NonNull @Override public RequestProcessorRegistry getRequestProcessorRegistry() { diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/context/InternalDriverContext.java b/core/src/main/java/com/datastax/oss/driver/internal/core/context/InternalDriverContext.java index 81349b0c665..236fce62b8a 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/context/InternalDriverContext.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/context/InternalDriverContext.java @@ -26,6 +26,7 @@ import com.datastax.oss.driver.internal.core.channel.ChannelFactory; import com.datastax.oss.driver.internal.core.channel.WriteCoalescer; import com.datastax.oss.driver.internal.core.control.ControlConnection; +import com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.LoadBalancingPolicyWrapper; import com.datastax.oss.driver.internal.core.metadata.MetadataManager; import com.datastax.oss.driver.internal.core.metadata.TopologyMonitor; @@ -108,6 +109,15 @@ public interface InternalDriverContext extends DriverContext { @NonNull ControlConnection getControlConnection(); + /** + * Returns the client routes handler if configured, or null if client routes are not enabled. + * + *

Client routes are enabled when a {@link + * com.datastax.oss.driver.api.core.config.ClientRoutesConfig} is provided to the session builder. + */ + @Nullable + ClientRoutesTopologyMonitor getClientRoutesHandler(); + @NonNull RequestProcessorRegistry getRequestProcessorRegistry(); diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java index 5e5e5fa84ee..fd3653f51d9 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnection.java @@ -31,6 +31,7 @@ import com.datastax.oss.driver.internal.core.channel.DriverChannelOptions; import com.datastax.oss.driver.internal.core.channel.EventCallback; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.metadata.ClientRoutesChangeEvent; import com.datastax.oss.driver.internal.core.metadata.DefaultTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.DistanceEvent; import com.datastax.oss.driver.internal.core.metadata.MetadataManager; @@ -190,6 +191,9 @@ public void onEvent(Message eventMessage) { case ProtocolConstants.EventType.SCHEMA_CHANGE: processSchemaChange(event); break; + case ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE: + processClientRoutesChange(event); + break; default: LOG.warn("[{}] Unsupported event type: {}", logPrefix, event.type); } @@ -242,6 +246,15 @@ private void processSchemaChange(Event event) { }); } + private void processClientRoutesChange(Event event) { + com.datastax.oss.protocol.internal.response.event.ClientRoutesChangeEvent crce = + (com.datastax.oss.protocol.internal.response.event.ClientRoutesChangeEvent) event; + LOG.debug("[{}] Received CLIENT_ROUTES_CHANGE event: {}", logPrefix, crce); + context + .getEventBus() + .fire(new ClientRoutesChangeEvent(crce.changeType, crce.connectionIds, crce.hostIds)); + } + private class SingleThreaded { private final InternalDriverContext context; private final DriverConfig config; @@ -292,7 +305,9 @@ private void init( } initWasCalled = true; try { - ImmutableList eventTypes = buildEventTypes(listenToClusterEvents); + boolean listenClientRoutesEvents = context.getClientRoutesHandler() != null; + ImmutableList eventTypes = + buildEventTypes(listenToClusterEvents, listenClientRoutesEvents); LOG.debug("[{}] Initializing with event types {}", logPrefix, eventTypes); channelOptions = DriverChannelOptions.builder() @@ -467,6 +482,8 @@ private void onSuccessfulReconnect() { // Otherwise, perform a full refresh (we don't know how long we were disconnected) if (!isFirstConnection) { + context.getEventBus().fire(ControlConnectionReconnectEvent.INSTANCE); + context .getMetadataManager() .refreshNodes() @@ -595,7 +612,7 @@ private boolean isAuthFailure(Throwable error) { if (error instanceof AllNodesFailedException) { Collection> errors = ((AllNodesFailedException) error).getAllErrors().values(); - if (errors.size() == 0) { + if (errors.isEmpty()) { return false; } for (List nodeErrors : errors) { @@ -609,7 +626,8 @@ private boolean isAuthFailure(Throwable error) { return true; } - private static ImmutableList buildEventTypes(boolean listenClusterEvents) { + private static ImmutableList buildEventTypes( + boolean listenClusterEvents, boolean listenClientRoutesEvents) { ImmutableList.Builder builder = ImmutableList.builder(); builder.add(ProtocolConstants.EventType.SCHEMA_CHANGE); if (listenClusterEvents) { @@ -617,6 +635,9 @@ private static ImmutableList buildEventTypes(boolean listenClusterEvents .add(ProtocolConstants.EventType.STATUS_CHANGE) .add(ProtocolConstants.EventType.TOPOLOGY_CHANGE); } + if (listenClientRoutesEvents) { + builder.add(ProtocolConstants.EventType.CLIENT_ROUTES_CHANGE); + } return builder.build(); } } diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnectionReconnectEvent.java b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnectionReconnectEvent.java new file mode 100644 index 00000000000..a3b2d9b8e83 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/control/ControlConnectionReconnectEvent.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.control; + +import net.jcip.annotations.Immutable; + +/** + * Fired on the internal event bus when the control connection successfully reconnects after a + * disconnection. Components that need to refresh their state after a reconnection (e.g. {@link + * com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor}) can listen for this + * event. + */ +@Immutable +public class ControlConnectionReconnectEvent { + + public static final ControlConnectionReconnectEvent INSTANCE = + new ControlConnectionReconnectEvent(); + + private ControlConnectionReconnectEvent() {} + + @Override + public String toString() { + return "ControlConnectionReconnectEvent"; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesChangeEvent.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesChangeEvent.java new file mode 100644 index 00000000000..d1c93b45d41 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesChangeEvent.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.List; +import net.jcip.annotations.Immutable; + +/** + * Fired on the internal event bus when a {@code CLIENT_ROUTES_CHANGE} protocol event is received on + * the control connection. The {@link ClientRoutesTopologyMonitor} listens for this event and + * triggers a refresh of the client routes cache. + * + *

Carries the data from the protocol event: + * + *

    + *
  • {@link #changeType} — the type of change (e.g. {@code "UPDATED"}) + *
  • {@link #connectionIds} — opaque string identifiers of the affected connections + *
  • {@link #hostIds} — UUIDs of the affected hosts + *
+ */ +@Immutable +public class ClientRoutesChangeEvent { + + public final String changeType; + public final List connectionIds; + public final List hostIds; + + public ClientRoutesChangeEvent( + @NonNull String changeType, + @NonNull List connectionIds, + @NonNull List hostIds) { + this.changeType = changeType; + this.connectionIds = connectionIds; + this.hostIds = hostIds; + } + + @Override + public String toString() { + return "ClientRoutesChangeEvent(" + + "changeType='" + + changeType + + "', connectionIds=" + + connectionIds + + ", hostIds=" + + hostIds + + ")"; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java new file mode 100644 index 00000000000..bd32a3ffe01 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesEndPoint.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.util.Objects; +import java.util.UUID; + +public class ClientRoutesEndPoint implements EndPoint { + + private final UUID hostID; + private final ClientRoutesTopologyMonitor topologyMonitor; + private final String metricPrefix; + + /** + * @param topologyMonitor the topology monitor used to resolve the endpoint address on demand. + * @param hostID the host UUID identifying this node in the cluster. + * @param broadcastInetAddress the node's broadcast address (from system.peers or system.local), + * used to build a stable metric prefix. May be {@code null} if the address could not be + * determined, in which case the hostID is used as the metric prefix instead. + */ + public ClientRoutesEndPoint( + @NonNull ClientRoutesTopologyMonitor topologyMonitor, + @NonNull UUID hostID, + @Nullable InetAddress broadcastInetAddress) { + this.topologyMonitor = + Objects.requireNonNull(topologyMonitor, "Topology monitor cannot be null"); + this.hostID = Objects.requireNonNull(hostID, "HOST uuid cannot be null"); + this.metricPrefix = buildMetricPrefix(broadcastInetAddress, hostID); + } + + @NonNull + public UUID getHostID() { + return hostID; + } + + @NonNull + @Override + public InetSocketAddress resolve() { + try { + return topologyMonitor.resolve(hostID); + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + + @Override + public boolean equals(Object other) { + if (other == this) { + return true; + } else if (other instanceof ClientRoutesEndPoint) { + ClientRoutesEndPoint that = (ClientRoutesEndPoint) other; + return this.hostID.equals(that.hostID); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(hostID); + } + + @Override + public String toString() { + // Note that this uses the original proxy address, so if there are multiple A-records it won't + // show which one was selected. If that turns out to be a problem for debugging, we might need + // to store the result of resolve() in Connection and log that instead of the endpoint. + return hostID.toString(); + } + + @NonNull + @Override + public String asMetricPrefix() { + return metricPrefix; + } + + private static String buildMetricPrefix(@Nullable InetAddress address, @NonNull UUID hostID) { + if (address == null) { + return hostID.toString(); + } + // getHostAddress() returns clean IP without leading slash: + // IPv4: "127.0.0.1" IPv6: "0:0:0:0:0:0:0:1" + // Replace dots for IPv4; colons are kept for IPv6 (consistent with DefaultEndPoint) + return address.getHostAddress().replace('.', '_') + '_' + hostID; + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java new file mode 100644 index 00000000000..6703fb911f7 --- /dev/null +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitor.java @@ -0,0 +1,404 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.metadata.EndPoint; +import com.datastax.oss.driver.internal.core.adminrequest.AdminRequestHandler; +import com.datastax.oss.driver.internal.core.adminrequest.AdminRow; +import com.datastax.oss.driver.internal.core.channel.DriverChannel; +import com.datastax.oss.driver.internal.core.clientroutes.CachingDnsResolver; +import com.datastax.oss.driver.internal.core.clientroutes.DnsResolver; +import com.datastax.oss.driver.internal.core.clientroutes.ResolvedClientRoute; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.control.ControlConnectionReconnectEvent; +import edu.umd.cs.findbugs.annotations.NonNull; +import edu.umd.cs.findbugs.annotations.Nullable; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; +import net.jcip.annotations.ThreadSafe; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@ThreadSafe +public class ClientRoutesTopologyMonitor extends DefaultTopologyMonitor { + private static final Logger LOG = LoggerFactory.getLogger(ClientRoutesTopologyMonitor.class); + + private static final String SELECT_ROUTES_COLUMNS = + "SELECT connection_id, host_id, address, port, tls_port FROM %s"; + + final InternalDriverContext context; + + private final ClientRoutesConfig config; + private final String logPrefix; + final AtomicReference> resolvedRoutesCache; + private final DnsResolver dnsResolver; + private final boolean useSSL; + private volatile boolean closed = false; + + private volatile Object clientRoutesChangeKey; + private volatile Object reconnectKey; + + public ClientRoutesTopologyMonitor( + @NonNull InternalDriverContext context, @NonNull ClientRoutesConfig config) { + this(context, config, new CachingDnsResolver(config.getDnsCacheDurationMillis())); + } + + /** Package-private constructor for testing — allows injecting a mock {@link DnsResolver}. */ + ClientRoutesTopologyMonitor( + @NonNull InternalDriverContext context, + @NonNull ClientRoutesConfig config, + @NonNull DnsResolver dnsResolver) { + super(context); + this.context = context; + this.config = config; + this.logPrefix = context.getSessionName(); + this.resolvedRoutesCache = new AtomicReference<>(new HashMap<>()); + this.dnsResolver = dnsResolver; + this.useSSL = context.getSslEngineFactory().isPresent(); + } + + @Override + public CompletionStage init() { + this.clientRoutesChangeKey = + context + .getEventBus() + .register(ClientRoutesChangeEvent.class, this::onClientRoutesChangeEvent); + this.reconnectKey = + context + .getEventBus() + .register(ControlConnectionReconnectEvent.class, this::onReconnectEvent); + // Pre-load client routes before topology discovery so that buildNodeEndPoint can decide + // whether to use ClientRoutesEndPoint (routes available) or fall back to DefaultEndPoint + // (table absent / empty — e.g. older Scylla versions without system.client_routes). + return queryAndResolveRoutes(false).thenCompose(ignored -> super.init()); + } + + /** Returns the {@link ClientRoutesConfig} this handler was built from. */ + @NonNull + public ClientRoutesConfig getClientRoutesConfig() { + return config; + } + + @NonNull + public InetSocketAddress resolve(@NonNull UUID hostId) + throws IllegalStateException, UnknownHostException { + if (closed) { + throw new IllegalStateException("Topology monitor is closed"); + } + ResolvedClientRoute route = resolvedRoutesCache.get().get(hostId); + if (route == null) { + throw new IllegalStateException( + String.format("No client route found for host_id=%s", hostId)); + } + + // Select port based on SSL configuration + Integer port; + if (useSSL) { + port = route.getNativeTransportPortSsl(); + if (port == null) { + port = route.getNativeTransportPort(); + LOG.warn( + "SSL requested for host_id={} ({}) but tls_port is not configured in client routes. " + + "Falling back to non-SSL port {}. This may indicate a configuration issue.", + hostId, + route.getHostname(), + port); + } + } else { + port = route.getNativeTransportPort(); + } + + if (port == null) { + throw new IllegalStateException( + String.format( + "No port configured for host_id=%s, hostname=%s. " + + "The system.client_routes table may be incomplete.", + hostId, route.getHostname())); + } + + return new InetSocketAddress(dnsResolver.resolve(route.getHostname()), port); + } + + /** + * Refreshes the client routes cache by querying the configured client routes table. Errors are + * logged but do not propagate, so that periodic refreshes don't interrupt driver operation. + */ + public CompletionStage refresh() { + return queryAndResolveRoutes(false, null, null); + } + + private CompletionStage queryAndResolveRoutes(boolean propagateErrors) { + return queryAndResolveRoutes(propagateErrors, null, null); + } + + private CompletionStage queryAndResolveRoutes( + boolean propagateErrors, + @Nullable List eventConnectionIds, + @Nullable List eventHostIds) { + DriverChannel channel = context.getControlConnection().channel(); + if (channel == null) { + LOG.warn("[{}] Control connection channel is null, cannot query client routes", logPrefix); + return CompletableFuture.completedFuture(null); + } + + List configuredConnectionIds = + config.getEndpoints().stream() + .map(ClientRoutesEndpoint::getConnectionId) + .collect(Collectors.toList()); + + String query = buildQuery(configuredConnectionIds, eventConnectionIds, eventHostIds); + // A targeted refresh (host IDs known) merges into the existing cache rather than replacing it + boolean isTargetedRefresh = eventHostIds != null && !eventHostIds.isEmpty(); + + Duration timeout = + context + .getConfig() + .getDefaultProfile() + .getDuration(DefaultDriverOption.CONTROL_CONNECTION_TIMEOUT); + + try { + CompletionStage future = + AdminRequestHandler.query(channel, query, timeout, Integer.MAX_VALUE, logPrefix) + .start() + .thenAccept( + adminResult -> { + Map newRoutes = new HashMap<>(); + for (AdminRow row : adminResult) { + if (row.isNull("host_id") || row.isNull("address") || row.isNull("port")) { + LOG.warn("[{}] Skipping incomplete client_routes row: {}", logPrefix, row); + continue; + } + UUID hostId = row.getUuid("host_id"); + String address = row.getString("address"); + Integer port = row.getInteger("port"); + Integer tlsPort = + row.contains("tls_port") && !row.isNull("tls_port") + ? row.getInteger("tls_port") + : null; + newRoutes.put( + hostId, new ResolvedClientRoute(hostId, address, port, tlsPort)); + } + + if (isTargetedRefresh) { + // Merge: update only the returned host IDs, keep all others unchanged + mergeRoutes(newRoutes); + LOG.debug( + "[{}] Merged {} client routes (targeted refresh)", + logPrefix, + newRoutes.size()); + } else { + resolvedRoutesCache.set(newRoutes); + LOG.debug( + "[{}] Updated client routes: {} routes loaded", + logPrefix, + newRoutes.size()); + } + }); + + if (propagateErrors) { + return future; + } else { + return future.exceptionally( + e -> { + LOG.warn("[{}] Failed to query client routes: {}", logPrefix, e.getMessage(), e); + return null; + }); + } + } catch (Exception e) { + LOG.warn("[{}] Exception while querying client routes: {}", logPrefix, e.getMessage(), e); + if (propagateErrors) { + CompletableFuture failed = new CompletableFuture<>(); + failed.completeExceptionally(e); + return failed; + } + return CompletableFuture.completedFuture(null); + } + } + + private void onClientRoutesChangeEvent(ClientRoutesChangeEvent event) { + if (closed) { + return; + } + LOG.debug("[{}] Received {}, refreshing routes", logPrefix, event); + queryAndResolveRoutes(false, event.connectionIds, event.hostIds); + } + + private void onReconnectEvent(@SuppressWarnings("unused") ControlConnectionReconnectEvent event) { + if (closed) { + return; + } + LOG.debug("[{}] Control connection reconnected, refreshing routes", logPrefix); + queryAndResolveRoutes(false, null, null); + } + + @NonNull + @Override + protected EndPoint buildNodeEndPoint( + @NonNull AdminRow row, + @Nullable InetSocketAddress broadcastRpcAddress, + @NonNull EndPoint localEndPoint) { + // If the routes cache is empty (e.g. system.client_routes does not exist on this server + // version), fall back to the default endpoint resolution so that the session can still + // function normally. + if (resolvedRoutesCache.get().isEmpty()) { + return super.buildNodeEndPoint(row, broadcastRpcAddress, localEndPoint); + } + UUID hostId = Objects.requireNonNull(row.getUuid("host_id")); + InetAddress broadcastInetAddress = null; + if (broadcastRpcAddress != null) { + broadcastInetAddress = broadcastRpcAddress.getAddress(); + } + if (broadcastInetAddress == null) { + broadcastInetAddress = row.getInetAddress("broadcast_address"); + } + if (broadcastInetAddress == null) { + broadcastInetAddress = row.getInetAddress("peer"); + } + return new ClientRoutesEndPoint(this, hostId, broadcastInetAddress); + } + + /** + * Builds the CQL query to fetch client routes. + * + *
    + *
  • Both connection IDs and host IDs present → {@code WHERE connection_id IN (...) AND + * host_id IN (...)} — no {@code ALLOW FILTERING} needed (both partition key components + * provided) + *
  • Connection IDs only → {@code WHERE connection_id IN (...) ALLOW FILTERING}; uses event + * connection IDs when present, otherwise falls back to all configured connection IDs + *
  • Neither → full scan with {@code ALLOW FILTERING} (should not occur in practice) + *
+ */ + @NonNull + private String buildQuery( + @NonNull List configuredConnectionIds, + @Nullable List eventConnectionIds, + @Nullable List eventHostIds) { + + // Use event connection IDs when present, otherwise fall back to all configured IDs + List connectionIds = + (eventConnectionIds != null && !eventConnectionIds.isEmpty()) + ? eventConnectionIds + : configuredConnectionIds; + + boolean hasConnectionIds = !connectionIds.isEmpty(); + boolean hasHostIds = eventHostIds != null && !eventHostIds.isEmpty(); + + StringBuilder stmt = + new StringBuilder(String.format(SELECT_ROUTES_COLUMNS, config.getTableName())); + + if (hasConnectionIds) { + stmt.append(" WHERE connection_id IN (").append(String.join(", ", connectionIds)).append(")"); + } + + if (hasHostIds) { + stmt.append(hasConnectionIds ? " AND" : " WHERE"); + stmt.append(" host_id IN (").append(String.join(", ", eventHostIds)).append(")"); + } + + // ALLOW FILTERING is required unless both connection_id and host_id are provided + // (matching gocql: isFullScan = len(hostIDs) == 0 || len(connectionIDs) == 0) + boolean isFullScan = !hasHostIds || !hasConnectionIds; + if (isFullScan) { + stmt.append(" ALLOW FILTERING"); + } + + return stmt.toString(); + } + + /** + * Merges freshly-queried routes into the cache: + * + *
    + *
  • Route unchanged → keep existing entry as-is (no unnecessary churn) + *
  • Route changed (hostname or ports differ) → replace and evict the old hostname from the + * DNS cache so the next {@link #resolve} call performs a fresh lookup + *
  • Route not yet in cache → append + *
+ * + *

The update is applied via a CAS loop. On collision the candidate is merged with the + * concurrent winner before retrying, matching gocql's {@code MergeWithResolved} retry loop. + */ + void mergeRoutes(Map incoming) { + Map current; + Map candidate; + // 10 retries is more than enough (matches gocql's ceiling) + for (int attempt = 0; attempt < 10; attempt++) { + current = resolvedRoutesCache.get(); + candidate = new HashMap<>(current); + + for (Map.Entry entry : incoming.entrySet()) { + UUID hostId = entry.getKey(); + ResolvedClientRoute next = entry.getValue(); + ResolvedClientRoute existing = candidate.get(hostId); + + if (existing == null) { + // New route — append it + candidate.put(hostId, next); + } else if (!existing.getHostname().equals(next.getHostname()) + || !Objects.equals(existing.getNativeTransportPort(), next.getNativeTransportPort()) + || !Objects.equals( + existing.getNativeTransportPortSsl(), next.getNativeTransportPortSsl())) { + // Route changed — evict the old hostname from DNS cache (forcedResolve equivalent) + // so the next resolve() performs a fresh lookup rather than returning a stale address. + if (!existing.getHostname().equals(next.getHostname())) { + dnsResolver.evict(existing.getHostname()); + } + candidate.put(hostId, next); + // else: route identical — keep existing, no DNS eviction needed + } + } + + if (resolvedRoutesCache.compareAndSet(current, candidate)) { + return; + } + // CAS collision: another thread updated the cache concurrently. + // Treat our candidate as the "incoming" side and merge it into whatever won, + // then retry — matches gocql's MergeWithResolved loop. + incoming = candidate; + } + LOG.warn("[{}] Failed to update client routes cache after 10 CAS attempts", logPrefix); + } + + @Override + public void close() { + closed = true; + if (clientRoutesChangeKey != null) { + context.getEventBus().unregister(clientRoutesChangeKey, ClientRoutesChangeEvent.class); + } + if (reconnectKey != null) { + context.getEventBus().unregister(reconnectKey, ControlConnectionReconnectEvent.class); + } + dnsResolver.clearCache(); + LOG.debug("[{}] ClientRoutesTopologyMonitor closed", logPrefix); + } +} diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java index 4236b8349dd..47d649f80e3 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitor.java @@ -429,7 +429,24 @@ protected EndPoint buildNodeEndPoint( broadcastRpcAddress, "broadcastRpcAddress cannot be null for a peer row"); // Deployments that use a custom EndPoint implementation will need their own TopologyMonitor. // One simple approach is to extend this class and override this method. - return new DefaultEndPoint(context.getAddressTranslator().translate(broadcastRpcAddress)); + + // Pass node metadata to translator for client routes support + UUID hostId = row.getUuid("host_id"); + String datacenter = row.getString("data_center"); + String rack = row.getString("rack"); + + // Warn if host_id is missing and client routes are configured, as client routes require it + if (hostId == null && context.getClientRoutesHandler() != null) { + LOG.warn( + "[{}] host_id is null for peer {} but client routes are configured. " + + "This may indicate corrupted system tables or a configuration issue.", + logPrefix, + broadcastRpcAddress); + } + + InetSocketAddress translatedAddress = + context.getAddressTranslator().translate(broadcastRpcAddress, hostId, datacenter, rack); + return new DefaultEndPoint(translatedAddress); } else { // Don't rely on system.local.rpc_address for the control node, because it mistakenly // reports the normal RPC address instead of the broadcast one (CASSANDRA-11181). We diff --git a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java index c9fee86f2c1..edb00dd835b 100644 --- a/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java +++ b/core/src/main/java/com/datastax/oss/driver/internal/core/session/DefaultSession.java @@ -41,6 +41,7 @@ import com.datastax.oss.driver.internal.core.channel.DriverChannel; import com.datastax.oss.driver.internal.core.context.InternalDriverContext; import com.datastax.oss.driver.internal.core.context.LifecycleListener; +import com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor; import com.datastax.oss.driver.internal.core.metadata.DefaultNode; import com.datastax.oss.driver.internal.core.metadata.MetadataManager; import com.datastax.oss.driver.internal.core.metadata.MetadataManager.RefreshSchemaResult; @@ -409,6 +410,7 @@ private void init(CqlIdentifier keyspace) { context .getTopologyMonitor() .init() + .thenCompose(v -> initClientRoutes()) .thenCompose(v -> metadataManager.refreshNodes()) .thenCompose(v -> checkProtocolVersion()) .thenCompose(v -> initialSchemaRefresh()) @@ -433,6 +435,25 @@ private void init(CqlIdentifier keyspace) { }); } + private CompletionStage initClientRoutes() { + ClientRoutesTopologyMonitor handler = context.getClientRoutesHandler(); + if (handler == null) { + return CompletableFuture.completedFuture(null); + } + return handler + .init() + .exceptionally( + e -> { + LOG.warn( + "[{}] Failed to initialize client routes (table may not exist on this server " + + "version — client-routes feature will be inactive): {}", + logPrefix, + e.getMessage(), + e); + return null; + }); + } + private CompletionStage checkProtocolVersion() { try { boolean protocolWasForced = @@ -678,6 +699,15 @@ private void closePolicies() { // Assume the policy had failed to initialize, and we don't need to close it => ignore } } + // Add ClientRoutesHandler if configured + try { + ClientRoutesTopologyMonitor handler = context.getClientRoutesHandler(); + if (handler != null) { + policies.add(handler); + } + } catch (Throwable t) { + // ignore + } try { context.getAuthProvider().ifPresent(policies::add); } catch (Throwable t) { diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index 40d56d67341..57734906e95 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -1110,6 +1110,57 @@ datastax-java-driver { # advertised-hostname = mycustomhostname } + # Client routes configuration for cloud private-endpoint (PrivateLink) deployments. + # + # Client routes enable the driver to discover and connect to nodes through a load balancer such + # as AWS PrivateLink, Azure Private Link, or GCP Private Service Connect. The driver reads + # per-node endpoint mappings from the `system.client_routes` table and translates node addresses + # by Host ID instead of by IP address. + # + # This section is mutually exclusive with a non-default advanced.address-translator. If both are + # set an IllegalStateException is thrown at session startup. + # + # Required: no (client routes are disabled when this section is absent or `endpoints` is unset) + # Modifiable at runtime: no + # Overridable in a profile: no + advanced.client-routes { + + # The list of PrivateLink endpoints to connect through. At least one entry is required when + # client routes are enabled. Each entry is a HOCON object with the following fields: + # + # connection-id (string, required) – opaque string identifying the connection in + # system.client_routes. Must match a `connection_id` row in that table. + # + # connection-addr (string, optional) – Hostname overriding the `address` column from + # system.client_routes, used as the seed contact point for the initial + # connection when no explicit contact points are provided. + # This is a plain hostname — do NOT include a port number; the port is + # read from the system.client_routes table. + # When absent the driver falls back to the address column in that table. + # + # Example: + # endpoints = [ + # { connection-id = "my-connection-1", connection-addr = "cluster.example.com" }, + # { connection-id = "my-connection-2" } + # ] + # + # Required: yes (if client routes are to be used) + // endpoints = [] + + # The name of the system table that stores the per-node endpoint mappings. + # + # Override this only for testing. In production leave it at the default. + # + # Required: yes + table-name = "system.client_routes" + + # How long the driver caches a resolved DNS address for a client-routes endpoint before + # re-resolving it. A shorter duration reduces staleness in dynamic environments (e.g. during + # rolling updates); a longer duration reduces DNS lookup overhead in stable environments. + # + # Required: yes + dns-cache-duration = 500 milliseconds + } # Whether to resolve the addresses passed to `basic.contact-points`. # # If this is true, addresses are created with `InetSocketAddress(String, int)`: the host name will diff --git a/core/src/test/java/com/datastax/oss/driver/api/core/config/ClientRoutesConfigTest.java b/core/src/test/java/com/datastax/oss/driver/api/core/config/ClientRoutesConfigTest.java new file mode 100644 index 00000000000..72cd9cc62c4 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/api/core/config/ClientRoutesConfigTest.java @@ -0,0 +1,164 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.core.config; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import org.junit.Test; + +public class ClientRoutesConfigTest { + + @Test + public void should_build_config_with_single_endpoint() { + String connectionId = "conn-id-1"; + String connectionAddr = "my-privatelink.us-east-1.aws.scylladb.com"; + + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId, connectionAddr)) + .build(); + + assertThat(config.getEndpoints()).hasSize(1); + assertThat(config.getEndpoints().get(0).getConnectionId()).isEqualTo(connectionId); + assertThat(config.getEndpoints().get(0).getConnectionAddr()).isEqualTo(connectionAddr); + assertThat(config.getTableName()).isEqualTo("system.client_routes"); + } + + @Test + public void should_build_config_with_multiple_endpoints() { + String connectionId1 = "conn-id-1"; + String connectionId2 = "conn-id-2"; + + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId1, "host1")) + .addEndpoint(new ClientRoutesEndpoint(connectionId2, "host2")) + .build(); + + assertThat(config.getEndpoints()).hasSize(2); + assertThat(config.getEndpoints().get(0).getConnectionId()).isEqualTo(connectionId1); + assertThat(config.getEndpoints().get(1).getConnectionId()).isEqualTo(connectionId2); + } + + @Test + public void should_build_config_with_custom_table_name() { + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint("conn-id-1")) + .withTableName("custom.client_routes_test") + .build(); + + assertThat(config.getTableName()).isEqualTo("custom.client_routes_test"); + } + + @Test + public void should_fail_when_no_endpoints_provided() { + assertThatThrownBy(() -> ClientRoutesConfig.builder().build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("At least one endpoint must be specified"); + } + + @Test + public void should_create_endpoint_without_connection_address() { + String connectionId = "conn-id-1"; + ClientRoutesEndpoint endpoint = new ClientRoutesEndpoint(connectionId); + + assertThat(endpoint.getConnectionId()).isEqualTo(connectionId); + assertThat(endpoint.getConnectionAddr()).isNull(); + } + + @Test + public void should_create_endpoint_with_connection_address() { + String connectionId = "conn-id-1"; + String connectionAddr = "host"; + ClientRoutesEndpoint endpoint = new ClientRoutesEndpoint(connectionId, connectionAddr); + + assertThat(endpoint.getConnectionId()).isEqualTo(connectionId); + assertThat(endpoint.getConnectionAddr()).isEqualTo(connectionAddr); + } + + @Test + public void should_fail_when_connection_id_is_null() { + assertThatThrownBy(() -> new ClientRoutesEndpoint(null)) + .isInstanceOf(NullPointerException.class) + .hasMessageContaining("connectionId must not be null"); + } + + @Test + public void should_replace_endpoints_with_withEndpoints() { + String connectionId1 = "conn-id-1"; + String connectionId2 = "conn-id-2"; + String connectionId3 = "conn-id-3"; + + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId1)) + .withEndpoints( + java.util.Arrays.asList( + new ClientRoutesEndpoint(connectionId2), + new ClientRoutesEndpoint(connectionId3))) + .build(); + + assertThat(config.getEndpoints()).hasSize(2); + assertThat(config.getEndpoints().get(0).getConnectionId()).isEqualTo(connectionId2); + assertThat(config.getEndpoints().get(1).getConnectionId()).isEqualTo(connectionId3); + } + + @Test + public void should_use_default_dns_cache_duration() { + ClientRoutesConfig config = + ClientRoutesConfig.builder().addEndpoint(new ClientRoutesEndpoint("conn-id-1")).build(); + + assertThat(config.getDnsCacheDurationMillis()).isEqualTo(500L); + } + + @Test + public void should_build_config_with_custom_dns_cache_duration() { + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint("conn-id-1")) + .withDnsCacheDuration(1000L) + .build(); + + assertThat(config.getDnsCacheDurationMillis()).isEqualTo(1000L); + } + + @Test + public void should_allow_zero_dns_cache_duration() { + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint("conn-id-1")) + .withDnsCacheDuration(0L) + .build(); + + assertThat(config.getDnsCacheDurationMillis()).isEqualTo(0L); + } + + @Test + public void should_fail_when_dns_cache_duration_is_negative() { + assertThatThrownBy( + () -> + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint("conn-id-1")) + .withDnsCacheDuration(-1L) + .build()) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("DNS cache duration must be non-negative"); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/api/core/config/TypedDriverOptionTest.java b/core/src/test/java/com/datastax/oss/driver/api/core/config/TypedDriverOptionTest.java index eee4000a459..aff83a59880 100644 --- a/core/src/test/java/com/datastax/oss/driver/api/core/config/TypedDriverOptionTest.java +++ b/core/src/test/java/com/datastax/oss/driver/api/core/config/TypedDriverOptionTest.java @@ -44,7 +44,10 @@ public void should_have_equivalents_for_all_builtin_untyped_options() { ImmutableSet.of( DefaultDriverOption.LOAD_BALANCING_POLICY, DefaultDriverOption.RETRY_POLICY, - DefaultDriverOption.SPECULATIVE_EXECUTION_POLICY); + DefaultDriverOption.SPECULATIVE_EXECUTION_POLICY, + // CLIENT_ROUTES_ENDPOINTS is a HOCON list-of-objects: it cannot be represented as a + // flat TypedDriverOption scalar/list, so it is excluded from this check. + DefaultDriverOption.CLIENT_ROUTES_ENDPOINTS); for (DriverOption option : ImmutableSet.builder() diff --git a/core/src/test/java/com/datastax/oss/driver/api/core/session/ClientRoutesSessionBuilderTest.java b/core/src/test/java/com/datastax/oss/driver/api/core/session/ClientRoutesSessionBuilderTest.java new file mode 100644 index 00000000000..480350ad133 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/api/core/session/ClientRoutesSessionBuilderTest.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.api.core.session; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; +import edu.umd.cs.findbugs.annotations.NonNull; +import java.util.UUID; +import org.junit.Test; + +public class ClientRoutesSessionBuilderTest { + + // --------------------------------------------------------------------------- + // SessionBuilder integration tests + // --------------------------------------------------------------------------- + + @Test + public void should_set_client_routes_config_programmatically() { + String connectionId = UUID.randomUUID().toString(); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId, "host")) + .build(); + + TestSessionBuilder builder = new TestSessionBuilder(); + builder.withClientRoutesConfig(config); + + assertThat(builder.clientRoutesConfig).isEqualTo(config); + assertThat(builder.programmaticArgumentsBuilder.build().getClientRoutesConfig()) + .isEqualTo(config); + } + + @Test + public void should_allow_null_client_routes_config() { + TestSessionBuilder builder = new TestSessionBuilder(); + builder.withClientRoutesConfig(null); + + assertThat(builder.clientRoutesConfig).isNull(); + assertThat(builder.programmaticArgumentsBuilder.build().getClientRoutesConfig()).isNull(); + } + + /** Test subclass to access protected fields. */ + private static class TestSessionBuilder extends SessionBuilder { + @Override + protected CqlSession wrap(@NonNull CqlSession defaultSession) { + return mock(CqlSession.class); + } + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/clientroutes/CachingDnsResolverTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/clientroutes/CachingDnsResolverTest.java new file mode 100644 index 00000000000..94168b54252 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/clientroutes/CachingDnsResolverTest.java @@ -0,0 +1,240 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.clientroutes; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.net.InetAddress; +import java.net.UnknownHostException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.junit.Test; + +public class CachingDnsResolverTest { + + private static final InetAddress ADDR_1; + private static final InetAddress ADDR_2; + + static { + try { + ADDR_1 = InetAddress.getByAddress(new byte[] {127, 0, 0, 1}); + ADDR_2 = InetAddress.getByAddress(new byte[] {127, 0, 0, 2}); + } catch (UnknownHostException e) { + throw new ExceptionInInitializerError(e); + } + } + + @Test + public void should_return_cached_address_within_ttl() throws UnknownHostException { + AtomicInteger callCount = new AtomicInteger(0); + CachingDnsResolver resolver = + new CachingDnsResolver( + 10_000, + hostname -> { + callCount.incrementAndGet(); + return ADDR_1; + }); + + InetAddress first = resolver.resolve("host1"); + InetAddress second = resolver.resolve("host1"); + + assertThat(first).isEqualTo(ADDR_1); + assertThat(second).isEqualTo(ADDR_1); + assertThat(callCount.get()).isEqualTo(1); + } + + @Test + public void should_re_resolve_after_ttl_expires() throws Exception { + AtomicInteger callCount = new AtomicInteger(0); + // 0ms TTL means cache entries are immediately stale + CachingDnsResolver resolver = + new CachingDnsResolver( + 0, + hostname -> { + callCount.incrementAndGet(); + return ADDR_1; + }); + + resolver.resolve("host1"); + Thread.sleep(5); // ensure expiry + resolver.resolve("host1"); + + assertThat(callCount.get()).isEqualTo(2); + } + + @Test + public void should_resolve_different_hostnames_independently() throws UnknownHostException { + CachingDnsResolver resolver = + new CachingDnsResolver(10_000, hostname -> hostname.equals("host1") ? ADDR_1 : ADDR_2); + + assertThat(resolver.resolve("host1")).isEqualTo(ADDR_1); + assertThat(resolver.resolve("host2")).isEqualTo(ADDR_2); + assertThat(resolver.resolve("host1")).isEqualTo(ADDR_1); + } + + @Test + public void should_fall_back_to_last_known_good_on_failure() throws Exception { + AtomicBoolean shouldFail = new AtomicBoolean(false); + CachingDnsResolver resolver = + new CachingDnsResolver( + 0, + hostname -> { + if (shouldFail.get()) { + throw new UnknownHostException("simulated failure"); + } + return ADDR_1; + }); + + // Populate last-known-good + assertThat(resolver.resolve("host1")).isEqualTo(ADDR_1); + + // Expire cache and make resolution fail + Thread.sleep(5); + shouldFail.set(true); + + // Should return last-known-good instead of throwing + assertThat(resolver.resolve("host1")).isEqualTo(ADDR_1); + } + + @Test + public void should_throw_when_no_fallback_available() { + CachingDnsResolver resolver = + new CachingDnsResolver( + 10_000, + hostname -> { + throw new UnknownHostException("always fails"); + }); + + assertThatThrownBy(() -> resolver.resolve("unknown-host")) + .isInstanceOf(UnknownHostException.class); + } + + @Test + public void should_clear_cache_on_clearCache() throws Exception { + AtomicInteger callCount = new AtomicInteger(0); + CachingDnsResolver resolver = + new CachingDnsResolver( + 10_000, + hostname -> { + callCount.incrementAndGet(); + return ADDR_1; + }); + + resolver.resolve("host1"); + assertThat(callCount.get()).isEqualTo(1); + + resolver.clearCache(); + resolver.resolve("host1"); + + assertThat(callCount.get()).isEqualTo(2); + } + + @Test + public void should_retain_last_known_good_after_clearCache() throws Exception { + AtomicBoolean shouldFail = new AtomicBoolean(false); + CachingDnsResolver resolver = + new CachingDnsResolver( + 10_000, + hostname -> { + if (shouldFail.get()) { + throw new UnknownHostException("simulated failure"); + } + return ADDR_1; + }); + + resolver.resolve("host1"); // populate last-known-good + resolver.clearCache(); // clear cache but not last-known-good + shouldFail.set(true); + + // Should still fall back to last-known-good after cache clear + assertThat(resolver.resolve("host1")).isEqualTo(ADDR_1); + } + + @Test + public void should_not_retain_semaphores_after_resolution() throws UnknownHostException { + CachingDnsResolver resolver = new CachingDnsResolver(10_000, hostname -> ADDR_1); + + // Resolve a batch of distinct hostnames to ensure the semaphore map does not accumulate them. + int hostCount = 20; + for (int i = 0; i < hostCount; i++) { + resolver.resolve("host-" + i); + } + + // After all resolutions complete no semaphore entries should be retained. + assertThat(resolver.semaphoreCount()).isZero(); + } + + @Test + public void should_not_retain_semaphores_after_concurrent_resolution() throws Exception { + int threadCount = 10; + CachingDnsResolver resolver = + new CachingDnsResolver( + 10_000, + hostname -> { + // Simulate a short delay to increase contention on the semaphore. + try { + Thread.sleep(1); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + return ADDR_1; + }); + + CountDownLatch start = new CountDownLatch(1); + ExecutorService exec = Executors.newFixedThreadPool(threadCount); + List unexpectedErrors = Collections.synchronizedList(new ArrayList<>()); + try { + for (int i = 0; i < threadCount; i++) { + final int idx = i; + exec.submit( + () -> { + try { + start.await(); + resolver.resolve("concurrent-host-" + idx); + } catch (InterruptedException e) { + // Expected during executor shutdown; restore the interrupt flag. + Thread.currentThread().interrupt(); + } catch (Throwable t) { + // Any other throwable (UnknownHostException, NPE, IllegalStateException, …) + // is unexpected and must not be silently swallowed. + unexpectedErrors.add(t); + } + }); + } + start.countDown(); + } finally { + exec.shutdown(); + assertThat(exec.awaitTermination(5, TimeUnit.SECONDS)).isTrue(); + } + + assertThat(unexpectedErrors) + .as("No unexpected exceptions should occur during concurrent resolution") + .isEmpty(); + + // All concurrent resolutions have finished; no semaphore should remain. + assertThat(resolver.semaphoreCount()).isZero(); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/context/ClientRoutesConfigFromFileTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/context/ClientRoutesConfigFromFileTest.java new file mode 100644 index 00000000000..8db7a574e15 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/context/ClientRoutesConfigFromFileTest.java @@ -0,0 +1,217 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.context; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; +import com.datastax.oss.driver.api.core.config.DriverConfigLoader; +import com.datastax.oss.driver.api.core.session.ProgrammaticArguments; +import com.datastax.oss.driver.internal.core.config.typesafe.DefaultDriverConfigLoader; +import com.typesafe.config.ConfigFactory; +import java.time.Duration; +import java.util.List; +import org.junit.Test; + +/** + * Unit tests for config-file-based client routes parsing in {@link DefaultDriverContext}. + * + *

Each test builds a minimal {@link DefaultDriverContext} from an inline HOCON snippet and + * verifies that {@link DefaultDriverContext#buildClientRoutesConfigFromFile()} correctly parses (or + * rejects) the {@code advanced.client-routes} section. + * + *

Note: {@code connection-addr} is a plain hostname — it must not include a port + * number. The port is read from the {@code system.client_routes} table at runtime. + */ +public class ClientRoutesConfigFromFileTest { + + // --------------------------------------------------------------------------- + // Helper + // --------------------------------------------------------------------------- + + /** + * Builds a {@link DefaultDriverContext} whose configuration is the driver's {@code + * reference.conf} merged on top of the supplied extra HOCON string. + */ + private DefaultDriverContext contextFromHocon(String extraHocon) { + DriverConfigLoader loader = + new DefaultDriverConfigLoader( + () -> { + ConfigFactory.invalidateCaches(); + return ConfigFactory.parseString(extraHocon) + .withFallback( + ConfigFactory.defaultReference() + .getConfig(DefaultDriverConfigLoader.DEFAULT_ROOT_PATH)); + }); + return new DefaultDriverContext(loader, ProgrammaticArguments.builder().build()); + } + + // --------------------------------------------------------------------------- + // Happy-path: endpoint parsing + // --------------------------------------------------------------------------- + + @Test + public void should_parse_single_endpoint_without_addr() { + DefaultDriverContext ctx = + contextFromHocon( + "advanced.client-routes.endpoints = [" + + " { connection-id = \"11111111-1111-1111-1111-111111111111\" }" + + "]"); + + ClientRoutesConfig cfg = ctx.buildClientRoutesConfigFromFile(); + + assertThat(cfg).isNotNull(); + assertThat(cfg.getEndpoints()).hasSize(1); + ClientRoutesEndpoint ep = cfg.getEndpoints().get(0); + assertThat(ep.getConnectionId()).isEqualTo("11111111-1111-1111-1111-111111111111"); + assertThat(ep.getConnectionAddr()).isNull(); + } + + @Test + public void should_parse_single_endpoint_with_addr() { + // connection-addr is a plain hostname — no port + DefaultDriverContext ctx = + contextFromHocon( + "advanced.client-routes.endpoints = [" + + " { connection-id = \"aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa\"," + + " connection-addr = \"cluster.example.com\" }" + + "]"); + + ClientRoutesConfig cfg = ctx.buildClientRoutesConfigFromFile(); + + assertThat(cfg).isNotNull(); + List eps = cfg.getEndpoints(); + assertThat(eps).hasSize(1); + assertThat(eps.get(0).getConnectionId()).isEqualTo("aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"); + assertThat(eps.get(0).getConnectionAddr()).isEqualTo("cluster.example.com"); + } + + @Test + public void should_parse_multiple_endpoints() { + DefaultDriverContext ctx = + contextFromHocon( + "advanced.client-routes.endpoints = [" + + " { connection-id = \"11111111-1111-1111-1111-111111111111\"," + + " connection-addr = \"node1.example.com\" }," + + " { connection-id = \"22222222-2222-2222-2222-222222222222\" }" + + "]"); + + ClientRoutesConfig cfg = ctx.buildClientRoutesConfigFromFile(); + + assertThat(cfg).isNotNull(); + assertThat(cfg.getEndpoints()).hasSize(2); + assertThat(cfg.getEndpoints().get(0).getConnectionId()) + .isEqualTo("11111111-1111-1111-1111-111111111111"); + assertThat(cfg.getEndpoints().get(0).getConnectionAddr()).isEqualTo("node1.example.com"); + assertThat(cfg.getEndpoints().get(1).getConnectionId()) + .isEqualTo("22222222-2222-2222-2222-222222222222"); + assertThat(cfg.getEndpoints().get(1).getConnectionAddr()).isNull(); + } + + // --------------------------------------------------------------------------- + // Happy-path: scalar options + // --------------------------------------------------------------------------- + + @Test + public void should_use_default_table_name() { + DefaultDriverContext ctx = + contextFromHocon( + "advanced.client-routes.endpoints = [" + + " { connection-id = \"11111111-1111-1111-1111-111111111111\" }" + + "]"); + + assertThat(ctx.buildClientRoutesConfigFromFile().getTableName()) + .isEqualTo("system.client_routes"); + } + + @Test + public void should_apply_custom_table_name() { + DefaultDriverContext ctx = + contextFromHocon( + "advanced.client-routes {\n" + + " endpoints = [{ connection-id = \"11111111-1111-1111-1111-111111111111\" }]\n" + + " table-name = \"test.custom_routes\"\n" + + "}"); + + assertThat(ctx.buildClientRoutesConfigFromFile().getTableName()) + .isEqualTo("test.custom_routes"); + } + + @Test + public void should_use_default_dns_cache_duration() { + DefaultDriverContext ctx = + contextFromHocon( + "advanced.client-routes.endpoints = [" + + " { connection-id = \"11111111-1111-1111-1111-111111111111\" }" + + "]"); + + assertThat(ctx.buildClientRoutesConfigFromFile().getDnsCacheDurationMillis()).isEqualTo(500L); + } + + @Test + public void should_apply_custom_dns_cache_duration() { + DefaultDriverContext ctx = + contextFromHocon( + "advanced.client-routes {\n" + + " endpoints = [{ connection-id = \"11111111-1111-1111-1111-111111111111\" }]\n" + + " dns-cache-duration = 2 seconds\n" + + "}"); + + assertThat(ctx.buildClientRoutesConfigFromFile().getDnsCacheDurationMillis()) + .isEqualTo(Duration.ofSeconds(2).toMillis()); + } + + // --------------------------------------------------------------------------- + // Absent / disabled + // --------------------------------------------------------------------------- + + @Test + public void should_return_null_when_endpoints_not_configured() { + // No endpoints key — reference.conf comments it out, so client routes are disabled + DefaultDriverContext ctx = contextFromHocon(""); + + assertThat(ctx.buildClientRoutesConfigFromFile()).isNull(); + } + + // --------------------------------------------------------------------------- + // Error cases + // --------------------------------------------------------------------------- + + @Test + public void should_throw_when_endpoints_list_is_empty() { + DefaultDriverContext ctx = contextFromHocon("advanced.client-routes.endpoints = []"); + + assertThatThrownBy(ctx::buildClientRoutesConfigFromFile) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("no entries"); + } + + @Test + public void should_throw_when_endpoint_missing_connection_id() { + // connection-addr without connection-id should fail validation + DefaultDriverContext ctx = + contextFromHocon( + "advanced.client-routes.endpoints = [{ connection-addr = \"host.example.com\" }]"); + + assertThatThrownBy(ctx::buildClientRoutesConfigFromFile) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("connection-id"); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java new file mode 100644 index 00000000000..d149285ac68 --- /dev/null +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/ClientRoutesTopologyMonitorTest.java @@ -0,0 +1,386 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.datastax.oss.driver.internal.core.metadata; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.Mockito.when; + +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.config.DriverConfig; +import com.datastax.oss.driver.api.core.config.DriverExecutionProfile; +import com.datastax.oss.driver.internal.core.clientroutes.DnsResolver; +import com.datastax.oss.driver.internal.core.clientroutes.ResolvedClientRoute; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.control.ControlConnection; +import com.datastax.oss.driver.shaded.guava.common.collect.ImmutableMap; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.UUID; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +@RunWith(MockitoJUnitRunner.class) +public class ClientRoutesTopologyMonitorTest { + + @Mock private InternalDriverContext context; + @Mock private ControlConnection controlConnection; + @Mock private DriverConfig driverConfig; + @Mock private DriverExecutionProfile defaultProfile; + + private TestableClientRoutesTopologyMonitor handler; + + /** + * Subclass exposing package-private {@code resolvedRoutesCache} so tests can inject test data + * without actually executing admin queries. Optionally accepts an injected {@link DnsResolver} to + * allow verification of cache-eviction behavior. + */ + @SuppressWarnings("NewClassNamingConvention") + static class TestableClientRoutesTopologyMonitor extends ClientRoutesTopologyMonitor { + TestableClientRoutesTopologyMonitor(InternalDriverContext ctx, ClientRoutesConfig cfg) { + super(ctx, cfg); + } + + TestableClientRoutesTopologyMonitor( + InternalDriverContext ctx, ClientRoutesConfig cfg, DnsResolver dnsResolver) { + super(ctx, cfg, dnsResolver); + } + + void setRoutes(Map routes) { + resolvedRoutesCache.set(new HashMap<>(routes)); + } + + Map getRoutes() { + return resolvedRoutesCache.get(); + } + + void mergeRoutesForTest(Map incoming) { + mergeRoutes(incoming); + } + } + + @Before + public void setup() { + when(context.getSessionName()).thenReturn("test-session"); + when(context.getControlConnection()).thenReturn(controlConnection); + when(context.getConfig()).thenReturn(driverConfig); + when(driverConfig.getDefaultProfile()).thenReturn(defaultProfile); + when(defaultProfile.getDuration(DefaultDriverOption.CONTROL_CONNECTION_TIMEOUT)) + .thenReturn(Duration.ofSeconds(5)); + when(defaultProfile.getBoolean(DefaultDriverOption.RECONNECT_ON_INIT)).thenReturn(false); + when(context.getSslEngineFactory()).thenReturn(Optional.empty()); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(UUID.randomUUID().toString(), "host1")) + .build(); + handler = new TestableClientRoutesTopologyMonitor(context, config); + } + + // ---- resolve() ------------------------------------------------------- + + @Test + public void should_throw_for_unknown_host_id() { + assertThatThrownBy(() -> handler.resolve(UUID.randomUUID())) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("No client route found"); + } + + @Test + public void should_resolve_known_host_id_non_ssl() throws UnknownHostException { + UUID hostId = UUID.randomUUID(); + handler.setRoutes( + ImmutableMap.of(hostId, new ResolvedClientRoute(hostId, "127.0.0.1", 9042, 9142))); + + InetSocketAddress result = handler.resolve(hostId); + + assertThat(result).isNotNull(); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_select_tls_port_when_ssl() throws UnknownHostException { + // Recreate handler with SSL enabled + when(context.getSslEngineFactory()) + .thenReturn( + Optional.of(Mockito.mock(com.datastax.oss.driver.api.core.ssl.SslEngineFactory.class))); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(UUID.randomUUID().toString(), "host1")) + .build(); + handler = new TestableClientRoutesTopologyMonitor(context, config); + + UUID hostId = UUID.randomUUID(); + handler.setRoutes( + ImmutableMap.of(hostId, new ResolvedClientRoute(hostId, "127.0.0.1", 9042, 9142))); + + InetSocketAddress result = handler.resolve(hostId); + + assertThat(result).isNotNull(); + assertThat(result.getPort()).isEqualTo(9142); + } + + @Test + public void should_fall_back_to_non_ssl_port_when_tls_port_absent() throws UnknownHostException { + // Recreate handler with SSL enabled + when(context.getSslEngineFactory()) + .thenReturn( + Optional.of(Mockito.mock(com.datastax.oss.driver.api.core.ssl.SslEngineFactory.class))); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(UUID.randomUUID().toString(), "host1")) + .build(); + handler = new TestableClientRoutesTopologyMonitor(context, config); + + UUID hostId = UUID.randomUUID(); + // tls_port is null — should warn and fall back to non-SSL port + handler.setRoutes( + ImmutableMap.of(hostId, new ResolvedClientRoute(hostId, "127.0.0.1", 9042, null))); + + InetSocketAddress result = handler.resolve(hostId); + + assertThat(result).isNotNull(); + assertThat(result.getPort()).isEqualTo(9042); + } + + @Test + public void should_throw_when_no_port_configured() { + UUID hostId = UUID.randomUUID(); + // Both ports null → IllegalStateException + handler.setRoutes( + ImmutableMap.of(hostId, new ResolvedClientRoute(hostId, "127.0.0.1", null, null))); + + assertThatThrownBy(() -> handler.resolve(hostId)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("No port configured"); + } + + @Test + public void should_throw_after_close() { + UUID hostId = UUID.randomUUID(); + handler.setRoutes( + ImmutableMap.of(hostId, new ResolvedClientRoute(hostId, "127.0.0.1", 9042, null))); + + handler.close(); + + assertThatThrownBy(() -> handler.resolve(hostId)) + .isInstanceOf(IllegalStateException.class) + .hasMessageContaining("closed"); + } + + @Test + public void should_throw_for_unresolvable_hostname() { + UUID hostId = UUID.randomUUID(); + // Use a hostname guaranteed not to resolve + handler.setRoutes( + ImmutableMap.of( + hostId, + new ResolvedClientRoute(hostId, "this.host.does.not.exist.invalid", 9042, null))); + + assertThatThrownBy(() -> handler.resolve(hostId)).isInstanceOf(UnknownHostException.class); + } + + @Test + public void should_refresh_updates_routes() throws UnknownHostException { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + + handler.setRoutes( + ImmutableMap.of(hostId1, new ResolvedClientRoute(hostId1, "127.0.0.1", 9042, null))); + assertThat(handler.resolve(hostId1)).isNotNull(); + assertThatThrownBy(() -> handler.resolve(hostId2)).isInstanceOf(IllegalStateException.class); + + // Simulate a refresh that swaps in a different set of routes + handler.setRoutes( + ImmutableMap.of(hostId2, new ResolvedClientRoute(hostId2, "127.0.0.2", 9042, null))); + + assertThatThrownBy(() -> handler.resolve(hostId1)).isInstanceOf(IllegalStateException.class); + assertThat(handler.resolve(hostId2)).isNotNull(); + } + + // ---- Merge behavior tests ----------------------------------------------- + + @Test + public void should_preserve_existing_routes_on_merge() throws UnknownHostException { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + UUID hostId3 = UUID.randomUUID(); + + // Initial routes: hostId1 and hostId2 + handler.setRoutes( + ImmutableMap.of( + hostId1, new ResolvedClientRoute(hostId1, "127.0.0.1", 9042, null), + hostId2, new ResolvedClientRoute(hostId2, "127.0.0.2", 9042, null))); + + // Verify initial state + assertThat(handler.resolve(hostId1).getPort()).isEqualTo(9042); + assertThat(handler.resolve(hostId2).getPort()).isEqualTo(9042); + assertThatThrownBy(() -> handler.resolve(hostId3)).isInstanceOf(IllegalStateException.class); + + // Simulate a targeted merge that adds hostId3 + Map current = handler.getRoutes(); + Map merged = new HashMap<>(current); + merged.put(hostId3, new ResolvedClientRoute(hostId3, "127.0.0.3", 9043, null)); + handler.setRoutes(merged); + + // All three hosts should now be resolvable + assertThat(handler.resolve(hostId1).getPort()).isEqualTo(9042); + assertThat(handler.resolve(hostId2).getPort()).isEqualTo(9042); + assertThat(handler.resolve(hostId3).getPort()).isEqualTo(9043); + } + + @Test + public void should_update_existing_route_on_merge() throws UnknownHostException { + UUID hostId1 = UUID.randomUUID(); + UUID hostId2 = UUID.randomUUID(); + + // Initial routes + handler.setRoutes( + ImmutableMap.of( + hostId1, new ResolvedClientRoute(hostId1, "127.0.0.1", 9042, null), + hostId2, new ResolvedClientRoute(hostId2, "127.0.0.2", 9042, null))); + + // Verify initial port + assertThat(handler.resolve(hostId1).getPort()).isEqualTo(9042); + assertThat(handler.resolve(hostId2).getPort()).isEqualTo(9042); + + // Simulate a targeted update that changes hostId1's port + Map current = handler.getRoutes(); + Map merged = new HashMap<>(current); + merged.put(hostId1, new ResolvedClientRoute(hostId1, "127.0.0.1", 9999, null)); + handler.setRoutes(merged); + + // hostId1 should have new port, hostId2 should be unchanged + assertThat(handler.resolve(hostId1).getPort()).isEqualTo(9999); + assertThat(handler.resolve(hostId2).getPort()).isEqualTo(9042); + } + + @Test + public void should_return_configured_connection_ids() { + // The handler was created with one endpoint in setup() + ClientRoutesConfig cfg = handler.getClientRoutesConfig(); + assertThat(cfg.getEndpoints()).hasSize(1); + assertThat(cfg.getEndpoints().get(0).getConnectionId()).isNotNull(); + } + + // ---- mergeRoutes DNS eviction tests ------------------------------------- + + @Test + public void should_evict_dns_cache_when_hostname_changes() { + DnsResolver mockDns = Mockito.mock(DnsResolver.class); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(UUID.randomUUID().toString(), "host1")) + .build(); + TestableClientRoutesTopologyMonitor h = + new TestableClientRoutesTopologyMonitor(context, config, mockDns); + + UUID hostId = UUID.randomUUID(); + h.setRoutes( + ImmutableMap.of( + hostId, new ResolvedClientRoute(hostId, "old-host.example.com", 9042, null))); + + // Merge a route with a changed hostname + Map incoming = new HashMap<>(); + incoming.put(hostId, new ResolvedClientRoute(hostId, "new-host.example.com", 9042, null)); + h.mergeRoutesForTest(incoming); + + // Old hostname must be evicted from DNS cache (forcedResolve equivalent) + Mockito.verify(mockDns).evict("old-host.example.com"); + assertThat(h.getRoutes().get(hostId).getHostname()).isEqualTo("new-host.example.com"); + } + + @Test + public void should_not_evict_dns_cache_when_hostname_unchanged() { + DnsResolver mockDns = Mockito.mock(DnsResolver.class); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(UUID.randomUUID().toString(), "host1")) + .build(); + TestableClientRoutesTopologyMonitor h = + new TestableClientRoutesTopologyMonitor(context, config, mockDns); + + UUID hostId = UUID.randomUUID(); + h.setRoutes( + ImmutableMap.of( + hostId, new ResolvedClientRoute(hostId, "same-host.example.com", 9042, null))); + + // Merge with same hostname but different port + Map incoming = new HashMap<>(); + incoming.put(hostId, new ResolvedClientRoute(hostId, "same-host.example.com", 9999, null)); + h.mergeRoutesForTest(incoming); + + // Hostname unchanged — no eviction, but route must be updated + Mockito.verifyZeroInteractions(mockDns); + assertThat(h.getRoutes().get(hostId).getNativeTransportPort()).isEqualTo(9999); + } + + @Test + public void should_not_evict_dns_cache_for_unchanged_route() { + DnsResolver mockDns = Mockito.mock(DnsResolver.class); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(UUID.randomUUID().toString(), "host1")) + .build(); + TestableClientRoutesTopologyMonitor h = + new TestableClientRoutesTopologyMonitor(context, config, mockDns); + + UUID hostId = UUID.randomUUID(); + h.setRoutes( + ImmutableMap.of( + hostId, new ResolvedClientRoute(hostId, "same-host.example.com", 9042, null))); + + // Merge with identical data + Map incoming = new HashMap<>(); + incoming.put(hostId, new ResolvedClientRoute(hostId, "same-host.example.com", 9042, null)); + h.mergeRoutesForTest(incoming); + + // No change — no eviction + Mockito.verifyZeroInteractions(mockDns); + } + + @Test + public void should_not_evict_dns_cache_for_new_route() { + DnsResolver mockDns = Mockito.mock(DnsResolver.class); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(UUID.randomUUID().toString(), "host1")) + .build(); + TestableClientRoutesTopologyMonitor h = + new TestableClientRoutesTopologyMonitor(context, config, mockDns); + + // Cache starts empty + UUID hostId = UUID.randomUUID(); + Map incoming = new HashMap<>(); + incoming.put(hostId, new ResolvedClientRoute(hostId, "brand-new.example.com", 9042, null)); + h.mergeRoutesForTest(incoming); + + // New route — nothing to evict + Mockito.verifyZeroInteractions(mockDns); + assertThat(h.getRoutes()).containsKey(hostId); + } +} diff --git a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java index 354bae82068..22fe64db158 100644 --- a/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java +++ b/core/src/test/java/com/datastax/oss/driver/internal/core/metadata/DefaultTopologyMonitorTest.java @@ -232,11 +232,16 @@ public void should_refresh_node_from_peers_if_broadcast_address_is_not_present() }); // The rpc_address in each row should have been tried, only the last row should have been // converted + // Note: getUuid("host_id") is called once in findInPeers for comparison verify(peer3).getUuid("host_id"); verify(peer3, never()).getString(anyString()); - verify(peer2, times(2)).getUuid("host_id"); - verify(peer2).getString("data_center"); + // Note: getUuid("host_id") called twice (once in findInPeers, once in buildNodeEndPoint) + // getString("data_center") called twice (once in nodeInfoBuilder, once in buildNodeEndPoint) + // getString("rack") called twice (once in nodeInfoBuilder, once in buildNodeEndPoint) + verify(peer2, times(3)).getUuid("host_id"); + verify(peer2, times(2)).getString("data_center"); + verify(peer2, times(2)).getString("rack"); } @Test @@ -265,8 +270,13 @@ public void should_refresh_node_from_peers_if_broadcast_address_is_not_present_V verify(peer3).getUuid("host_id"); verify(peer3, never()).getString(anyString()); - verify(peer2, times(2)).getUuid("host_id"); - verify(peer2).getString("data_center"); + // Note: getUuid("host_id") called three times total (once in findInPeers, once in + // nodeInfoBuilder, once in buildNodeEndPoint) + // getString("data_center") called twice (once in nodeInfoBuilder, once in buildNodeEndPoint) + // getString("rack") called twice (once in nodeInfoBuilder, once in buildNodeEndPoint) + verify(peer2, times(3)).getUuid("host_id"); + verify(peer2, times(2)).getString("data_center"); + verify(peer2, times(2)).getString("rack"); } @Test @@ -298,8 +308,12 @@ public void should_get_new_node_from_peers() { verify(peer2).getInetAddress("rpc_address"); verify(peer2, never()).getString(anyString()); + // Note: getString calls increased due to buildNodeEndPoint calling getString("data_center"), + // getString("rack") + // in addition to nodeInfoBuilder calling them verify(peer1).getInetAddress("rpc_address"); - verify(peer1).getString("data_center"); + verify(peer1, times(2)).getString("data_center"); + verify(peer1, times(2)).getString("rack"); } @Test @@ -331,8 +345,12 @@ public void should_get_new_node_from_peers_v2() { verify(peer2).getInetAddress("native_address"); verify(peer2, never()).getString(anyString()); + // Note: getString calls increased due to buildNodeEndPoint calling getString("data_center"), + // getString("rack") + // in addition to nodeInfoBuilder calling them verify(peer1).getInetAddress("native_address"); - verify(peer1).getString("data_center"); + verify(peer1, times(2)).getString("data_center"); + verify(peer1, times(2)).getString("rack"); } @Test diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java new file mode 100644 index 00000000000..dc94e245a49 --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT.java @@ -0,0 +1,424 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2025 ScyllaDB + * + * Modified by ScyllaDB + */ +package com.datastax.oss.driver.core.clientroutes; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.awaitility.Awaitility.await; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.testinfra.ScyllaOnly; +import com.datastax.oss.driver.api.testinfra.ScyllaRequirement; +import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.session.SessionRule; +import com.datastax.oss.driver.api.testinfra.session.SessionUtils; +import com.datastax.oss.driver.categories.IsolatedTests; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor; +import java.net.InetSocketAddress; +import java.net.UnknownHostException; +import java.time.Duration; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.junit.AssumptionViolatedException; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Integration tests for the PrivateLink client routes feature. + * + *

These tests require a ScyllaDB instance that exposes the {@code system.client_routes} table + * (introduced in scylladb/scylladb#27323). Each test opens its own CQL sessions to populate the + * test table and configure {@link ClientRoutesConfig} precisely. + * + *

The positive tests use a separate keyspace-level table (not the actual {@code + * system.client_routes}) where write access is available, configured via {@link + * ClientRoutesConfig.Builder#withTableName}. This allows testing the full query and translation + * pipeline without requiring privileged write access to the {@code system} keyspace. + * + *

A dedicated negative test ({@link + * ClientRoutesUnsupportedVersionIT#should_not_activate_on_unsupported_version()}) is annotated to + * run only on versions that do not support the feature and verifies that the handler + * remains empty when {@code system.client_routes} does not exist. + */ +@Category(IsolatedTests.class) +@ScyllaOnly(description = "system.client_routes is a ScyllaDB-only system table") +@ScyllaRequirement( + minEnterprise = "2026.1", + description = + "system.client_routes requires ScyllaDB Enterprise >= 2026.1 (scylladb/scylladb#27323)") +public class ClientRoutesIT { + + private static final Logger LOG = LoggerFactory.getLogger(ClientRoutesIT.class); + + /** + * Name of a test-keyspace table that mirrors the {@code system.client_routes} schema. Tests + * INSERT into this table via a plain session, then point {@link ClientRoutesConfig} at it. + */ + private static final String TEST_TABLE = "test_client_routes.client_routes"; + + private static final String CREATE_KEYSPACE = + "CREATE KEYSPACE IF NOT EXISTS test_client_routes" + + " WITH replication = {'class':'SimpleStrategy','replication_factor':1}"; + + /** + * DDL matching the server-side {@code system.client_routes} schema. Using a user keyspace lets + * the test insert rows freely without requiring internal permissions. + */ + private static final String CREATE_TABLE = + "CREATE TABLE IF NOT EXISTS " + + TEST_TABLE + + " (" + + " connection_id uuid," + + " host_id uuid," + + " address text," + + " port int," + + " tls_port int," + + " PRIMARY KEY (connection_id, host_id)" + + ")"; + + public static final CustomCcmRule CCM_RULE = CustomCcmRule.builder().withNodes(1).build(); + + private static final SessionRule SESSION_RULE = + SessionRule.builder(CCM_RULE).withKeyspace(false).build(); + + @ClassRule + public static final TestRule CHAIN = RuleChain.outerRule(CCM_RULE).around(SESSION_RULE); + + // ---- helpers ----------------------------------------------------------- + + /** Returns the IP address of CCM node 1 (e.g. {@code "127.0.2.1"}). */ + private String nodeAddress() { + return CCM_RULE.getCcmBridge().getNodeIpAddress(1); + } + + /** Opens a plain (no client-routes) session against CCM to set up the test table and rows. */ + private CqlSession openAdminSession() { + return (CqlSession) + SessionUtils.baseBuilder() + .addContactEndPoints(CCM_RULE.getContactPoints()) + .withConfigLoader( + SessionUtils.configLoaderBuilder() + .withDuration(DefaultDriverOption.REQUEST_TIMEOUT, Duration.ofSeconds(10)) + .build()) + .build(); + } + + /** Ensures the test keyspace and table exist and are empty. */ + private void prepareTable(CqlSession admin) { + admin.execute(CREATE_KEYSPACE); + admin.execute(CREATE_TABLE); + admin.execute("TRUNCATE TABLE " + TEST_TABLE); + } + + /** Inserts one row into the test client_routes table. */ + private void insertRoute( + CqlSession admin, String connectionId, UUID hostId, String address, int port) { + admin.execute( + "INSERT INTO " + + TEST_TABLE + + " (connection_id, host_id, address, port)" + + " VALUES (" + + connectionId + + ", " + + hostId + + ", '" + + address + + "', " + + port + + ")"); + } + + /** Builds a {@link CqlSession} with the given {@link ClientRoutesConfig}. */ + private CqlSession openClientRoutesSession(ClientRoutesConfig config) { + return (CqlSession) + SessionUtils.baseBuilder() + .addContactEndPoints(CCM_RULE.getContactPoints()) + .withClientRoutesConfig(config) + .withConfigLoader( + SessionUtils.configLoaderBuilder() + .withDuration(DefaultDriverOption.REQUEST_TIMEOUT, Duration.ofSeconds(10)) + .build()) + .build(); + } + + /** Returns the {@link ClientRoutesTopologyMonitor} from an open session's context. */ + private ClientRoutesTopologyMonitor handlerOf(CqlSession session) { + ClientRoutesTopologyMonitor handler = + ((InternalDriverContext) session.getContext()).getClientRoutesHandler(); + assertThat(handler) + .as("ClientRoutesHandler must be non-null when ClientRoutesConfig is set") + .isNotNull(); + return handler; + } + + /** + * Guards a positive test: confirms that {@code system.client_routes} really exists on the running + * server. If it doesn't, the test is skipped via an {@link AssumptionViolatedException} rather + * than failing, which provides a clear signal that the feature is not yet available. + * + *

This is a defence-in-depth check on top of the class-level {@link ScyllaRequirement} — it + * catches cases where the version annotation is wrong or the binary predates the feature despite + * its reported version. + */ + private void requireSystemClientRoutesTable(CqlSession admin) { + Row row = + admin + .execute( + "SELECT table_name FROM system_schema.tables" + + " WHERE keyspace_name='system' AND table_name='client_routes'") + .one(); + if (row == null) { + throw new AssumptionViolatedException( + "system.client_routes does not exist on this server — " + + "feature requires ScyllaDB Enterprise >= 2026.1 (not yet available on OSS). " + + "Skipping positive client-routes test."); + } + } + + /** + * Tries to resolve a host ID and returns the result, or null if no route is found (i.e. + * IllegalStateException is thrown). + */ + private InetSocketAddress tryResolve(ClientRoutesTopologyMonitor handler, UUID hostId) { + try { + return handler.resolve(hostId); + } catch (IllegalStateException e) { + return null; + } catch (UnknownHostException e) { + throw new RuntimeException("DNS resolution failed for host_id=" + hostId, e); + } + } + + // ---- tests ------------------------------------------------------------- + + /** + * Verifies that {@link ClientRoutesTopologyMonitor#init()} reads rows from the configured table + * and populates the internal route map on session startup. + */ + @Test + public void should_load_routes_from_table_on_init() throws UnknownHostException { + String connectionId = UUID.randomUUID().toString(); + String nodeAddr = nodeAddress(); + + try (CqlSession admin = openAdminSession()) { + requireSystemClientRoutesTable(admin); + prepareTable(admin); + + // Discover the node host_id from system.local + Row localRow = admin.execute("SELECT host_id FROM system.local WHERE key='local'").one(); + assertThat(localRow).isNotNull(); + UUID hostId = localRow.getUuid("host_id"); + assertThat(hostId).isNotNull(); + + insertRoute(admin, connectionId, hostId, nodeAddr, 9042); + + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId, nodeAddr + ":9042")) + .withTableName(TEST_TABLE) + .build(); + + try (CqlSession session = openClientRoutesSession(config)) { + // Session must be usable + ResultSet rs = + session.execute("SELECT release_version FROM system.local WHERE key='local'"); + assertThat(rs.one()).isNotNull(); + + // Handler must have loaded the route for the known host_id + InetSocketAddress translated = handlerOf(session).resolve(hostId); + assertThat(translated) + .as("Handler should have a translated address for host_id=%s", hostId) + .isNotNull(); + assertThat(translated.getPort()).isEqualTo(9042); + + LOG.info("Translated host_id={} -> {}", hostId, translated); + } + } + } + + /** + * Verifies that calling {@link ClientRoutesTopologyMonitor#refresh()} after inserting new rows + * picks up the new routes without restarting the session. + */ + @Test + public void should_refresh_routes_after_table_update() throws Exception { + String connectionId = UUID.randomUUID().toString(); + UUID hostId = UUID.randomUUID(); // synthetic — tests the query/map pipeline + String nodeAddr = nodeAddress(); + + try (CqlSession admin = openAdminSession()) { + requireSystemClientRoutesTable(admin); + prepareTable(admin); // empty initially + + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId, nodeAddr + ":9042")) + .withTableName(TEST_TABLE) + .build(); + + try (CqlSession session = openClientRoutesSession(config)) { + ClientRoutesTopologyMonitor handler = handlerOf(session); + + // No route yet + assertThatThrownBy(() -> handler.resolve(hostId)).isInstanceOf(IllegalStateException.class); + + // Insert a new route and explicitly trigger a refresh + insertRoute(admin, connectionId, hostId, nodeAddr, 9042); + handler.refresh().toCompletableFuture().get(10, TimeUnit.SECONDS); + + InetSocketAddress translated = handler.resolve(hostId); + assertThat(translated).isNotNull(); + assertThat(translated.getPort()).isEqualTo(9042); + } + } + } + + /** + * Verifies that a session with client routes configured against an empty table still starts + * successfully and remains usable (nodes accessible via untranslated addresses). + */ + @Test + public void should_start_session_when_table_is_empty() { + String nodeAddr = nodeAddress(); + String connectionId = UUID.randomUUID().toString(); + + try (CqlSession admin = openAdminSession()) { + requireSystemClientRoutesTable(admin); + prepareTable(admin); // intentionally empty + + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId, nodeAddr + ":9042")) + .withTableName(TEST_TABLE) + .build(); + + try (CqlSession session = openClientRoutesSession(config)) { + ResultSet rs = + session.execute("SELECT release_version FROM system.local WHERE key='local'"); + assertThat(rs.one()).isNotNull(); + + // No routes loaded — resolve throws for any host_id + assertThatThrownBy(() -> handlerOf(session).resolve(UUID.randomUUID())) + .isInstanceOf(IllegalStateException.class); + } + } + } + + /** + * Verifies that a route row with a {@code tls_port} column is correctly stored and the non-SSL + * port is used when SSL is not configured. + */ + @Test + public void should_select_non_ssl_port_when_ssl_not_configured() throws UnknownHostException { + String connectionId = UUID.randomUUID().toString(); + UUID hostId = UUID.randomUUID(); + String nodeAddr = nodeAddress(); + + try (CqlSession admin = openAdminSession()) { + requireSystemClientRoutesTable(admin); + prepareTable(admin); + + admin.execute( + "INSERT INTO " + + TEST_TABLE + + " (connection_id, host_id, address, port, tls_port)" + + " VALUES (" + + connectionId + + ", " + + hostId + + ", '" + + nodeAddr + + "', 9042, 9142)"); + + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId, nodeAddr + ":9042")) + .withTableName(TEST_TABLE) + .build(); + + try (CqlSession session = openClientRoutesSession(config)) { + ClientRoutesTopologyMonitor handler = handlerOf(session); + + InetSocketAddress resolved = handler.resolve(hostId); + assertThat(resolved).isNotNull(); + // Non-SSL session should use the regular port + assertThat(resolved.getPort()).isEqualTo(9042); + } + } + } + + /** + * Verifies that after a control connection force-reconnect, {@link + * ClientRoutesTopologyMonitor#refresh()} is called automatically and picks up new rows inserted + * while the session was running. + */ + @Test + public void should_refresh_routes_after_control_connection_reconnect() { + String connectionId = UUID.randomUUID().toString(); + UUID hostId = UUID.randomUUID(); + String nodeAddr = nodeAddress(); + + try (CqlSession admin = openAdminSession()) { + requireSystemClientRoutesTable(admin); + prepareTable(admin); // no rows initially + + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId, nodeAddr + ":9042")) + .withTableName(TEST_TABLE) + .build(); + + try (CqlSession session = openClientRoutesSession(config)) { + ClientRoutesTopologyMonitor handler = handlerOf(session); + assertThat(tryResolve(handler, hostId)).isNull(); + + // Insert a new route while the session is running + insertRoute(admin, connectionId, hostId, nodeAddr, 9042); + + // Force control connection reconnect — onSuccessfulReconnect triggers handler.refresh() + ((InternalDriverContext) session.getContext()).getControlConnection().reconnectNow(); + + // Routes are refreshed asynchronously; wait for the translation to become available + await().atMost(15, TimeUnit.SECONDS).until(() -> tryResolve(handler, hostId) != null); + + assertThat(tryResolve(handler, hostId)) + .as("Route should be available after reconnect and refresh for host_id=%s", hostId) + .isNotNull() + .extracting(InetSocketAddress::getPort) + .isEqualTo(9042); + } + } + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT2.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT2.java new file mode 100644 index 00000000000..f56325cc829 --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesIT2.java @@ -0,0 +1,363 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2025 ScyllaDB + * + * Modified by ScyllaDB + */ +package com.datastax.oss.driver.core.clientroutes; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Awaitility.await; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.core.metadata.Node; +import com.datastax.oss.driver.api.testinfra.ScyllaOnly; +import com.datastax.oss.driver.api.testinfra.ScyllaRequirement; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; +import com.datastax.oss.driver.api.testinfra.session.SessionUtils; +import com.datastax.oss.driver.categories.IsolatedTests; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.InetSocketAddress; +import java.net.URL; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.TimeUnit; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * End-to-end integration test for the PrivateLink client routes feature with a full NLB simulation. + * + *

This test: + * + *

    + *
  1. Creates a 2-node CCM cluster + *
  2. Sets up an NLB simulator (TCP proxy) with a discovery port and per-node ports + *
  3. Posts client routes via the ScyllaDB REST API + *
  4. Creates a CQL session through the NLB (not directly to nodes) + *
  5. Verifies queries work + *
  6. Adds 2 new nodes, reconfigures the NLB and routes + *
  7. Decommissions the original 2 nodes, reconfigures the NLB and routes + *
  8. Verifies the session still works (all connections via proxy, including control connection) + *
+ */ +@Category(IsolatedTests.class) +@ScyllaOnly(description = "system.client_routes is a ScyllaDB-only feature") +@ScyllaRequirement( + minEnterprise = "2026.1", + description = "system.client_routes requires ScyllaDB Enterprise >= 2026.1") +public class ClientRoutesIT2 { + + private static final Logger LOG = LoggerFactory.getLogger(ClientRoutesIT2.class); + + private static final int NLB_BASE_PORT = 29042; + private static final String CONNECTION_ID = "11111111-1111-1111-1111-111111111111"; + private static final String DC_NAME = "dc1"; + + @Test + public void should_survive_full_node_replacement_through_nlb() throws Exception { + try (CcmBridge ccm = CcmBridge.builder().withNodes(2).build()) { + ccm.create(); + ccm.start(); + + NlbSimulator nlb = new NlbSimulator(ccm, NLB_BASE_PORT); + try { + // ---- Stage 1: Set up NLB for initial 2 nodes ---- + LOG.info("Stage 1: Setting up NLB for initial 2 nodes"); + nlb.addNode(1); + nlb.addNode(2); + + // Collect host IDs + Map hostIds = collectHostIds(ccm, 2); + LOG.info("Host IDs: {}", hostIds); + + // Post client routes via REST API + postClientRoutes(ccm, hostIds, nlb); + + // ---- Stage 2: Create session through NLB ---- + LOG.info("Stage 2: Creating session through NLB"); + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(CONNECTION_ID, "127.0.0.1:" + NLB_BASE_PORT)) + .build(); + + try (CqlSession session = openClientRoutesSession(config, nlb)) { + // Verify session works + assertQueryWorks(session); + + // Verify the handler is active + ClientRoutesTopologyMonitor handler = + ((InternalDriverContext) session.getContext()).getClientRoutesHandler(); + assertThat(handler).isNotNull(); + + // Verify all connections go through proxy (check endpoints) + verifyConnectionsThroughProxy(session); + + LOG.info("Stage 2: Session created and verified through NLB"); + + // ---- Stage 3: Add 2 new nodes ---- + LOG.info("Stage 3: Adding nodes 3 and 4"); + ccm.add(3, DC_NAME); + ccm.add(4, DC_NAME); + + // Wait for new nodes to appear in metadata + await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until(() -> session.getMetadata().getNodes().size() >= 4); + + // Add new nodes to NLB + nlb.addNode(3); + nlb.addNode(4); + + // Collect all host IDs (including new nodes) + Map allHostIds = collectHostIds(ccm, 4); + LOG.info("All host IDs after expansion: {}", allHostIds); + + // Update client routes with all nodes + postClientRoutes(ccm, allHostIds, nlb); + + // Refresh routes in the driver + handler.refresh().toCompletableFuture().get(10, TimeUnit.SECONDS); + + // Verify queries still work + assertQueryWorks(session); + LOG.info("Stage 3: Nodes 3 and 4 added, routes updated"); + + // ---- Stage 4: Decommission original nodes ---- + LOG.info("Stage 4: Decommissioning original nodes 1 and 2"); + + // Decommission node 1 + ccm.decommission(1); + nlb.removeNode(1); + allHostIds.remove(1); + postClientRoutes(ccm, allHostIds, nlb); + handler.refresh().toCompletableFuture().get(10, TimeUnit.SECONDS); + + // Wait for metadata to update + await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + assertQueryWorks(session); + return session.getMetadata().getNodes().size() <= 3; + }); + LOG.info("Node 1 decommissioned"); + + // Decommission node 2 + ccm.decommission(2); + nlb.removeNode(2); + allHostIds.remove(2); + postClientRoutes(ccm, allHostIds, nlb); + handler.refresh().toCompletableFuture().get(10, TimeUnit.SECONDS); + + await() + .atMost(60, TimeUnit.SECONDS) + .pollInterval(2, TimeUnit.SECONDS) + .until( + () -> { + assertQueryWorks(session); + return session.getMetadata().getNodes().size() <= 2; + }); + LOG.info("Node 2 decommissioned"); + + // ---- Stage 5: Verify everything works with only new nodes ---- + LOG.info("Stage 5: Verifying session works with only nodes 3 and 4"); + assertQueryWorks(session); + verifyConnectionsThroughProxy(session); + + // Verify only nodes 3 and 4 remain + Collection nodes = session.getMetadata().getNodes().values(); + assertThat(nodes).hasSize(2); + LOG.info("Final nodes: {}", nodes); + + // Run a few more queries to be sure + for (int i = 0; i < 10; i++) { + assertQueryWorks(session); + } + + LOG.info("PASS: Full node replacement test completed successfully"); + } + } finally { + nlb.close(); + } + } + } + + // ---- helpers ----------------------------------------------------------- + + private CqlSession openClientRoutesSession(ClientRoutesConfig config, NlbSimulator nlb) { + return (CqlSession) + SessionUtils.baseBuilder() + .addContactPoint(new InetSocketAddress("127.0.0.1", nlb.getDiscoveryPort())) + .withClientRoutesConfig(config) + .withConfigLoader( + SessionUtils.configLoaderBuilder() + .withDuration(DefaultDriverOption.REQUEST_TIMEOUT, Duration.ofSeconds(15)) + .withDuration( + DefaultDriverOption.CONNECTION_INIT_QUERY_TIMEOUT, Duration.ofSeconds(15)) + .withDuration( + DefaultDriverOption.CONTROL_CONNECTION_TIMEOUT, Duration.ofSeconds(15)) + .build()) + .build(); + } + + private void assertQueryWorks(CqlSession session) { + ResultSet rs = session.execute("SELECT release_version FROM system.local WHERE key='local'"); + Row row = rs.one(); + assertThat(row).as("Query via NLB should return a result").isNotNull(); + } + + private void verifyConnectionsThroughProxy(CqlSession session) { + Collection nodes = session.getMetadata().getNodes().values(); + for (Node node : nodes) { + InetSocketAddress addr = (InetSocketAddress) node.getEndPoint().resolve(); + LOG.info("Node endpoint: {} (port={})", addr, addr.getPort()); + // All connections should go through proxy ports (NLB_BASE_PORT + nodeId), + // not directly to 9042 + assertThat(addr.getPort()) + .as("Node %s should connect through proxy, not directly", node) + .isNotEqualTo(9042); + } + } + + /** + * Collects host IDs for nodes 1..nodeCount by querying system.local and system.peers via CQL on + * node 1. + */ + private Map collectHostIds(CcmBridge ccm, int nodeCount) { + Map hostIds = new HashMap<>(); + + // Query each node individually for its host_id from system.local + for (int i = 1; i <= nodeCount; i++) { + String nodeIp = ccm.getNodeIpAddress(i); + try (CqlSession adminSession = + (CqlSession) + SessionUtils.baseBuilder() + .addContactPoint(new InetSocketAddress(nodeIp, 9042)) + .withConfigLoader( + SessionUtils.configLoaderBuilder() + .withDuration(DefaultDriverOption.REQUEST_TIMEOUT, Duration.ofSeconds(10)) + .build()) + .build()) { + Row row = adminSession.execute("SELECT host_id FROM system.local WHERE key='local'").one(); + if (row != null) { + hostIds.put(i, row.getUuid("host_id")); + } + } + } + + return hostIds; + } + + /** + * Posts client routes to the ScyllaDB REST API (port 10000). Each route maps a connection_id + + * host_id to an NLB proxy port. + */ + private void postClientRoutes(CcmBridge ccm, Map hostIds, NlbSimulator nlb) + throws IOException { + // Find the first active node to send the API request to + int apiNodeId = -1; + for (int nodeId : hostIds.keySet()) { + apiNodeId = nodeId; + break; + } + if (apiNodeId < 0) { + throw new IllegalStateException("No active nodes to post routes to"); + } + + // Build JSON array of routes + StringBuilder json = new StringBuilder("["); + boolean first = true; + for (Map.Entry entry : hostIds.entrySet()) { + int nodeId = entry.getKey(); + UUID hostId = entry.getValue(); + int proxyPort = nlb.getNodePort(nodeId); + + if (!first) json.append(","); + first = false; + json.append("{") + .append("\"connection_id\":\"") + .append(CONNECTION_ID) + .append("\",") + .append("\"host_id\":\"") + .append(hostId) + .append("\",") + .append("\"address\":\"127.0.0.1\",") + .append("\"port\":") + .append(proxyPort) + .append("}"); + } + json.append("]"); + + String apiUrl = "http://" + ccm.getNodeIpAddress(apiNodeId) + ":10000/v2/client-routes"; + LOG.info("Posting routes to {}: {}", apiUrl, json); + + HttpURLConnection conn = (HttpURLConnection) new URL(apiUrl).openConnection(); + try { + conn.setRequestMethod("POST"); + conn.setRequestProperty("Content-Type", "application/json"); + conn.setDoOutput(true); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + + try (OutputStream os = conn.getOutputStream()) { + os.write(json.toString().getBytes(StandardCharsets.UTF_8)); + } + + int responseCode = conn.getResponseCode(); + if (responseCode >= 400) { + StringBuilder errorBody = new StringBuilder(); + try (BufferedReader br = + new BufferedReader( + new InputStreamReader(conn.getErrorStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = br.readLine()) != null) { + errorBody.append(line); + } + } + throw new IOException( + "Failed to post client routes: HTTP " + responseCode + " - " + errorBody); + } + + LOG.info("Routes posted successfully (HTTP {})", responseCode); + } finally { + conn.disconnect(); + } + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesUnsupportedVersionIT.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesUnsupportedVersionIT.java new file mode 100644 index 00000000000..528eaa0d6da --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/ClientRoutesUnsupportedVersionIT.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2025 ScyllaDB + * + * Modified by ScyllaDB + */ +package com.datastax.oss.driver.core.clientroutes; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; +import com.datastax.oss.driver.api.core.config.DefaultDriverOption; +import com.datastax.oss.driver.api.core.cql.ResultSet; +import com.datastax.oss.driver.api.core.cql.Row; +import com.datastax.oss.driver.api.testinfra.ScyllaOnly; +import com.datastax.oss.driver.api.testinfra.ScyllaRequirement; +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; +import com.datastax.oss.driver.api.testinfra.ccm.CustomCcmRule; +import com.datastax.oss.driver.api.testinfra.session.SessionRule; +import com.datastax.oss.driver.api.testinfra.session.SessionUtils; +import com.datastax.oss.driver.categories.IsolatedTests; +import com.datastax.oss.driver.internal.core.context.InternalDriverContext; +import com.datastax.oss.driver.internal.core.metadata.ClientRoutesTopologyMonitor; +import java.time.Duration; +import java.util.UUID; +import org.junit.AssumptionViolatedException; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.RuleChain; +import org.junit.rules.TestRule; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Negative integration test for the PrivateLink client routes feature. + * + *

This test class is intentionally not gated with a minimum version requirement: it + * must run on older ScyllaDB Enterprise versions (before 2026.1) where the {@code + * system.client_routes} table does not exist. + * + *

The single test here verifies that: + * + *

    + *
  1. {@code system.client_routes} is truly absent on the running server — if it is present, the + * test fails loudly to signal that the version gate in {@link ClientRoutesIT} needs updating. + *
  2. The driver gracefully opens a session even when the table is missing (no crash on init). + *
  3. The {@link ClientRoutesTopologyMonitor} returns {@code null} for all host IDs — the feature + * is not accidentally activated. + *
+ * + *

This class is annotated with {@code @ScyllaOnly} because {@code system.client_routes} is + * specific to ScyllaDB Enterprise (not yet available on OSS) and has no equivalent in Apache + * Cassandra. The {@code maxEnterprise = "2026.1"} constraint ensures this test is skipped on + * versions that do support the feature (it would have nothing meaningful to assert there, + * and the positive tests in {@link ClientRoutesIT} cover that range). + */ +@Category(IsolatedTests.class) +@ScyllaOnly(description = "system.client_routes is a ScyllaDB Enterprise-only feature") +@ScyllaRequirement( + maxEnterprise = "2026.1", + description = + "Negative test for Enterprise versions where system.client_routes does not exist. " + + "Skipped on Enterprise >= 2026.1 (use ClientRoutesIT for those versions).") +public class ClientRoutesUnsupportedVersionIT { + + private static final Logger LOG = LoggerFactory.getLogger(ClientRoutesUnsupportedVersionIT.class); + + public static final CustomCcmRule CCM_RULE = CustomCcmRule.builder().withNodes(1).build(); + + private static final SessionRule SESSION_RULE = + SessionRule.builder(CCM_RULE).withKeyspace(false).build(); + + @ClassRule + public static final TestRule CHAIN = RuleChain.outerRule(CCM_RULE).around(SESSION_RULE); + + // ---- helpers ----------------------------------------------------------- + + private String nodeAddress() { + return CCM_RULE.getCcmBridge().getNodeIpAddress(1); + } + + private CqlSession openAdminSession() { + return (CqlSession) + SessionUtils.baseBuilder() + .addContactEndPoints(CCM_RULE.getContactPoints()) + .withConfigLoader( + SessionUtils.configLoaderBuilder() + .withDuration(DefaultDriverOption.REQUEST_TIMEOUT, Duration.ofSeconds(10)) + .build()) + .build(); + } + + private CqlSession openClientRoutesSession(ClientRoutesConfig config) { + return (CqlSession) + SessionUtils.baseBuilder() + .addContactEndPoints(CCM_RULE.getContactPoints()) + .withClientRoutesConfig(config) + .withConfigLoader( + SessionUtils.configLoaderBuilder() + .withDuration(DefaultDriverOption.REQUEST_TIMEOUT, Duration.ofSeconds(10)) + .build()) + .build(); + } + + // ---- test -------------------------------------------------------------- + + /** + * Verifies that on a ScyllaDB Enterprise version that predates the {@code system.client_routes} + * table (i.e. Enterprise < 2026.1), the driver behaves correctly: + * + *

    + *
  • The system table does not exist — confirmed via {@code system_schema.tables}. + *
  • A session with {@link ClientRoutesConfig} can still be opened (graceful degradation). + *
  • The {@link ClientRoutesTopologyMonitor} stays empty; {@code translate()} returns {@code + * null}. + *
+ * + *

The assertion on the absent table acts as a built-in guard against stale version gates: if + * {@code system.client_routes} ever appears on a version thought to be unsupported, this test + * fails with a clear message rather than silently passing. + */ + @Test + public void should_not_activate_on_unsupported_version() { + // This test targets Scylla Enterprise only; OSS boundary is not yet confirmed. + if (!CcmBridge.SCYLLA_ENTERPRISE) { + throw new AssumptionViolatedException( + "Negative client-routes test targets Scylla Enterprise only " + + "(OSS feature boundary not yet confirmed). Skipping on OSS."); + } + + try (CqlSession admin = openAdminSession()) { + + // ── Step 1 ── confirm the table is genuinely absent ─────────────────────────────────────── + // If system.client_routes already exists on this build, the maxEnterprise gate in this + // class's @ScyllaRequirement is wrong (or the feature shipped earlier than 2026.1). + // Fail with an explicit message so the gate can be corrected. + Row tableRow = + admin + .execute( + "SELECT table_name FROM system_schema.tables" + + " WHERE keyspace_name='system' AND table_name='client_routes'") + .one(); + assertThat(tableRow) + .as( + "system.client_routes MUST NOT exist on Enterprise < 2026.1 — " + + "if this assertion fails the @ScyllaRequirement(minEnterprise) gate in " + + "ClientRoutesIT (or maxEnterprise here) is out of date and must be corrected.") + .isNull(); + + // ── Step 2 ── session must still open (graceful degradation) ────────────────────────────── + String connectionId = UUID.randomUUID().toString(); + String nodeAddr = nodeAddress(); + + // Use the real default table name (system.client_routes) — no user-space mirror — so any + // successful query result would mean the feature is genuinely active (it should not be). + ClientRoutesConfig config = + ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint(connectionId, nodeAddr + ":9042")) + .build(); + + try (CqlSession session = openClientRoutesSession(config)) { + // The session must be fully usable despite the missing table. + ResultSet rs = + session.execute("SELECT release_version FROM system.local WHERE key='local'"); + assertThat(rs.one()) + .as("Session opened with ClientRoutesConfig must be usable even on unsupported server") + .isNotNull(); + + // ── Step 3 ── translation must return null ──────────────────────────────────────────── + ClientRoutesTopologyMonitor handler = + ((InternalDriverContext) session.getContext()).getClientRoutesHandler(); + assertThat(handler) + .as("ClientRoutesHandler must be non-null whenever ClientRoutesConfig is set") + .isNotNull(); + + UUID anyHostId = UUID.randomUUID(); + try { + handler.resolve(anyHostId); + fail( + "ClientRoutesTopologyMonitor.resolve() must throw on an unsupported server — " + + "the feature must NOT be silently active when system.client_routes is absent"); + } catch (IllegalStateException | java.net.UnknownHostException e) { + assertThat(e.getMessage()).contains("No client route found"); + } + + LOG.info( + "Confirmed: client-routes feature correctly inactive on unsupported Enterprise {}", + CCM_RULE.getCcmBridge().getScyllaUnparsedVersion().orElse("")); + } + } + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/NlbSimulator.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/NlbSimulator.java new file mode 100644 index 00000000000..6c21974d5eb --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/NlbSimulator.java @@ -0,0 +1,169 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2025 ScyllaDB + * + * Modified by ScyllaDB + */ +package com.datastax.oss.driver.core.clientroutes; + +import com.datastax.oss.driver.api.testinfra.ccm.CcmBridge; +import java.io.Closeable; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Simulates a Network Load Balancer (NLB) for PrivateLink testing. + * + *

Provides: + * + *

    + *
  • A discovery port that round-robins across all registered nodes (simulates the + * cluster-level NLB endpoint). + *
  • A per-node port for each registered node (simulates per-node NLB routing driven by + * system.client_routes). + *
+ * + *

Nodes can be added and removed dynamically, mirroring real NLB reconfiguration during cluster + * topology changes. + */ +public class NlbSimulator implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(NlbSimulator.class); + + private final CcmBridge ccmBridge; + private final int basePort; + + // Discovery port proxy: round-robins across all nodes + private volatile RoundRobinProxy discoveryProxy; + + // Per-node proxies: nodeId -> TcpProxy + private final Map nodeProxies = + Collections.synchronizedMap(new LinkedHashMap<>()); + private final CopyOnWriteArrayList activeNodes = new CopyOnWriteArrayList<>(); + + /** + * Creates an NLB simulator. + * + * @param ccmBridge the CCM bridge to get node addresses from + * @param basePort the base port for NLB (discovery on basePort, per-node on basePort+nodeId) + */ + public NlbSimulator(CcmBridge ccmBridge, int basePort) { + this.ccmBridge = ccmBridge; + this.basePort = basePort; + } + + /** Returns the discovery port (the "cluster endpoint" that round-robins to all nodes). */ + public int getDiscoveryPort() { + return basePort; + } + + /** Returns the per-node proxy port for a given CCM node ID. */ + public int getNodePort(int nodeId) { + return basePort + nodeId; + } + + /** Returns the list of currently active node IDs. */ + public List getActiveNodes() { + return new ArrayList<>(activeNodes); + } + + /** + * Adds a node to the NLB. Creates a per-node proxy and adds the node to the discovery round-robin + * pool. + */ + public synchronized void addNode(int nodeId) throws IOException { + if (activeNodes.contains(nodeId)) { + LOG.warn("Node {} already registered in NLB", nodeId); + return; + } + + String nodeIp = ccmBridge.getNodeIpAddress(nodeId); + InetSocketAddress nodeAddr = new InetSocketAddress(nodeIp, 9042); + + // Create per-node proxy + int nodePort = getNodePort(nodeId); + TcpProxy proxy = new TcpProxy(nodePort, nodeAddr); + nodeProxies.put(nodeId, proxy); + activeNodes.add(nodeId); + + // Rebuild discovery proxy with updated node list + rebuildDiscoveryProxy(); + + LOG.info("NLB: added node{} ({}:{}) -> proxy port {}", nodeId, nodeIp, 9042, nodePort); + } + + /** Removes a node from the NLB. Closes its per-node proxy and removes it from discovery. */ + public synchronized void removeNode(int nodeId) throws IOException { + TcpProxy proxy = nodeProxies.remove(nodeId); + if (proxy != null) { + proxy.close(); + } + activeNodes.remove(Integer.valueOf(nodeId)); + + // Rebuild discovery proxy with updated node list + rebuildDiscoveryProxy(); + + LOG.info("NLB: removed node{}", nodeId); + } + + private void rebuildDiscoveryProxy() throws IOException { + // Close old discovery proxy + if (discoveryProxy != null) { + discoveryProxy.close(); + discoveryProxy = null; + } + + if (activeNodes.isEmpty()) { + return; + } + + // Build target list from active nodes + List targets = new ArrayList<>(); + for (int nodeId : activeNodes) { + String nodeIp = ccmBridge.getNodeIpAddress(nodeId); + targets.add(new InetSocketAddress(nodeIp, 9042)); + } + + discoveryProxy = new RoundRobinProxy(basePort, targets); + LOG.info( + "NLB: discovery proxy on port {} -> {} nodes: {}", basePort, targets.size(), activeNodes); + } + + @Override + public void close() { + if (discoveryProxy != null) { + discoveryProxy.close(); + discoveryProxy = null; + } + for (TcpProxy proxy : nodeProxies.values()) { + proxy.close(); + } + nodeProxies.clear(); + activeNodes.clear(); + LOG.info("NLB simulator closed"); + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/RoundRobinProxy.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/RoundRobinProxy.java new file mode 100644 index 00000000000..7a850e17017 --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/RoundRobinProxy.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2025 ScyllaDB + * + * Modified by ScyllaDB + */ +package com.datastax.oss.driver.core.clientroutes; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A TCP proxy that round-robins incoming connections across multiple backend targets. Each new + * connection is forwarded to the next backend in the list. + */ +public class RoundRobinProxy implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(RoundRobinProxy.class); + private static final int BUFFER_SIZE = 8192; + + private final ServerSocket serverSocket; + private final List targets; + private final AtomicInteger counter = new AtomicInteger(0); + private final AtomicBoolean closed = new AtomicBoolean(false); + private final CopyOnWriteArrayList activeSockets = new CopyOnWriteArrayList<>(); + private final Thread acceptThread; + + public RoundRobinProxy(int listenPort, List targets) throws IOException { + if (targets.isEmpty()) { + throw new IllegalArgumentException("At least one target required"); + } + this.targets = targets; + this.serverSocket = new ServerSocket(); + this.serverSocket.setReuseAddress(true); + this.serverSocket.bind(new InetSocketAddress("127.0.0.1", listenPort)); + this.acceptThread = + new Thread(this::acceptLoop, "rr-proxy-accept-" + serverSocket.getLocalPort()); + this.acceptThread.setDaemon(true); + this.acceptThread.start(); + LOG.debug( + "RoundRobinProxy listening on {} -> {} targets", serverSocket.getLocalPort(), targets); + } + + public int getLocalPort() { + return serverSocket.getLocalPort(); + } + + private void acceptLoop() { + while (!closed.get()) { + try { + Socket client = serverSocket.accept(); + if (closed.get()) { + client.close(); + break; + } + activeSockets.add(client); + + InetSocketAddress target = targets.get(counter.getAndIncrement() % targets.size()); + Socket remote = new Socket(); + activeSockets.add(remote); + remote.connect(target, 5000); + + Thread c2r = + new Thread( + () -> pipe(client, remote), + "rr-proxy-c2r-" + serverSocket.getLocalPort() + "-" + client.getPort()); + Thread r2c = + new Thread( + () -> pipe(remote, client), + "rr-proxy-r2c-" + serverSocket.getLocalPort() + "-" + client.getPort()); + c2r.setDaemon(true); + r2c.setDaemon(true); + c2r.start(); + r2c.start(); + } catch (SocketException e) { + if (!closed.get()) { + LOG.warn("RoundRobinProxy accept error", e); + } + } catch (IOException e) { + if (!closed.get()) { + LOG.warn("RoundRobinProxy accept error", e); + } + } + } + } + + private void pipe(Socket from, Socket to) { + byte[] buf = new byte[BUFFER_SIZE]; + try { + InputStream in = from.getInputStream(); + OutputStream out = to.getOutputStream(); + int n; + while ((n = in.read(buf)) >= 0) { + out.write(buf, 0, n); + out.flush(); + } + } catch (IOException e) { + // expected when sockets close + } finally { + closeQuietly(from); + closeQuietly(to); + } + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + closeQuietly(serverSocket); + for (Socket s : activeSockets) { + closeQuietly(s); + } + activeSockets.clear(); + } + } + + private static void closeQuietly(Closeable c) { + try { + if (c != null) c.close(); + } catch (IOException ignored) { + } + } +} diff --git a/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/TcpProxy.java b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/TcpProxy.java new file mode 100644 index 00000000000..11f719588ad --- /dev/null +++ b/integration-tests/src/test/java/com/datastax/oss/driver/core/clientroutes/TcpProxy.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Copyright (C) 2025 ScyllaDB + * + * Modified by ScyllaDB + */ +package com.datastax.oss.driver.core.clientroutes; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicBoolean; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A simple TCP proxy that forwards connections from a local port to a remote target. Each accepted + * connection spawns two threads to pipe data in both directions. + */ +public class TcpProxy implements Closeable { + private static final Logger LOG = LoggerFactory.getLogger(TcpProxy.class); + private static final int BUFFER_SIZE = 8192; + + private final ServerSocket serverSocket; + private final InetSocketAddress target; + private final AtomicBoolean closed = new AtomicBoolean(false); + private final CopyOnWriteArrayList activeSockets = new CopyOnWriteArrayList<>(); + private final Thread acceptThread; + + /** + * Creates and starts a TCP proxy. + * + * @param listenPort the local port to listen on (0 for any available port) + * @param target the remote address to forward connections to + */ + public TcpProxy(int listenPort, InetSocketAddress target) throws IOException { + this.target = target; + this.serverSocket = new ServerSocket(); + this.serverSocket.setReuseAddress(true); + this.serverSocket.bind(new InetSocketAddress("127.0.0.1", listenPort)); + this.acceptThread = + new Thread(this::acceptLoop, "tcp-proxy-accept-" + serverSocket.getLocalPort()); + this.acceptThread.setDaemon(true); + this.acceptThread.start(); + LOG.debug("TcpProxy listening on {} -> {}", serverSocket.getLocalPort(), target); + } + + /** Returns the local port this proxy is listening on. */ + public int getLocalPort() { + return serverSocket.getLocalPort(); + } + + private void acceptLoop() { + while (!closed.get()) { + try { + Socket client = serverSocket.accept(); + if (closed.get()) { + client.close(); + break; + } + activeSockets.add(client); + Socket remote = new Socket(); + activeSockets.add(remote); + remote.connect(target, 5000); + + Thread c2r = + new Thread( + () -> pipe(client, remote), + "tcp-proxy-c2r-" + serverSocket.getLocalPort() + "-" + client.getPort()); + Thread r2c = + new Thread( + () -> pipe(remote, client), + "tcp-proxy-r2c-" + serverSocket.getLocalPort() + "-" + client.getPort()); + c2r.setDaemon(true); + r2c.setDaemon(true); + c2r.start(); + r2c.start(); + } catch (SocketException e) { + if (!closed.get()) { + LOG.warn("TcpProxy accept error", e); + } + } catch (IOException e) { + if (!closed.get()) { + LOG.warn("TcpProxy accept error", e); + } + } + } + } + + private void pipe(Socket from, Socket to) { + byte[] buf = new byte[BUFFER_SIZE]; + try { + InputStream in = from.getInputStream(); + OutputStream out = to.getOutputStream(); + int n; + while ((n = in.read(buf)) >= 0) { + out.write(buf, 0, n); + out.flush(); + } + } catch (IOException e) { + // expected when sockets close + } finally { + closeQuietly(from); + closeQuietly(to); + } + } + + @Override + public void close() { + if (closed.compareAndSet(false, true)) { + closeQuietly(serverSocket); + for (Socket s : activeSockets) { + closeQuietly(s); + } + activeSockets.clear(); + LOG.debug("TcpProxy on port {} closed", serverSocket.getLocalPort()); + } + } + + private static void closeQuietly(Closeable c) { + try { + if (c != null) c.close(); + } catch (IOException ignored) { + } + } +} diff --git a/manual/core/address_resolution/README.md b/manual/core/address_resolution/README.md index 84efb4a796c..6fb868492de 100644 --- a/manual/core/address_resolution/README.md +++ b/manual/core/address_resolution/README.md @@ -118,6 +118,88 @@ datastax-java-driver.advanced.address-translator.class = com.mycompany.MyAddress Note: the contact points provided while creating the `CqlSession` are not translated, only addresses retrieved from or sent by Cassandra nodes are. +### Client Routes (cloud private endpoint deployments) + +For cloud deployments using private endpoint services (such as AWS PrivateLink, Azure Private Link, +or GCP Private Service Connect) or similar technologies (e.g., ScyllaDB Cloud), nodes are accessed +through private DNS endpoints rather than direct IP addresses. The driver +provides a built-in client routes feature that handles address translation automatically. + +Client routes configuration is done programmatically and is **mutually exclusive** with: +- A custom `AddressTranslator` (if both are provided, an `IllegalStateException` is thrown) +- Cloud secure connect bundles (if both are provided, an `IllegalStateException` is thrown) + +#### Quick start + +```java +import com.datastax.oss.driver.api.core.CqlSession; +import com.datastax.oss.driver.api.core.config.ClientRoutesConfig; +import com.datastax.oss.driver.api.core.config.ClientRoutesEndpoint; +import java.util.UUID; + +ClientRoutesConfig config = ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint( + UUID.fromString("12345678-1234-1234-1234-123456789012"), + "my-cluster-endpoint.example.com")) + .build(); + +// Contact points are seeded automatically from endpoint addresses. +CqlSession session = CqlSession.builder() + .withClientRoutesConfig(config) + .withLocalDatacenter("datacenter1") + .build(); +``` + +#### How it works + +1. **Startup** — after the control connection is established, the driver queries + `system.client_routes` (filtered to the configured `connection_id` values) and builds an + in-memory map of `host_id → (hostname, port, tls_port)`. +2. **Translation** — every time the driver opens a connection to a peer node, it looks up the + node's `host_id` in the route map and resolves the associated DNS hostname. Contact points bypass + translation so the initial seed addresses are used as-is. +3. **Event-driven updates** — the driver registers for `CLIENT_ROUTES_CHANGE` server events. When + one arrives, it re-queries the table and atomically swaps the route map. +4. **Reconnect** — if the control connection is recreated the driver performs a full re-read of the + route table before refreshing node metadata. + +#### DNS caching + +DNS is resolved at connection time (not at route discovery time). The driver caches resolved +addresses per hostname with a configurable TTL (default: **500 ms**): + +```java +ClientRoutesConfig config = ClientRoutesConfig.builder() + .addEndpoint(...) + .withDnsCacheDuration(1_000) // milliseconds + .build(); +``` + +Additional properties of the DNS resolver: + +- **Per-hostname concurrency limit**: at most one in-flight DNS lookup per hostname; concurrent + callers wait and reuse the result. +- **Last-known-good fallback**: if a DNS lookup fails, the resolver returns the last successfully + resolved address instead of throwing, so transient DNS hiccups do not cause connection failures. +- **Route-map refresh** — the driver re-queries `system.client_routes` and atomically swaps the + in-memory route map in two situations: + - a `CLIENT_ROUTES_CHANGE` server event is received, or + - the control connection reconnects after a failure (`onSuccessfulReconnect`). + + A route-map refresh does **not** flush the DNS cache. Existing cached addresses remain valid + until their TTL expires; new hostnames are resolved on first use. +- **`clearCache()`** — called exactly once, when the session is closed (`CqlSession.close()`). It + is not called during route-map refreshes and is not intended for manual or administrative use. + Last-known-good entries are intentionally kept across a cache clear: should the same hostname be + re-resolved after a close/reopen cycle, the fallback address is still available. + +#### Limitations + +- Requires ScyllaDB Enterprise ≥ 2026.1 with `system.client_routes` support + (scylladb/scylladb#27323). Not yet available on ScyllaDB OSS. +- Not supported on Apache Cassandra. +- Mutually exclusive with custom `AddressTranslator` and with cloud secure connect bundles. + ### EC2 multi-region If you deploy both Cassandra and client applications on Amazon EC2, and your cluster spans multiple regions, you'll have diff --git a/pom.xml b/pom.xml index 4e1ac550503..1b8ea4d948e 100644 --- a/pom.xml +++ b/pom.xml @@ -171,7 +171,7 @@ com.scylladb native-protocol - 1.5.2.1 + 1.5.2.2 io.netty diff --git a/upgrade_guide/README.md b/upgrade_guide/README.md index 8fdf068919a..aa2bd02c32f 100644 --- a/upgrade_guide/README.md +++ b/upgrade_guide/README.md @@ -19,6 +19,42 @@ under the License. ## Upgrade guide +### 4.19.0 + +#### PrivateLink support via client routes + +The driver now supports automatic address translation for ScyllaDB Cloud PrivateLink deployments +through the new client routes feature. When enabled, the driver reads endpoint mappings from the +`system.client_routes` system table and translates peer addresses transparently at connection time, +with TTL-based DNS caching and automatic refresh on `CLIENT_ROUTES_CHANGE` events. + +Configure it programmatically on the session builder: + +```java +ClientRoutesConfig config = ClientRoutesConfig.builder() + .addEndpoint(new ClientRoutesEndpoint( + UUID.fromString(""), + "my-cluster.region.provider.scylladb.com:9042")) + .withDnsCacheDuration(500) // optional, default 500 ms + .build(); + +CqlSession session = CqlSession.builder() + .withClientRoutesConfig(config) + .withLocalDatacenter("datacenter1") + .build(); +``` + +Key points: + +- **Contact points are seeded automatically** from the endpoint `connectionAddr` values when no + explicit contact points are configured. +- **Mutually exclusive** with a custom `AddressTranslator` and with cloud secure connect bundles — + providing both throws `IllegalStateException` at session build time. +- **Requires ScyllaDB Enterprise ≥ 2026.1** (scylladb/scylladb#27323). The feature is not + available on ScyllaDB OSS or Apache Cassandra. + +See [Address resolution — Client Routes](../manual/core/address_resolution/) for full details. + ### 4.18.1 #### Keystore reloading in DefaultSslEngineFactory