Skip to content

Commit 1fa95a9

Browse files
nimfJennnnnyolavloite
authored
feat: add gRPC-GCP channel pool as an option (#1227)
* add grpc-gcp extensions * remove local files * update read apiconfig * update read apiconfig * Add custom pool size for GCP extension * add one more entry in aip config file * change gcp package name * Adjust grpc-gcp package name and group * Set grpc-gcp low streams watermark to 1 This allows sessions to be spread across channels when the Spanner client starts up. * Add gRPC-GCP extension options to SpannerOptions * feat: add gRPC-GCP channel pool as an option This enables a user to opt-in for using the gRPC-GCP extension channel pool and configure its options. * Addressed comments. * Fixed linting issues. * Add integration test with enabled gRPC-GCP extension * Remake gRPC-GCP extension related SpannerOptions * Add ChannelUsageTest Co-authored-by: Zhenli(Jenny) Jiang <[email protected]> Co-authored-by: Knut Olav Løite <[email protected]>
1 parent b2a56c6 commit 1fa95a9

File tree

6 files changed

+569
-2
lines changed

6 files changed

+569
-2
lines changed

‎google-cloud-spanner/pom.xml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,12 @@
117117
</build>
118118

119119
<dependencies>
120+
121+
<dependency>
122+
<groupId>com.google.cloud</groupId>
123+
<artifactId>grpc-gcp</artifactId>
124+
<version>1.0.0</version>
125+
</dependency>
120126
<dependency>
121127
<groupId>io.grpc</groupId>
122128
<artifactId>grpc-api</artifactId>

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerOptions.java

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,9 @@
2929
import com.google.cloud.ServiceOptions;
3030
import com.google.cloud.ServiceRpc;
3131
import com.google.cloud.TransportOptions;
32+
import com.google.cloud.grpc.GcpManagedChannelOptions;
3233
import com.google.cloud.grpc.GrpcTransportOptions;
3334
import com.google.cloud.spanner.Options.QueryOption;
34-
import com.google.cloud.spanner.SpannerOptions.CallContextConfigurator;
35-
import com.google.cloud.spanner.SpannerOptions.SpannerCallContextTimeoutConfigurator;
3635
import com.google.cloud.spanner.admin.database.v1.DatabaseAdminSettings;
3736
import com.google.cloud.spanner.admin.database.v1.stub.DatabaseAdminStubSettings;
3837
import com.google.cloud.spanner.admin.instance.v1.InstanceAdminSettings;
@@ -103,6 +102,8 @@ public class SpannerOptions extends ServiceOptions<Spanner, SpannerOptions> {
103102
private final InstanceAdminStubSettings instanceAdminStubSettings;
104103
private final DatabaseAdminStubSettings databaseAdminStubSettings;
105104
private final Duration partitionedDmlTimeout;
105+
private final boolean grpcGcpExtensionEnabled;
106+
private final GcpManagedChannelOptions grpcGcpOptions;
106107
private final boolean autoThrottleAdministrativeRequests;
107108
private final RetrySettings retryAdministrativeRequestsSettings;
108109
private final boolean trackTransactionStarter;
@@ -554,6 +555,8 @@ private SpannerOptions(Builder builder) {
554555
throw SpannerExceptionFactory.newSpannerException(e);
555556
}
556557
partitionedDmlTimeout = builder.partitionedDmlTimeout;
558+
grpcGcpExtensionEnabled = builder.grpcGcpExtensionEnabled;
559+
grpcGcpOptions = builder.grpcGcpOptions;
557560
autoThrottleAdministrativeRequests = builder.autoThrottleAdministrativeRequests;
558561
retryAdministrativeRequestsSettings = builder.retryAdministrativeRequestsSettings;
559562
trackTransactionStarter = builder.trackTransactionStarter;
@@ -658,6 +661,8 @@ public static class Builder
658661
private DatabaseAdminStubSettings.Builder databaseAdminStubSettingsBuilder =
659662
DatabaseAdminStubSettings.newBuilder();
660663
private Duration partitionedDmlTimeout = Duration.ofHours(2L);
664+
private boolean grpcGcpExtensionEnabled = false;
665+
private GcpManagedChannelOptions grpcGcpOptions;
661666
private RetrySettings retryAdministrativeRequestsSettings =
662667
DEFAULT_ADMIN_REQUESTS_LIMIT_EXCEEDED_RETRY_SETTINGS;
663668
private boolean autoThrottleAdministrativeRequests = false;
@@ -707,6 +712,8 @@ private Builder() {
707712
this.instanceAdminStubSettingsBuilder = options.instanceAdminStubSettings.toBuilder();
708713
this.databaseAdminStubSettingsBuilder = options.databaseAdminStubSettings.toBuilder();
709714
this.partitionedDmlTimeout = options.partitionedDmlTimeout;
715+
this.grpcGcpExtensionEnabled = options.grpcGcpExtensionEnabled;
716+
this.grpcGcpOptions = options.grpcGcpOptions;
710717
this.autoThrottleAdministrativeRequests = options.autoThrottleAdministrativeRequests;
711718
this.retryAdministrativeRequestsSettings = options.retryAdministrativeRequestsSettings;
712719
this.trackTransactionStarter = options.trackTransactionStarter;
@@ -1035,6 +1042,28 @@ public Builder setHost(String host) {
10351042
return this;
10361043
}
10371044

1045+
/** Enables gRPC-GCP extension with the default settings. */
1046+
public Builder enableGrpcGcpExtension() {
1047+
this.grpcGcpExtensionEnabled = true;
1048+
return this;
1049+
}
1050+
1051+
/**
1052+
* Enables gRPC-GCP extension and uses provided options for configuration. The metric registry
1053+
* and default Spanner metric labels will be added automatically.
1054+
*/
1055+
public Builder enableGrpcGcpExtension(GcpManagedChannelOptions options) {
1056+
this.grpcGcpExtensionEnabled = true;
1057+
this.grpcGcpOptions = options;
1058+
return this;
1059+
}
1060+
1061+
/** Disables gRPC-GCP extension. */
1062+
public Builder disableGrpcGcpExtension() {
1063+
this.grpcGcpExtensionEnabled = false;
1064+
return this;
1065+
}
1066+
10381067
/**
10391068
* Sets the host of an emulator to use. By default the value is read from an environment
10401069
* variable. If the environment variable is not set, this will be <code>null</code>.
@@ -1128,6 +1157,14 @@ public Duration getPartitionedDmlTimeout() {
11281157
return partitionedDmlTimeout;
11291158
}
11301159

1160+
public boolean isGrpcGcpExtensionEnabled() {
1161+
return grpcGcpExtensionEnabled;
1162+
}
1163+
1164+
public GcpManagedChannelOptions getGrpcGcpOptions() {
1165+
return grpcGcpOptions;
1166+
}
1167+
11311168
public boolean isAutoThrottleAdministrativeRequests() {
11321169
return autoThrottleAdministrativeRequests;
11331170
}

‎google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/GapicSpannerRpc.java

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818

1919
import static com.google.cloud.spanner.SpannerExceptionFactory.newSpannerException;
2020

21+
import com.google.api.core.ApiFunction;
2122
import com.google.api.core.ApiFuture;
2223
import com.google.api.core.InternalApi;
2324
import com.google.api.core.NanoClock;
@@ -54,6 +55,9 @@
5455
import com.google.api.pathtemplate.PathTemplate;
5556
import com.google.cloud.RetryHelper;
5657
import com.google.cloud.RetryHelper.RetryHelperException;
58+
import com.google.cloud.grpc.GcpManagedChannelBuilder;
59+
import com.google.cloud.grpc.GcpManagedChannelOptions;
60+
import com.google.cloud.grpc.GcpManagedChannelOptions.GcpMetricsOptions;
5761
import com.google.cloud.grpc.GrpcTransportOptions;
5862
import com.google.cloud.spanner.AdminRequestsPerMinuteExceededException;
5963
import com.google.cloud.spanner.ErrorCode;
@@ -80,6 +84,7 @@
8084
import com.google.common.base.Preconditions;
8185
import com.google.common.collect.ImmutableList;
8286
import com.google.common.collect.ImmutableSet;
87+
import com.google.common.io.Resources;
8388
import com.google.common.util.concurrent.RateLimiter;
8489
import com.google.common.util.concurrent.ThreadFactoryBuilder;
8590
import com.google.iam.v1.GetIamPolicyRequest;
@@ -156,10 +161,13 @@
156161
import com.google.spanner.v1.Transaction;
157162
import io.grpc.CallCredentials;
158163
import io.grpc.Context;
164+
import io.grpc.ManagedChannelBuilder;
159165
import io.grpc.MethodDescriptor;
166+
import io.opencensus.metrics.Metrics;
160167
import java.io.IOException;
161168
import java.io.UnsupportedEncodingException;
162169
import java.net.URLDecoder;
170+
import java.nio.charset.Charset;
163171
import java.nio.charset.StandardCharsets;
164172
import java.util.Comparator;
165173
import java.util.HashMap;
@@ -249,6 +257,7 @@ private void awaitTermination() throws InterruptedException {
249257
private static final String CLIENT_LIBRARY_LANGUAGE = "spanner-java";
250258
public static final String DEFAULT_USER_AGENT =
251259
CLIENT_LIBRARY_LANGUAGE + "/" + GaxProperties.getLibraryVersion(GapicSpannerRpc.class);
260+
private static final String API_FILE = "grpc-gcp-apiconfig.json";
252261

253262
private final ManagedInstantiatingExecutorProvider executorProvider;
254263
private boolean rpcIsClosed;
@@ -368,6 +377,9 @@ public GapicSpannerRpc(final SpannerOptions options) {
368377
// whether the attempt is allowed is totally controlled by service owner.
369378
.setAttemptDirectPath(true);
370379

380+
// If it is enabled in options uses the channel pool provided by the gRPC-GCP extension.
381+
maybeEnableGrpcGcpExtension(defaultChannelProviderBuilder, options);
382+
371383
TransportChannelProvider channelProvider =
372384
MoreObjects.firstNonNull(
373385
options.getChannelProvider(), defaultChannelProviderBuilder.build());
@@ -509,6 +521,62 @@ public <RequestT, ResponseT> UnaryCallable<RequestT, ResponseT> createUnaryCalla
509521
}
510522
}
511523

524+
private static String parseGrpcGcpApiConfig() {
525+
try {
526+
return Resources.toString(
527+
GapicSpannerRpc.class.getResource(API_FILE), Charset.forName("UTF8"));
528+
} catch (IOException e) {
529+
throw newSpannerException(e);
530+
}
531+
}
532+
533+
// Enhance metric options for gRPC-GCP extension. Adds metric registry if not specified.
534+
private static GcpManagedChannelOptions grpcGcpOptionsWithMetrics(SpannerOptions options) {
535+
GcpManagedChannelOptions grpcGcpOptions =
536+
MoreObjects.firstNonNull(options.getGrpcGcpOptions(), new GcpManagedChannelOptions());
537+
GcpMetricsOptions metricsOptions =
538+
MoreObjects.firstNonNull(
539+
grpcGcpOptions.getMetricsOptions(), GcpMetricsOptions.newBuilder().build());
540+
GcpMetricsOptions.Builder metricsOptionsBuilder = GcpMetricsOptions.newBuilder(metricsOptions);
541+
if (metricsOptions.getMetricRegistry() == null) {
542+
metricsOptionsBuilder.withMetricRegistry(Metrics.getMetricRegistry());
543+
}
544+
// TODO: Add default labels with values: client_id, database, instance_id.
545+
if (metricsOptions.getNamePrefix().equals("")) {
546+
metricsOptionsBuilder.withNamePrefix("cloud.google.com/java/spanner/gcp-channel-pool/");
547+
}
548+
return GcpManagedChannelOptions.newBuilder(grpcGcpOptions)
549+
.withMetricsOptions(metricsOptionsBuilder.build())
550+
.build();
551+
}
552+
553+
@SuppressWarnings("rawtypes")
554+
private static void maybeEnableGrpcGcpExtension(
555+
InstantiatingGrpcChannelProvider.Builder defaultChannelProviderBuilder,
556+
final SpannerOptions options) {
557+
if (!options.isGrpcGcpExtensionEnabled()) {
558+
return;
559+
}
560+
561+
final String jsonApiConfig = parseGrpcGcpApiConfig();
562+
final GcpManagedChannelOptions grpcGcpOptions = grpcGcpOptionsWithMetrics(options);
563+
564+
ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> apiFunction =
565+
channelBuilder -> {
566+
if (options.getChannelConfigurator() != null) {
567+
channelBuilder = options.getChannelConfigurator().apply(channelBuilder);
568+
}
569+
return GcpManagedChannelBuilder.forDelegateBuilder(channelBuilder)
570+
.withApiConfigJsonString(jsonApiConfig)
571+
.withOptions(grpcGcpOptions)
572+
.setPoolSize(options.getNumChannels());
573+
};
574+
575+
// Disable the GAX channel pooling functionality by setting the GAX channel pool size to 1.
576+
// Enable gRPC-GCP channel pool via the channel configurator.
577+
defaultChannelProviderBuilder.setPoolSize(1).setChannelConfigurator(apiFunction);
578+
}
579+
512580
private static HeaderProvider headerProviderWithUserAgentFrom(HeaderProvider headerProvider) {
513581
final Map<String, String> headersWithUserAgent = new HashMap<>(headerProvider.getHeaders());
514582
String userAgent = null;

‎google-cloud-spanner/src/main/resources/com/google/cloud/spanner/spi/v1/grpc-gcp-apiconfig.json

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
{
2+
"channelPool": {
3+
"maxSize": 3,
4+
"maxConcurrentStreamsLowWatermark": 0
5+
},
6+
"method": [
7+
{
8+
"name": ["google.spanner.v1.Spanner/CreateSession"],
9+
"affinity" : {
10+
"command": "BIND",
11+
"affinityKey": "name"
12+
}
13+
},
14+
{
15+
"name": ["google.spanner.v1.Spanner/BatchCreateSessions"],
16+
"affinity" : {
17+
"command": "BIND",
18+
"affinityKey": "session.name"
19+
}
20+
},
21+
{
22+
"name": ["google.spanner.v1.Spanner/GetSession"],
23+
"affinity": {
24+
"command": "BOUND",
25+
"affinityKey": "name"
26+
}
27+
},
28+
{
29+
"name": ["google.spanner.v1.Spanner/DeleteSession"],
30+
"affinity": {
31+
"command": "UNBIND",
32+
"affinityKey": "name"
33+
}
34+
},
35+
{
36+
"name": ["google.spanner.v1.Spanner/ExecuteSql"],
37+
"affinity": {
38+
"command": "BOUND",
39+
"affinityKey": "session"
40+
}
41+
},
42+
{
43+
"name": ["google.spanner.v1.Spanner/ExecuteBatchDml"],
44+
"affinity": {
45+
"command": "BOUND",
46+
"affinityKey": "session"
47+
}
48+
},
49+
{
50+
"name": ["google.spanner.v1.Spanner/ExecuteStreamingSql"],
51+
"affinity": {
52+
"command": "BOUND",
53+
"affinityKey": "session"
54+
}
55+
},
56+
{
57+
"name": ["google.spanner.v1.Spanner/Read"],
58+
"affinity": {
59+
"command": "BOUND",
60+
"affinityKey": "session"
61+
}
62+
},
63+
{
64+
"name": ["google.spanner.v1.Spanner/StreamingRead"],
65+
"affinity": {
66+
"command": "BOUND",
67+
"affinityKey": "session"
68+
}
69+
},
70+
{
71+
"name": ["google.spanner.v1.Spanner/BeginTransaction"],
72+
"affinity": {
73+
"command": "BOUND",
74+
"affinityKey": "session"
75+
}
76+
},
77+
{
78+
"name": ["google.spanner.v1.Spanner/Commit"],
79+
"affinity": {
80+
"command": "BOUND",
81+
"affinityKey": "session"
82+
}
83+
},
84+
{
85+
"name": ["google.spanner.v1.Spanner/PartitionRead"],
86+
"affinity": {
87+
"command": "BOUND",
88+
"affinityKey": "session"
89+
}
90+
},
91+
{
92+
"name": ["google.spanner.v1.Spanner/PartitionQuery"],
93+
"affinity": {
94+
"command": "BOUND",
95+
"affinityKey": "session"
96+
}
97+
},
98+
{
99+
"name": ["google.spanner.v1.Spanner/Rollback"],
100+
"affinity": {
101+
"command": "BOUND",
102+
"affinityKey": "session"
103+
}
104+
}
105+
]
106+
}

0 commit comments

Comments
 (0)