-
Notifications
You must be signed in to change notification settings - Fork 546
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
dark cluster schema and serializer changes
RB=1882175 BUG=REX-1809,REVIEW-15710 G=si-core-reviewers R=cxu,ssheng,crzhang A=swiser,crzhang
- Loading branch information
David Hoa
committed
Dec 20, 2019
1 parent
aa4d897
commit d2d1c09
Showing
9 changed files
with
352 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,7 @@ | ||
| 28.1.4 | ||
| ------ | ||
|
|
||
|
|
||
| 28.1.3 | ||
| ------ | ||
| (RB=1920382) | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
26 changes: 26 additions & 0 deletions
26
d2-schemas/src/main/pegasus/com/linkedin/d2/DarkClusterConfig.pdsc
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| { | ||
| "type" : "record", | ||
| "name" : "DarkClusterConfig", | ||
| "namespace" : "com.linkedin.d2", | ||
| "doc" : "Configuration for a dark canary cluster. Dark Canaries are instances of a service that have production traffic tee'd off to them, but the results are ignored. These are used for early validation of code, configs, and A/B ramps.", | ||
| "fields" : [ | ||
| { | ||
| "name" : "multiplier", | ||
| "type" : "float", | ||
| "doc" : "Constant multiplier. The dispatcher(s) will send a multiple of the original requests", | ||
| "default" : 0 | ||
| }, | ||
| { | ||
| "name" : "dispatcherOutboundTargetRate", | ||
| "type" : "int", | ||
| "doc" : "Desired query rate to be maintained to dark canaries. Measured in qps.", | ||
| "default" : 0 | ||
| }, | ||
| { | ||
| "name" : "dispatcherOutboundMaxRate", | ||
| "type" : "int", | ||
| "doc" : "Max rate dispatcher can send to dark canary. Measured in qps. Will act as upper bound to protect canaries in case of traffic spikes", | ||
| "default" : 2147483647 | ||
| } | ||
| ] | ||
| } |
84 changes: 84 additions & 0 deletions
84
d2/src/main/java/com/linkedin/d2/balancer/config/DarkClustersConverter.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,84 @@ | ||
| /* | ||
| Copyright (c) 2019 LinkedIn Corp. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package com.linkedin.d2.balancer.config; | ||
|
|
||
| import com.linkedin.d2.DarkClusterConfigMap; | ||
| import com.linkedin.d2.balancer.util.JacksonUtil; | ||
| import com.linkedin.data.codec.JacksonDataCodec; | ||
| import com.linkedin.data.schema.validation.CoercionMode; | ||
| import com.linkedin.data.schema.validation.RequiredMode; | ||
| import com.linkedin.data.schema.validation.ValidateDataAgainstSchema; | ||
| import com.linkedin.data.schema.validation.ValidationOptions; | ||
| import java.io.IOException; | ||
| import java.util.Collections; | ||
| import java.util.Map; | ||
|
|
||
|
|
||
| /** | ||
| * This class converts {@link DarkClusterConfigMap} into a Map | ||
| * that can be stored in zookeeper and vice versa. | ||
| * | ||
| * @author David Hoa (dhoa@linkedin.com) | ||
| */ | ||
| public class DarkClustersConverter | ||
| { | ||
| private static final JacksonDataCodec CODEC = new JacksonDataCodec(); | ||
| private static final ValidationOptions VALIDATION_OPTIONS = | ||
| new ValidationOptions(RequiredMode.FIXUP_ABSENT_WITH_DEFAULT, CoercionMode.STRING_TO_PRIMITIVE); | ||
|
|
||
| @SuppressWarnings("unchecked") | ||
| public static Map<String, Object> toProperties(DarkClusterConfigMap config) | ||
| { | ||
| if (config == null) | ||
| { | ||
| return Collections.emptyMap(); | ||
| } | ||
| else | ||
| { | ||
| try | ||
| { | ||
| String json = CODEC.mapToString(config.data()); | ||
| return JacksonUtil.getObjectMapper().readValue(json, Map.class); | ||
| } | ||
| catch (IOException e) | ||
| { | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public static DarkClusterConfigMap toConfig(Map<String, Object> properties) | ||
| { | ||
| try | ||
| { | ||
| if (properties == null) | ||
| { | ||
| return new DarkClusterConfigMap(); | ||
| } | ||
| String json = JacksonUtil.getObjectMapper().writeValueAsString(properties); | ||
| DarkClusterConfigMap darkClusterConfigMap = new DarkClusterConfigMap(CODEC.stringToMap(json)); | ||
| //fixes are applied in place | ||
| ValidateDataAgainstSchema.validate(darkClusterConfigMap, VALIDATION_OPTIONS); | ||
|
|
||
| return darkClusterConfigMap; | ||
| } | ||
| catch (IOException e) | ||
| { | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
121 changes: 121 additions & 0 deletions
121
d2/src/test/java/com/linkedin/d2/balancer/config/DarkClustersConverterTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,121 @@ | ||
| /* | ||
| Copyright (c) 2019 LinkedIn Corp. | ||
| Licensed under the Apache License, Version 2.0 (the "License"); | ||
| you may not use this file except in compliance with the License. | ||
| You may obtain a copy of the License at | ||
| http://www.apache.org/licenses/LICENSE-2.0 | ||
| Unless required by applicable law or agreed to in writing, software | ||
| distributed under the License is distributed on an "AS IS" BASIS, | ||
| WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| See the License for the specific language governing permissions and | ||
| limitations under the License. | ||
| */ | ||
|
|
||
| package com.linkedin.d2.balancer.config; | ||
|
|
||
| import com.linkedin.d2.DarkClusterConfig; | ||
| import com.linkedin.d2.DarkClusterConfigMap; | ||
| import org.testng.Assert; | ||
| import org.testng.annotations.DataProvider; | ||
| import org.testng.annotations.Test; | ||
|
|
||
| import static com.linkedin.d2.balancer.properties.ClusterProperties.DARK_CLUSTER_DEFAULT_MAX_RATE; | ||
| import static com.linkedin.d2.balancer.properties.ClusterProperties.DARK_CLUSTER_DEFAULT_MULTIPLIER; | ||
| import static com.linkedin.d2.balancer.properties.ClusterProperties.DARK_CLUSTER_DEFAULT_TARGET_RATE; | ||
|
|
||
|
|
||
| public class DarkClustersConverterTest | ||
| { | ||
| private static String DARK_CLUSTER_KEY = "foobar1dark"; | ||
|
|
||
| @DataProvider | ||
| public Object[][] provideKeys() | ||
| { | ||
| return new Object[][] { | ||
| new Object[] {true, new DarkClusterConfig().setMultiplier(0.5f).setDispatcherOutboundTargetRate(0).setDispatcherOutboundMaxRate(1234566)}, | ||
| // multiplier is default, the default will be filled in | ||
| new Object[] {false, new DarkClusterConfig().setDispatcherOutboundTargetRate(456).setDispatcherOutboundMaxRate(1234566)}, | ||
| // dynamic multiplier defaults, the default will be filled in | ||
| new Object[] {false, new DarkClusterConfig().setMultiplier(0.5f)}, | ||
| // test zeros | ||
| new Object[] {true, new DarkClusterConfig().setMultiplier(0.0f).setDispatcherOutboundTargetRate(0).setDispatcherOutboundMaxRate(0)}, | ||
| // negative multiplier not allowed | ||
| new Object[] {false, new DarkClusterConfig().setMultiplier(-1.0f).setDispatcherOutboundTargetRate(0).setDispatcherOutboundMaxRate(1234566)}, | ||
| // netative target rate not allowed | ||
| new Object[] {false, new DarkClusterConfig().setMultiplier(0.0f).setDispatcherOutboundTargetRate(-1).setDispatcherOutboundMaxRate(1234566)}, | ||
| // negative max rate not allowed | ||
| new Object[] {false, new DarkClusterConfig().setMultiplier(1.0f).setDispatcherOutboundTargetRate(0).setDispatcherOutboundMaxRate(-1)}, | ||
| // maxRate should not be greater than OutboundTargetRate, multiplier is set. | ||
| new Object[] {false, new DarkClusterConfig().setMultiplier(1.0f).setDispatcherOutboundTargetRate(500).setDispatcherOutboundMaxRate(400)}, | ||
| // maxRate should not be greater than OutboundTargetRate | ||
| new Object[] {false, new DarkClusterConfig().setMultiplier(0.0f).setDispatcherOutboundTargetRate(500).setDispatcherOutboundMaxRate(400)} | ||
| }; | ||
| } | ||
|
|
||
| @Test | ||
| public void testDarkClustersConverterEmpty() | ||
| { | ||
| DarkClusterConfigMap configMap = new DarkClusterConfigMap(); | ||
| DarkClusterConfigMap resultConfigMap = DarkClustersConverter.toConfig(DarkClustersConverter.toProperties(configMap)); | ||
| Assert.assertEquals(resultConfigMap, configMap); | ||
| } | ||
|
|
||
| @Test(dataProvider = "provideKeys") | ||
| public void testDarkClustersConverter(boolean successExpected, DarkClusterConfig darkClusterConfig) | ||
| { | ||
| DarkClusterConfigMap configMap = new DarkClusterConfigMap(); | ||
| configMap.put(DARK_CLUSTER_KEY, darkClusterConfig); | ||
| try | ||
| { | ||
| Assert.assertEquals(DarkClustersConverter.toConfig(DarkClustersConverter.toProperties(configMap)), configMap); | ||
| } | ||
| catch (Exception | AssertionError e) | ||
| { | ||
| if (successExpected) | ||
| { | ||
| Assert.fail("expected success for conversion of: " + darkClusterConfig, e); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| @Test | ||
| public void testDarkClustersConverterDefaults() | ||
| { | ||
| DarkClusterConfigMap configMap = new DarkClusterConfigMap(); | ||
| DarkClusterConfig config = new DarkClusterConfig(); | ||
| configMap.put(DARK_CLUSTER_KEY, config); | ||
|
|
||
| DarkClusterConfig resultConfig = DarkClustersConverter.toConfig(DarkClustersConverter.toProperties(configMap)).get(DARK_CLUSTER_KEY); | ||
| Assert.assertEquals(resultConfig.getMultiplier(), DARK_CLUSTER_DEFAULT_MULTIPLIER); | ||
| Assert.assertEquals((int)resultConfig.getDispatcherOutboundTargetRate(), DARK_CLUSTER_DEFAULT_TARGET_RATE); | ||
| Assert.assertEquals((int)resultConfig.getDispatcherOutboundMaxRate(), DARK_CLUSTER_DEFAULT_MAX_RATE); | ||
| } | ||
|
|
||
| @Test | ||
| public void testEntriesInClusterConfig() | ||
| { | ||
| DarkClusterConfigMap configMap = new DarkClusterConfigMap(); | ||
| DarkClusterConfig config = new DarkClusterConfig() | ||
| .setDispatcherOutboundTargetRate(454) | ||
| .setDispatcherOutboundMaxRate(1234566); | ||
| config.data().put("blahblah", "random string"); | ||
|
|
||
| configMap.put(DARK_CLUSTER_KEY, config); | ||
|
|
||
| DarkClusterConfigMap expectedConfigMap = new DarkClusterConfigMap(); | ||
| DarkClusterConfig expectedConfig = new DarkClusterConfig(config.data()); | ||
| expectedConfig.setMultiplier(0); | ||
| expectedConfigMap.put(DARK_CLUSTER_KEY, expectedConfig); | ||
| DarkClusterConfigMap resultConfigMap = DarkClustersConverter.toConfig(DarkClustersConverter.toProperties(configMap)); | ||
| Assert.assertEquals(resultConfigMap, expectedConfigMap); | ||
| // random entries in the map are carried over because of the pass thru nature of dataMaps. | ||
| Assert.assertTrue(resultConfigMap.get(DARK_CLUSTER_KEY).data().containsKey("blahblah")); | ||
| // verify values are converted properly. | ||
| Assert.assertEquals(resultConfigMap.get(DARK_CLUSTER_KEY).getMultiplier(),0.0f, "unexpected multiplier"); | ||
| Assert.assertEquals((int)resultConfigMap.get(DARK_CLUSTER_KEY).getDispatcherOutboundTargetRate(), 454, "unexpected target rate"); | ||
| Assert.assertEquals((int)resultConfigMap.get(DARK_CLUSTER_KEY).getDispatcherOutboundMaxRate(), 1234566, "unexpected maxRate"); | ||
| } | ||
| } |
Oops, something went wrong.