1 package org.apache.helix.healthcheck;
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 import java.util.Date;
23 import java.util.HashMap;
24 import java.util.Map;
25
26 import org.apache.helix.ConfigAccessor;
27 import org.apache.helix.model.ConfigScope;
28 import org.apache.helix.model.builder.ConfigScopeBuilder;
29 import org.apache.helix.model.builder.HelixConfigScopeBuilder;
30 import org.apache.helix.HelixDataAccessor;
31 import org.apache.helix.HelixManager;
32 import org.apache.helix.ZNRecord;
33 import org.apache.helix.PropertyKey.Builder;
34 import org.apache.helix.integration.ZkStandAloneCMTestBaseWithPropertyServerCheck;
35 import org.apache.helix.model.ExternalView;
36 import org.apache.helix.model.HealthStat;
37 import org.apache.helix.model.HelixConfigScope;
38 import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
39 import org.apache.helix.model.InstanceConfig;
40 import org.apache.helix.model.InstanceConfig.InstanceConfigProperty;
41 import org.apache.helix.tools.ClusterStateVerifier;
42 import org.apache.helix.tools.ClusterStateVerifier.BestPossAndExtViewZkVerifier;
43 import org.testng.Assert;
44 import org.testng.annotations.Test;
45
46
47 public class TestAlertActionTriggering extends
48 ZkStandAloneCMTestBaseWithPropertyServerCheck
49 {
50 String _statName = "TestStat@DB=db1";
51 String _stat = "TestStat";
52 String metricName1 = "TestMetric1";
53 String metricName2 = "TestMetric2";
54 void setHealthData(int[] val1, int[] val2)
55 {
56 for (int i = 0; i < NODE_NR; i++)
57 {
58 String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
59 HelixManager manager = _startCMResultMap.get(instanceName)._manager;
60 ZNRecord record = new ZNRecord(_stat);
61 Map<String, String> valMap = new HashMap<String, String>();
62 valMap.put(metricName1, val1[i] + "");
63 valMap.put(metricName2, val2[i] + "");
64 record.setSimpleField("TimeStamp", new Date().getTime() + "");
65 record.setMapField(_statName, valMap);
66 HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
67 Builder keyBuilder = helixDataAccessor.keyBuilder();
68 helixDataAccessor
69 .setProperty(keyBuilder.healthReport( manager.getInstanceName(), record.getId()), new HealthStat(record));
70 }
71 try
72 {
73 Thread.sleep(1000);
74 }
75 catch (InterruptedException e)
76 {
77
78 e.printStackTrace();
79 }
80 }
81
82 void setHealthData2(int[] val1)
83 {
84 for (int i = 0; i < NODE_NR; i++)
85 {
86 String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
87 HelixManager manager = _startCMResultMap.get(instanceName)._manager;
88 ZNRecord record = new ZNRecord(_stat);
89 Map<String, String> valMap = new HashMap<String, String>();
90 valMap.put(metricName2, val1[i] + "");
91 record.setSimpleField("TimeStamp", new Date().getTime() + "");
92 record.setMapField("TestStat@DB=TestDB;Partition=TestDB_3", valMap);
93 HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
94 Builder keyBuilder = helixDataAccessor.keyBuilder();
95 helixDataAccessor
96 .setProperty(keyBuilder.healthReport( manager.getInstanceName(), record.getId()), new HealthStat(record));
97 }
98 try
99 {
100 Thread.sleep(1000);
101 }
102 catch (InterruptedException e)
103 {
104
105 e.printStackTrace();
106 }
107 }
108
109 @Test
110 public void testAlertActionDisableNode() throws InterruptedException
111 {
112
113 HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(CLUSTER_NAME).build();
114 Map<String, String> properties = new HashMap<String, String>();
115 properties.put("healthChange.enabled", "true");
116 _setupTool.getClusterManagementTool().setConfig(scope, properties);
117
118 String alertStr1 = "EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric1))CMP(GREATER)CON(20)ACTION(DISABLE_INSTANCE)";
119 String alertStr2 = "EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric2))CMP(GREATER)CON(120)ACTION(DISABLE_INSTANCE)";
120 String alertStr3 = "EXP(decay(1.0)(localhost_*.TestStat@DB=TestDB;Partition=*.TestMetric2))CMP(GREATER)CON(160)ACTION(DISABLE_PARTITION)";
121
122 _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr1);
123 _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr2);
124 _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, alertStr3);
125
126 int[] metrics1 = {10, 15, 22, 12, 16};
127 int[] metrics2 = {22, 115, 22, 163,16};
128 int[] metrics3 = {0, 0, 0, 0, 0};
129 setHealthData(metrics1, metrics2);
130
131 String controllerName = CONTROLLER_PREFIX + "_0";
132 HelixManager manager = _startCMResultMap.get(controllerName)._manager;
133
134 HealthStatsAggregationTask task = new HealthStatsAggregationTask(_startCMResultMap.get(controllerName)._manager);
135 task.run();
136 Thread.sleep(4000);
137 HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
138 Builder keyBuilder = helixDataAccessor.keyBuilder();
139
140 boolean result =
141 ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
142 CLUSTER_NAME));
143 Assert.assertTrue(result);
144
145 Builder kb = manager.getHelixDataAccessor().keyBuilder();
146 ExternalView externalView = manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB"));
147
148 String participant1 = "localhost_" + (START_PORT + 3);
149 String participant2 = "localhost_" + (START_PORT + 2);
150 ConfigAccessor configAccessor = manager.getConfigAccessor();
151
152 scope = new HelixConfigScopeBuilder(ConfigScopeProperty.PARTICIPANT)
153 .forCluster(manager.getClusterName())
154 .forParticipant(participant1)
155 .build();
156 String isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
157 Assert.assertFalse(Boolean.parseBoolean(isEnabled));
158
159
160 scope = new HelixConfigScopeBuilder(ConfigScopeProperty.PARTICIPANT)
161 .forCluster(manager.getClusterName())
162 .forParticipant(participant2)
163 .build();
164 isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
165 Assert.assertFalse(Boolean.parseBoolean(isEnabled));
166
167 for(String partitionName : externalView.getRecord().getMapFields().keySet())
168 {
169 for(String hostName : externalView.getRecord().getMapField(partitionName).keySet())
170 {
171 if(hostName.equals(participant1) || hostName.equals(participant2))
172 {
173 Assert.assertEquals(externalView.getRecord().getMapField(partitionName).get(hostName), "OFFLINE");
174 }
175 }
176 }
177
178
179 setHealthData(metrics3, metrics3);
180 task.run();
181 Thread.sleep(1000);
182
183 manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant2, true);
184 manager.getClusterManagmentTool().enableInstance(manager.getClusterName(), participant1, true);
185
186 result =
187 ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
188 CLUSTER_NAME));
189 Assert.assertTrue(result);
190
191
192 int[] metrics4 = {22, 115, 22, 16,163};
193 setHealthData2(metrics4);
194 task.run();
195
196
197 scope = new HelixConfigScopeBuilder(ConfigScopeProperty.PARTICIPANT)
198 .forCluster(manager.getClusterName())
199 .forParticipant(participant1)
200 .build();
201 isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
202 Assert.assertTrue(Boolean.parseBoolean(isEnabled));
203
204
205 scope = new HelixConfigScopeBuilder(ConfigScopeProperty.PARTICIPANT)
206 .forCluster(manager.getClusterName())
207 .forParticipant(participant2)
208 .build();
209 isEnabled = configAccessor.get(scope, "HELIX_ENABLED");
210 Assert.assertTrue(Boolean.parseBoolean(isEnabled));
211
212 result =
213 ClusterStateVerifier.verifyByZkCallback(new BestPossAndExtViewZkVerifier(ZK_ADDR,
214 CLUSTER_NAME));
215 Assert.assertTrue(result);
216 String participant3 = "localhost_" + (START_PORT + 4);
217 externalView = manager.getHelixDataAccessor().getProperty(kb.externalView("TestDB"));
218 Assert.assertTrue(externalView.getRecord().getMapField("TestDB_3").get(participant3).equalsIgnoreCase("OFFLINE"));
219
220 InstanceConfig nodeConfig =
221 helixDataAccessor.getProperty(keyBuilder.instanceConfig(participant3));
222 Assert.assertTrue(
223 nodeConfig.getRecord().getListField(InstanceConfigProperty.HELIX_DISABLED_PARTITION.toString())
224 .contains("TestDB_3"));
225
226 }
227 }