View Javadoc

1   package org.apache.helix.healthcheck;
2   
3   /*
4    * Licensed to the Apache Software Foundation (ASF) under one
5    * or more contributor license agreements.  See the NOTICE file
6    * distributed with this work for additional information
7    * regarding copyright ownership.  The ASF licenses this file
8    * to you under the Apache License, Version 2.0 (the
9    * "License"); you may not use this file except in compliance
10   * with the License.  You may obtain a copy of the License at
11   *
12   *   http://www.apache.org/licenses/LICENSE-2.0
13   *
14   * Unless required by applicable law or agreed to in writing,
15   * software distributed under the License is distributed on an
16   * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17   * KIND, either express or implied.  See the License for the
18   * specific language governing permissions and limitations
19   * under the License.
20   */
21  
22  import java.util.Date;
23  import java.util.HashMap;
24  import java.util.Map;
25  import java.util.TreeMap;
26  
27  import org.apache.helix.model.ConfigScope;
28  import org.apache.helix.model.HelixConfigScope.ConfigScopeProperty;
29  import org.apache.helix.model.builder.ConfigScopeBuilder;
30  import org.apache.helix.model.builder.HelixConfigScopeBuilder;
31  import org.apache.helix.HelixDataAccessor;
32  import org.apache.helix.HelixManager;
33  import org.apache.helix.HelixProperty;
34  import org.apache.helix.ZNRecord;
35  import org.apache.helix.PropertyKey.Builder;
36  import org.apache.helix.integration.ZkStandAloneCMTestBaseWithPropertyServerCheck;
37  import org.apache.helix.model.AlertHistory;
38  import org.apache.helix.model.HealthStat;
39  import org.apache.helix.model.HelixConfigScope;
40  import org.testng.Assert;
41  import org.testng.annotations.Test;
42  
43  
44  /**
45   *
46   * setup a storage cluster and start a zk-based cluster controller in stand-alone mode
47   * start 5 dummy participants verify the current states at end
48   */
49  
50  public class TestAlertFireHistory extends ZkStandAloneCMTestBaseWithPropertyServerCheck
51  {
52    String _statName = "TestStat@DB=db1";
53    String _stat = "TestStat";
54    String metricName1 = "TestMetric1";
55    String metricName2 = "TestMetric2";
56    
57    String _alertStr1 = "EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric1))CMP(GREATER)CON(20)";
58    String _alertStr2 = "EXP(decay(1.0)(localhost_*.TestStat@DB=db1.TestMetric2))CMP(GREATER)CON(100)";
59    
60    void setHealthData(int[] val1, int[] val2)
61    { 
62      for (int i = 0; i < NODE_NR; i++)
63      {
64        String instanceName = PARTICIPANT_PREFIX + "_" + (START_PORT + i);
65        HelixManager manager = _startCMResultMap.get(instanceName)._manager;
66        ZNRecord record = new ZNRecord(_stat);
67        Map<String, String> valMap = new HashMap<String, String>();
68        valMap.put(metricName1, val1[i] + "");
69        valMap.put(metricName2, val2[i] + "");
70        record.setSimpleField("TimeStamp", new Date().getTime() + "");
71        record.setMapField(_statName, valMap);
72        HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
73        Builder keyBuilder = helixDataAccessor.keyBuilder();
74        helixDataAccessor
75          .setProperty(keyBuilder.healthReport( manager.getInstanceName(), record.getId()), new HealthStat(record));  
76      }
77      try
78      {
79        Thread.sleep(1000);
80      }
81      catch (InterruptedException e)
82      {
83        // TODO Auto-generated catch block
84        e.printStackTrace();
85      }
86    }
87    
88    @Test
89    public void TestAlertDisable() throws InterruptedException
90    {
91  
92      int[] metrics1 = {10, 15, 22, 24, 16};
93      int[] metrics2 = {22, 115, 22, 141,16};
94      setHealthData(metrics1, metrics2);
95      
96      String controllerName = CONTROLLER_PREFIX + "_0";
97      HelixManager manager = _startCMResultMap.get(controllerName)._manager;
98      manager.startTimerTasks();
99      
100     _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, _alertStr1);
101     _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, _alertStr2);
102     
103     // ConfigScope scope = new ConfigScopeBuilder().forCluster(CLUSTER_NAME).build();
104     HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER)
105                                         .forCluster(CLUSTER_NAME)
106                                         .build();
107     Map<String, String> properties = new HashMap<String, String>();
108     properties.put("healthChange.enabled", "false");
109     _setupTool.getClusterManagementTool().setConfig(scope, properties);
110     
111     HealthStatsAggregationTask task = new HealthStatsAggregationTask(_startCMResultMap.get(controllerName)._manager);
112     task.run();
113     Thread.sleep(100);
114     HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
115     Builder keyBuilder = helixDataAccessor.keyBuilder();
116 
117     AlertHistory history = manager.getHelixDataAccessor().getProperty(keyBuilder.alertHistory());
118     // 
119     Assert.assertEquals(history, null);
120     
121     properties.put("healthChange.enabled", "true");
122     _setupTool.getClusterManagementTool().setConfig(scope, properties);
123     
124     task.run();
125     Thread.sleep(100);
126     
127     history = manager.getHelixDataAccessor().getProperty(keyBuilder.alertHistory());
128     // 
129     Assert.assertNotNull(history);
130     Assert.assertEquals(history.getRecord().getMapFields().size(), 1);
131   }
132   
133   @Test
134   public void TestAlertHistory() throws InterruptedException
135   {
136     int[] metrics1 = {10, 15, 22, 24, 16};
137     int[] metrics2 = {22, 115, 22, 141,16};
138     setHealthData(metrics1, metrics2);
139 
140     String controllerName = CONTROLLER_PREFIX + "_0";
141     HelixManager manager = _startCMResultMap.get(controllerName)._manager;
142     manager.stopTimerTasks();
143     
144     _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, _alertStr1);
145     _setupTool.getClusterManagementTool().addAlert(CLUSTER_NAME, _alertStr2);
146 
147     int historySize = 0;
148     HelixDataAccessor helixDataAccessor = manager.getHelixDataAccessor();
149     Builder keyBuilder = helixDataAccessor.keyBuilder();
150     HelixProperty property = helixDataAccessor.getProperty(keyBuilder.alertHistory());
151     ZNRecord history = null;
152     if(property != null)
153     {
154       history = property.getRecord();
155       historySize = property.getRecord().getMapFields().size();
156     }
157     
158     HealthStatsAggregationTask task = new HealthStatsAggregationTask(_startCMResultMap.get(controllerName)._manager);
159     task.run();
160     Thread.sleep(100);
161     
162     
163     history = helixDataAccessor.getProperty(keyBuilder.alertHistory()).getRecord();
164     // 
165     Assert.assertEquals(history.getMapFields().size(), 1 + historySize);
166     TreeMap<String, Map<String, String>> recordMap = new TreeMap<String, Map<String, String>>();
167     recordMap.putAll( history.getMapFields());
168     Map<String, String> lastRecord = recordMap.firstEntry().getValue();
169     Assert.assertTrue(lastRecord.size() == 4);
170     Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
171     Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
172     Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
173     Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
174     
175     setHealthData(metrics1, metrics2);
176     task.run();
177     Thread.sleep(100);
178     history = helixDataAccessor.getProperty(keyBuilder.alertHistory()).getRecord();
179     // no change
180     Assert.assertEquals(history.getMapFields().size(), 1 + historySize);
181     recordMap = new TreeMap<String, Map<String, String>>();
182     recordMap.putAll( history.getMapFields());
183     lastRecord = recordMap.firstEntry().getValue();
184     Assert.assertTrue(lastRecord.size() == 4);
185     Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
186     Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
187     Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
188     Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
189     
190     int [] metrics3 = {21, 44, 22, 14, 16};
191     int [] metrics4 = {122, 115, 222, 41,16};
192     setHealthData(metrics3, metrics4);
193     task.run();
194     Thread.sleep(100);
195     history = helixDataAccessor.getProperty(keyBuilder.alertHistory()).getRecord();
196     // new delta should be recorded
197     Assert.assertEquals(history.getMapFields().size(), 2 + historySize);
198     recordMap = new TreeMap<String, Map<String, String>>();
199     recordMap.putAll( history.getMapFields());
200     lastRecord = recordMap.lastEntry().getValue();
201     Assert.assertEquals(lastRecord.size(), 6);
202     Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
203     Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
204     Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
205     Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
206     Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
207     Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
208     
209     int [] metrics5 = {0, 0, 0, 0, 0};
210     int [] metrics6 = {0, 0, 0, 0,0};
211     setHealthData(metrics5, metrics6);
212     task.run();
213     
214     for (int i = 0; i < 10; i++) 
215     {
216         Thread.sleep(500);
217         history = helixDataAccessor.getProperty(keyBuilder.alertHistory()).getRecord();
218         recordMap = new TreeMap<String, Map<String, String>>();
219         recordMap.putAll( history.getMapFields());
220         lastRecord = recordMap.lastEntry().getValue();
221 
222         if (history.getMapFields().size() == 3 + historySize && lastRecord.size() == 6) {
223         	break;
224         }
225     }
226     
227     // reset everything
228     Assert.assertEquals(history.getMapFields().size(), 3 + historySize, 
229     		"expect history-map-field size is " + (3 + historySize) + ", but was " + history);
230     Assert.assertTrue(lastRecord.size() == 6, "expect last-record size is 6, but was " + lastRecord);
231     
232     Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
233     Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
234     Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
235     Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
236     Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
237     Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
238     
239     // Size of the history should be 30
240     for(int i = 0;i < 27; i++)
241     {
242       int x = i % 2;
243       int y = (i+1) % 2;
244       int[] metricsx = {19 + 3*x, 19 + 3*y, 19 + 4*x, 18+4*y, 17+5*y};
245       int[] metricsy = {99 + 3*x, 99 + 3*y, 98 + 4*x, 98+4*y, 97+5*y};
246       
247       setHealthData(metricsx, metricsy);
248       task.run();
249       Thread.sleep(100);
250       history = helixDataAccessor.getProperty(keyBuilder.alertHistory()).getRecord();
251       
252       Assert.assertEquals(history.getMapFields().size(), Math.min(3 + i + 1 + historySize, 30));
253       recordMap = new TreeMap<String, Map<String, String>>();
254       recordMap.putAll( history.getMapFields());
255       lastRecord = recordMap.lastEntry().getValue();
256       if(i == 0)
257       {
258         Assert.assertTrue(lastRecord.size() == 6);
259         Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
260         Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
261         Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
262         Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
263         Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
264         Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
265       }
266       else
267       {
268         System.out.println(lastRecord.size());
269         Assert.assertEquals(lastRecord.size() , 10);
270         if(x == 0)
271         {
272           Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
273           Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
274           Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
275           Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
276           Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
277           Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
278           Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
279           Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
280           Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
281           Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
282         }
283         else
284         {
285           Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
286           Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
287           Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
288           Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
289           Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
290           Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
291           Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
292           Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
293           Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
294           Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
295         }
296       }
297     }
298     // size limit is 30
299     for(int i = 0;i < 10; i++)
300     {
301       int x = i % 2;
302       int y = (i+1) % 2;
303       int[] metricsx = {19 + 3*x, 19 + 3*y, 19 + 4*x, 18+4*y, 17+5*y};
304       int[] metricsy = {99 + 3*x, 99 + 3*y, 98 + 4*x, 98+4*y, 97+5*y};
305       
306       setHealthData(metricsx, metricsy);
307       task.run();
308       for (int j = 0; j < 10; j++) {
309           Thread.sleep(100);
310           history = helixDataAccessor.getProperty(keyBuilder.alertHistory()).getRecord();
311           recordMap = new TreeMap<String, Map<String, String>>();
312           recordMap.putAll( history.getMapFields());
313           lastRecord = recordMap.lastEntry().getValue();
314           
315           if (history.getMapFields().size() == 30 && lastRecord.size() == 10)
316         	  break;
317       }
318       Assert.assertEquals(history.getMapFields().size(), 30, "expect history.map-field size is 30, but was " + history);
319       Assert.assertEquals(lastRecord.size() , 10, "expect last-record size is 10, but was " + lastRecord);
320       
321       if(x == 0)
322       {
323         Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
324         Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
325         Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
326         Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
327         Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
328         Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
329         Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
330         Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
331         Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
332         Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
333       }
334       else
335       {
336         Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
337         Assert.assertTrue(lastRecord.get("(localhost_12922.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
338         Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
339         Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("ON"));
340         Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
341         Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
342         Assert.assertTrue(lastRecord.get("(localhost_12920.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
343         Assert.assertTrue(lastRecord.get("(localhost_12921.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("OFF"));
344         Assert.assertTrue(lastRecord.get("(localhost_12918.TestStat@DB#db1.TestMetric2)GREATER(100)").equals("ON"));
345         Assert.assertTrue(lastRecord.get("(localhost_12919.TestStat@DB#db1.TestMetric1)GREATER(20)").equals("OFF"));
346       }
347     }
348     
349   }
350 
351 }