Line data Source code
1 : /* Copyright (c) 2016-2021, The Tor Project, Inc. */
2 : /* See LICENSE for licensing information */
3 :
4 : /* Unit tests for OOS handler */
5 :
6 : #define CONNECTION_PRIVATE
7 :
8 : #include "core/or/or.h"
9 : #include "app/config/config.h"
10 : #include "core/mainloop/connection.h"
11 : #include "core/or/connection_or.h"
12 : #include "feature/dircommon/directory.h"
13 : #include "core/mainloop/mainloop.h"
14 : #include "test/test.h"
15 :
16 : #include "feature/dircommon/dir_connection_st.h"
17 : #include "core/or/or_connection_st.h"
18 :
19 : static or_options_t mock_options;
20 :
21 : static void
22 1 : reset_options_mock(void)
23 : {
24 1 : memset(&mock_options, 0, sizeof(or_options_t));
25 1 : }
26 :
27 : static const or_options_t *
28 50 : mock_get_options(void)
29 : {
30 50 : return &mock_options;
31 : }
32 :
33 : static int moribund_calls = 0;
34 : static int moribund_conns = 0;
35 :
36 : static int
37 10 : mock_connection_count_moribund(void)
38 : {
39 10 : ++moribund_calls;
40 :
41 10 : return moribund_conns;
42 : }
43 :
44 : /*
45 : * For unit test purposes it's sufficient to tell that
46 : * kill_conn_list_for_oos() was called with an approximately
47 : * sane argument; it's just the thing we returned from the
48 : * mock for pick_oos_victims().
49 : */
50 :
51 : static int kill_conn_list_calls = 0;
52 : static int kill_conn_list_killed = 0;
53 :
54 : static void
55 6 : kill_conn_list_mock(smartlist_t *conns)
56 : {
57 6 : ++kill_conn_list_calls;
58 :
59 6 : tt_ptr_op(conns, OP_NE, NULL);
60 :
61 6 : kill_conn_list_killed += smartlist_len(conns);
62 :
63 6 : done:
64 6 : return;
65 : }
66 :
67 : static int pick_oos_mock_calls = 0;
68 : static int pick_oos_mock_fail = 0;
69 : static int pick_oos_mock_last_n = 0;
70 :
71 : static smartlist_t *
72 7 : pick_oos_victims_mock(int n)
73 : {
74 7 : smartlist_t *l = NULL;
75 7 : int i;
76 :
77 7 : ++pick_oos_mock_calls;
78 :
79 7 : tt_int_op(n, OP_GT, 0);
80 :
81 7 : if (!pick_oos_mock_fail) {
82 : /*
83 : * connection_check_oos() just passes the list onto
84 : * kill_conn_list_for_oos(); we don't need to simulate
85 : * its content for this mock, just its existence, but
86 : * we do need to check the parameter.
87 : */
88 6 : l = smartlist_new();
89 58 : for (i = 0; i < n; ++i) smartlist_add(l, NULL);
90 : } else {
91 : l = NULL;
92 : }
93 :
94 7 : pick_oos_mock_last_n = n;
95 :
96 7 : done:
97 7 : return l;
98 : }
99 :
100 : /** Unit test for the logic in connection_check_oos(), which is concerned
101 : * with comparing thresholds and connection counts to decide if an OOS has
102 : * occurred and if so, how many connections to try to kill, and then using
103 : * pick_oos_victims() and kill_conn_list_for_oos() to carry out its grim
104 : * duty.
105 : */
106 : static void
107 1 : test_oos_connection_check_oos(void *arg)
108 : {
109 1 : (void)arg;
110 :
111 : /* Set up mocks */
112 1 : reset_options_mock();
113 : /* OOS handling is only sensitive to these fields */
114 1 : mock_options.ConnLimit = 32;
115 1 : mock_options.ConnLimit_ = 64;
116 1 : mock_options.ConnLimit_high_thresh = 60;
117 1 : mock_options.ConnLimit_low_thresh = 50;
118 1 : MOCK(get_options, mock_get_options);
119 1 : moribund_calls = 0;
120 1 : moribund_conns = 0;
121 1 : MOCK(connection_count_moribund, mock_connection_count_moribund);
122 1 : kill_conn_list_calls = 0;
123 1 : kill_conn_list_killed = 0;
124 1 : MOCK(kill_conn_list_for_oos, kill_conn_list_mock);
125 1 : pick_oos_mock_calls = 0;
126 1 : pick_oos_mock_fail = 0;
127 1 : MOCK(pick_oos_victims, pick_oos_victims_mock);
128 :
129 : /* No OOS case */
130 1 : connection_check_oos(50, 0);
131 1 : tt_int_op(moribund_calls, OP_EQ, 0);
132 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 0);
133 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 0);
134 :
135 : /* OOS from socket count, nothing moribund */
136 1 : connection_check_oos(62, 0);
137 1 : tt_int_op(moribund_calls, OP_EQ, 1);
138 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 1);
139 : /* 12 == 62 - ConnLimit_low_thresh */
140 1 : tt_int_op(pick_oos_mock_last_n, OP_EQ, 12);
141 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 1);
142 1 : tt_int_op(kill_conn_list_killed, OP_EQ, 12);
143 :
144 : /* OOS from socket count, some are moribund */
145 1 : kill_conn_list_killed = 0;
146 1 : moribund_conns = 5;
147 1 : connection_check_oos(62, 0);
148 1 : tt_int_op(moribund_calls, OP_EQ, 2);
149 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 2);
150 : /* 7 == 62 - ConnLimit_low_thresh - moribund_conns */
151 1 : tt_int_op(pick_oos_mock_last_n, OP_EQ, 7);
152 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 2);
153 1 : tt_int_op(kill_conn_list_killed, OP_EQ, 7);
154 :
155 : /* OOS from socket count, but pick fails */
156 1 : kill_conn_list_killed = 0;
157 1 : moribund_conns = 0;
158 1 : pick_oos_mock_fail = 1;
159 1 : connection_check_oos(62, 0);
160 1 : tt_int_op(moribund_calls, OP_EQ, 3);
161 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
162 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 2);
163 1 : tt_int_op(kill_conn_list_killed, OP_EQ, 0);
164 1 : pick_oos_mock_fail = 0;
165 :
166 : /*
167 : * OOS from socket count with so many moribund conns
168 : * we have none to kill.
169 : */
170 1 : kill_conn_list_killed = 0;
171 1 : moribund_conns = 15;
172 1 : connection_check_oos(62, 0);
173 1 : tt_int_op(moribund_calls, OP_EQ, 4);
174 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 3);
175 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 2);
176 :
177 : /*
178 : * OOS from socket exhaustion; OOS handler will try to
179 : * kill 1/10 (5) of the connections.
180 : */
181 1 : kill_conn_list_killed = 0;
182 1 : moribund_conns = 0;
183 1 : connection_check_oos(50, 1);
184 1 : tt_int_op(moribund_calls, OP_EQ, 5);
185 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 4);
186 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 3);
187 1 : tt_int_op(kill_conn_list_killed, OP_EQ, 5);
188 :
189 : /* OOS from socket exhaustion with moribund conns */
190 1 : kill_conn_list_killed = 0;
191 1 : moribund_conns = 2;
192 1 : connection_check_oos(50, 1);
193 1 : tt_int_op(moribund_calls, OP_EQ, 6);
194 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
195 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 4);
196 1 : tt_int_op(kill_conn_list_killed, OP_EQ, 3);
197 :
198 : /* OOS from socket exhaustion with many moribund conns */
199 1 : kill_conn_list_killed = 0;
200 1 : moribund_conns = 7;
201 1 : connection_check_oos(50, 1);
202 1 : tt_int_op(moribund_calls, OP_EQ, 7);
203 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 5);
204 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 4);
205 :
206 : /* OOS with both socket exhaustion and above-threshold */
207 1 : kill_conn_list_killed = 0;
208 1 : moribund_conns = 0;
209 1 : connection_check_oos(62, 1);
210 1 : tt_int_op(moribund_calls, OP_EQ, 8);
211 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 6);
212 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 5);
213 1 : tt_int_op(kill_conn_list_killed, OP_EQ, 12);
214 :
215 : /*
216 : * OOS with both socket exhaustion and above-threshold with some
217 : * moribund conns
218 : */
219 1 : kill_conn_list_killed = 0;
220 1 : moribund_conns = 5;
221 1 : connection_check_oos(62, 1);
222 1 : tt_int_op(moribund_calls, OP_EQ, 9);
223 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
224 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 6);
225 1 : tt_int_op(kill_conn_list_killed, OP_EQ, 7);
226 :
227 : /*
228 : * OOS with both socket exhaustion and above-threshold with many
229 : * moribund conns
230 : */
231 1 : kill_conn_list_killed = 0;
232 1 : moribund_conns = 15;
233 1 : connection_check_oos(62, 1);
234 1 : tt_int_op(moribund_calls, OP_EQ, 10);
235 1 : tt_int_op(pick_oos_mock_calls, OP_EQ, 7);
236 1 : tt_int_op(kill_conn_list_calls, OP_EQ, 6);
237 :
238 1 : done:
239 :
240 1 : UNMOCK(pick_oos_victims);
241 1 : UNMOCK(kill_conn_list_for_oos);
242 1 : UNMOCK(connection_count_moribund);
243 1 : UNMOCK(get_options);
244 :
245 1 : return;
246 : }
247 :
248 : static int cfe_calls = 0;
249 :
250 : static void
251 1 : close_for_error_mock(or_connection_t *orconn, int flush)
252 : {
253 1 : (void)flush;
254 :
255 1 : tt_ptr_op(orconn, OP_NE, NULL);
256 1 : ++cfe_calls;
257 :
258 1 : done:
259 1 : return;
260 : }
261 :
262 : static int mark_calls = 0;
263 :
264 : static void
265 1 : mark_for_close_oos_mock(connection_t *conn,
266 : int line, const char *file)
267 : {
268 1 : (void)line;
269 1 : (void)file;
270 :
271 1 : tt_ptr_op(conn, OP_NE, NULL);
272 1 : ++mark_calls;
273 :
274 1 : done:
275 1 : return;
276 : }
277 :
278 : static void
279 1 : test_oos_kill_conn_list(void *arg)
280 : {
281 1 : connection_t *c1, *c2;
282 1 : or_connection_t *or_c1 = NULL;
283 1 : dir_connection_t *dir_c2 = NULL;
284 1 : smartlist_t *l = NULL;
285 1 : (void)arg;
286 :
287 : /* Set up mocks */
288 1 : mark_calls = 0;
289 1 : MOCK(connection_mark_for_close_internal_, mark_for_close_oos_mock);
290 1 : cfe_calls = 0;
291 1 : MOCK(connection_or_close_for_error, close_for_error_mock);
292 :
293 : /* Make fake conns */
294 1 : or_c1 = tor_malloc_zero(sizeof(*or_c1));
295 1 : or_c1->base_.magic = OR_CONNECTION_MAGIC;
296 1 : or_c1->base_.type = CONN_TYPE_OR;
297 1 : c1 = TO_CONN(or_c1);
298 1 : dir_c2 = tor_malloc_zero(sizeof(*dir_c2));
299 1 : dir_c2->base_.magic = DIR_CONNECTION_MAGIC;
300 1 : dir_c2->base_.type = CONN_TYPE_DIR;
301 1 : dir_c2->base_.state = DIR_CONN_STATE_MIN_;
302 1 : dir_c2->base_.purpose = DIR_PURPOSE_MIN_;
303 1 : c2 = TO_CONN(dir_c2);
304 :
305 1 : tt_ptr_op(c1, OP_NE, NULL);
306 1 : tt_ptr_op(c2, OP_NE, NULL);
307 :
308 : /* Make list */
309 1 : l = smartlist_new();
310 1 : smartlist_add(l, c1);
311 1 : smartlist_add(l, c2);
312 :
313 : /* Run kill_conn_list_for_oos() */
314 1 : kill_conn_list_for_oos(l);
315 :
316 : /* Check call counters */
317 1 : tt_int_op(mark_calls, OP_EQ, 1);
318 1 : tt_int_op(cfe_calls, OP_EQ, 1);
319 :
320 1 : done:
321 :
322 1 : UNMOCK(connection_or_close_for_error);
323 1 : UNMOCK(connection_mark_for_close_internal_);
324 :
325 1 : if (l) smartlist_free(l);
326 1 : tor_free(or_c1);
327 1 : tor_free(dir_c2);
328 :
329 1 : return;
330 : }
331 :
332 : static smartlist_t *conns_for_mock = NULL;
333 :
334 : static smartlist_t *
335 4 : get_conns_mock(void)
336 : {
337 4 : return conns_for_mock;
338 : }
339 :
340 : /*
341 : * For this mock, we pretend all conns have either zero or one circuits,
342 : * depending on if this appears on the list of things to say have a circuit.
343 : */
344 :
345 : static smartlist_t *conns_with_circs = NULL;
346 :
347 : static int
348 4 : get_num_circuits_mock(or_connection_t *conn)
349 : {
350 4 : int circs = 0;
351 :
352 4 : tt_ptr_op(conn, OP_NE, NULL);
353 :
354 8 : if (conns_with_circs &&
355 4 : smartlist_contains(conns_with_circs, TO_CONN(conn))) {
356 2 : circs = 1;
357 : }
358 :
359 2 : done:
360 4 : return circs;
361 : }
362 :
363 : static void
364 1 : test_oos_pick_oos_victims(void *arg)
365 : {
366 1 : (void)arg;
367 1 : or_connection_t *ortmp;
368 1 : dir_connection_t *dirtmp;
369 1 : smartlist_t *picked;
370 :
371 : /* Set up mocks */
372 1 : conns_for_mock = smartlist_new();
373 1 : MOCK(get_connection_array, get_conns_mock);
374 1 : conns_with_circs = smartlist_new();
375 1 : MOCK(connection_or_get_num_circuits, get_num_circuits_mock);
376 :
377 : /* Make some fake connections */
378 1 : ortmp = tor_malloc_zero(sizeof(*ortmp));
379 1 : ortmp->base_.magic = OR_CONNECTION_MAGIC;
380 1 : ortmp->base_.type = CONN_TYPE_OR;
381 1 : smartlist_add(conns_for_mock, TO_CONN(ortmp));
382 : /* We'll pretend this one has a circuit too */
383 1 : smartlist_add(conns_with_circs, TO_CONN(ortmp));
384 : /* Next one */
385 1 : ortmp = tor_malloc_zero(sizeof(*ortmp));
386 1 : ortmp->base_.magic = OR_CONNECTION_MAGIC;
387 1 : ortmp->base_.type = CONN_TYPE_OR;
388 1 : smartlist_add(conns_for_mock, TO_CONN(ortmp));
389 : /* Next one is moribund */
390 1 : ortmp = tor_malloc_zero(sizeof(*ortmp));
391 1 : ortmp->base_.magic = OR_CONNECTION_MAGIC;
392 1 : ortmp->base_.type = CONN_TYPE_OR;
393 1 : ortmp->base_.marked_for_close = 1;
394 1 : smartlist_add(conns_for_mock, TO_CONN(ortmp));
395 : /* Last one isn't an orconn */
396 1 : dirtmp = tor_malloc_zero(sizeof(*dirtmp));
397 1 : dirtmp->base_.magic = DIR_CONNECTION_MAGIC;
398 1 : dirtmp->base_.type = CONN_TYPE_DIR;
399 1 : smartlist_add(conns_for_mock, TO_CONN(dirtmp));
400 :
401 : /* Try picking one */
402 1 : picked = pick_oos_victims(1);
403 : /* It should be the one with circuits */
404 1 : tt_ptr_op(picked, OP_NE, NULL);
405 1 : tt_int_op(smartlist_len(picked), OP_EQ, 1);
406 1 : tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
407 1 : smartlist_free(picked);
408 :
409 : /* Try picking none */
410 1 : picked = pick_oos_victims(0);
411 : /* We should get an empty list */
412 1 : tt_ptr_op(picked, OP_NE, NULL);
413 1 : tt_int_op(smartlist_len(picked), OP_EQ, 0);
414 1 : smartlist_free(picked);
415 :
416 : /* Try picking two */
417 1 : picked = pick_oos_victims(2);
418 : /* We should get both active orconns */
419 1 : tt_ptr_op(picked, OP_NE, NULL);
420 1 : tt_int_op(smartlist_len(picked), OP_EQ, 2);
421 1 : tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
422 1 : tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
423 1 : smartlist_free(picked);
424 :
425 : /* Try picking three - only two are eligible */
426 1 : picked = pick_oos_victims(3);
427 1 : tt_int_op(smartlist_len(picked), OP_EQ, 2);
428 1 : tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 0)));
429 1 : tt_assert(smartlist_contains(picked, smartlist_get(conns_for_mock, 1)));
430 1 : smartlist_free(picked);
431 :
432 1 : done:
433 :
434 : /* Free leftover stuff */
435 1 : if (conns_with_circs) {
436 1 : smartlist_free(conns_with_circs);
437 1 : conns_with_circs = NULL;
438 : }
439 :
440 1 : UNMOCK(connection_or_get_num_circuits);
441 :
442 1 : if (conns_for_mock) {
443 5 : SMARTLIST_FOREACH(conns_for_mock, connection_t *, c, tor_free(c));
444 1 : smartlist_free(conns_for_mock);
445 1 : conns_for_mock = NULL;
446 : }
447 :
448 1 : UNMOCK(get_connection_array);
449 :
450 1 : return;
451 : }
452 :
453 : struct testcase_t oos_tests[] = {
454 : { "connection_check_oos", test_oos_connection_check_oos,
455 : TT_FORK, NULL, NULL },
456 : { "kill_conn_list", test_oos_kill_conn_list, TT_FORK, NULL, NULL },
457 : { "pick_oos_victims", test_oos_pick_oos_victims, TT_FORK, NULL, NULL },
458 : END_OF_TESTCASES
459 : };
|
