Tor  0.4.7.0-alpha-dev
tinytest.c
1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2  *
3  * Redistribution and use in source and binary forms, with or without
4  * modification, are permitted provided that the following conditions
5  * are met:
6  * 1. Redistributions of source code must retain the above copyright
7  * notice, this list of conditions and the following disclaimer.
8  * 2. Redistributions in binary form must reproduce the above copyright
9  * notice, this list of conditions and the following disclaimer in the
10  * documentation and/or other materials provided with the distribution.
11  * 3. The name of the author may not be used to endorse or promote products
12  * derived from this software without specific prior written permission.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24  */
25 #ifdef TINYTEST_LOCAL
26 #include "tinytest_local.h"
27 #endif
28 #define TINYTEST_POSTFORK
29 
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <assert.h>
34 
35 #ifndef NO_FORKING
36 
37 #ifdef _WIN32
38 #include <windows.h>
39 #else
40 #include <sys/types.h>
41 #include <sys/wait.h>
42 #include <unistd.h>
43 #endif
44 
45 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
46 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
47  __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
48 /* Workaround for a stupid bug in OSX 10.6 */
49 #define FORK_BREAKS_GCOV
50 #include <vproc.h>
51 #endif
52 #endif
53 
54 #endif /* !NO_FORKING */
55 
56 #ifndef __GNUC__
57 #define __attribute__(x)
58 #endif
59 
60 #include "tinytest.h"
61 #include "tinytest_macros.h"
62 
63 #define LONGEST_TEST_NAME 16384
64 
65 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
66 static int n_ok = 0; /**< Number of tests that have passed */
67 static int n_bad = 0; /**< Number of tests that have failed. */
68 static int n_skipped = 0; /**< Number of tests that have been skipped. */
69 
70 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
71 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
72 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
73 static const char *verbosity_flag = "";
74 
75 static const struct testlist_alias_t *cfg_aliases=NULL;
76 
77 enum outcome { SKIP=2, OK=1, FAIL=0 };
78 static enum outcome cur_test_outcome = 0;
79 /** prefix of the current test group */
80 static const char *cur_test_prefix = NULL;
81 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
82 static const char *cur_test_name = NULL;
83 
84 #ifdef _WIN32
85 /* Copy of argv[0] for win32. */
86 static char commandname[MAX_PATH+1];
87 #endif
88 
89 static void usage(struct testgroup_t *groups, int list_groups)
90  __attribute__((noreturn));
91 static int process_test_option(struct testgroup_t *groups, const char *test);
92 
93 static enum outcome
94 testcase_run_bare_(const struct testcase_t *testcase)
95 {
96  void *env = NULL;
97  int outcome;
98  if (testcase->setup) {
99  env = testcase->setup->setup_fn(testcase);
100  if (!env)
101  return FAIL;
102  else if (env == (void*)TT_SKIP)
103  return SKIP;
104  }
105 
106  cur_test_outcome = OK;
107  testcase->fn(env);
108  outcome = cur_test_outcome;
109 
110  if (testcase->setup) {
111  if (testcase->setup->cleanup_fn(testcase, env) == 0)
112  outcome = FAIL;
113  }
114 
115  return outcome;
116 }
117 
118 #define MAGIC_EXITCODE 42
119 
120 #ifndef NO_FORKING
121 
122 #ifdef TINYTEST_POSTFORK
123 void tinytest_prefork(void);
124 void tinytest_postfork(void);
125 #else
126 static void tinytest_prefork(void) { }
127 static void tinytest_postfork(void) { }
128 #endif
129 
130 static enum outcome
131 testcase_run_forked_(const struct testgroup_t *group,
132  const struct testcase_t *testcase)
133 {
134 #ifdef _WIN32
135  /* Fork? On Win32? How primitive! We'll do what the smart kids do:
136  we'll invoke our own exe (whose name we recall from the command
137  line) with a command line that tells it to run just the test we
138  want, and this time without forking.
139 
140  (No, threads aren't an option. The whole point of forking is to
141  share no state between tests.)
142  */
143  int ok;
144  char buffer[LONGEST_TEST_NAME+256];
145  STARTUPINFOA si;
146  PROCESS_INFORMATION info;
147  DWORD exitcode;
148 
149  if (!in_tinytest_main) {
150  printf("\nERROR. On Windows, testcase_run_forked_ must be"
151  " called from within tinytest_main.\n");
152  abort();
153  }
154  if (opt_verbosity>0)
155  printf("[forking] ");
156 
157  snprintf(buffer, sizeof(buffer), "\"%s\" --RUNNING-FORKED %s %s%s",
158  commandname, verbosity_flag, group->prefix, testcase->name);
159 
160  memset(&si, 0, sizeof(si));
161  memset(&info, 0, sizeof(info));
162  si.cb = sizeof(si);
163 
164  ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
165  0, NULL, NULL, &si, &info);
166  if (!ok) {
167  printf("CreateProcess failed!\n");
168  return 0;
169  }
170  WaitForSingleObject(info.hProcess, INFINITE);
171  GetExitCodeProcess(info.hProcess, &exitcode);
172  CloseHandle(info.hProcess);
173  CloseHandle(info.hThread);
174  if (exitcode == 0)
175  return OK;
176  else if (exitcode == MAGIC_EXITCODE)
177  return SKIP;
178  else
179  return FAIL;
180 #else
181  int outcome_pipe[2];
182  pid_t pid;
183  (void)group;
184 
185  if (pipe(outcome_pipe))
186  perror("opening pipe");
187 
188  if (opt_verbosity>0)
189  printf("[forking] ");
190  tinytest_prefork();
191  pid = fork();
192 #ifdef FORK_BREAKS_GCOV
193  vproc_transaction_begin(0);
194 #endif
195  tinytest_postfork();
196  if (!pid) {
197  /* child. */
198  int test_r, write_r;
199  char b[1];
200  close(outcome_pipe[0]);
201  test_r = testcase_run_bare_(testcase);
202  assert(0<=(int)test_r && (int)test_r<=2);
203  b[0] = "NYS"[test_r];
204  write_r = (int)write(outcome_pipe[1], b, 1);
205  if (write_r != 1) {
206  perror("write outcome to pipe");
207  exit(1);
208  }
209  exit(0);
210  return FAIL; /* unreachable */
211  } else {
212  /* parent */
213  int status, r;
214  char b[1];
215  /* Close this now, so that if the other side closes it,
216  * our read fails. */
217  close(outcome_pipe[1]);
218  r = (int)read(outcome_pipe[0], b, 1);
219  if (r == 0) {
220  printf("[Lost connection!] ");
221  return FAIL;
222  } else if (r != 1) {
223  perror("read outcome from pipe");
224  }
225  r = waitpid(pid, &status, 0);
226  close(outcome_pipe[0]);
227  if (r == -1) {
228  perror("waitpid");
229  return FAIL;
230  }
231  if (! WIFEXITED(status) || WEXITSTATUS(status) != 0) {
232  printf("[did not exit cleanly.]");
233  return FAIL;
234  }
235  return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
236  }
237 #endif
238 }
239 
240 #endif /* !NO_FORKING */
241 
242 int
243 testcase_run_one(const struct testgroup_t *group,
244  const struct testcase_t *testcase)
245 {
246  enum outcome outcome;
247 
248  if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
249  if (opt_verbosity>0)
250  printf("%s%s: %s\n",
251  group->prefix, testcase->name,
252  (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
253  ++n_skipped;
254  return SKIP;
255  }
256 
257  if (opt_verbosity>0 && !opt_forked) {
258  printf("%s%s: ", group->prefix, testcase->name);
259  } else {
260  if (opt_verbosity==0) printf(".");
261  cur_test_prefix = group->prefix;
262  cur_test_name = testcase->name;
263  }
264 
265 #ifndef NO_FORKING
266  if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
267  outcome = testcase_run_forked_(group, testcase);
268  } else {
269 #else
270  {
271 #endif
272  outcome = testcase_run_bare_(testcase);
273  }
274 
275  if (outcome == OK) {
276  ++n_ok;
277  if (opt_verbosity>0 && !opt_forked)
278  puts(opt_verbosity==1?"OK":"");
279  } else if (outcome == SKIP) {
280  ++n_skipped;
281  if (opt_verbosity>0 && !opt_forked)
282  puts("SKIPPED");
283  } else {
284  ++n_bad;
285  if (!opt_forked)
286  printf("\n [%s FAILED]\n", testcase->name);
287  }
288 
289  if (opt_forked) {
290  exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
291  return 1; /* unreachable */
292  } else {
293  return (int)outcome;
294  }
295 }
296 
297 int
298 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
299 {
300  int i, j;
301  size_t length = LONGEST_TEST_NAME;
302  char fullname[LONGEST_TEST_NAME];
303  int found=0;
304  if (strstr(arg, ".."))
305  length = strstr(arg,"..")-arg;
306  for (i=0; groups[i].prefix; ++i) {
307  for (j=0; groups[i].cases[j].name; ++j) {
308  struct testcase_t *testcase = &groups[i].cases[j];
309  snprintf(fullname, sizeof(fullname), "%s%s",
310  groups[i].prefix, testcase->name);
311  if (!flag) { /* Hack! */
312  printf(" %s", fullname);
313  if (testcase->flags & TT_OFF_BY_DEFAULT)
314  puts(" (Off by default)");
315  else if (testcase->flags & TT_SKIP)
316  puts(" (DISABLED)");
317  else
318  puts("");
319  }
320  if (!strncmp(fullname, arg, length)) {
321  if (set)
322  testcase->flags |= flag;
323  else
324  testcase->flags &= ~flag;
325  ++found;
326  }
327  }
328  }
329  return found;
330 }
331 
332 static void
333 usage(struct testgroup_t *groups, int list_groups)
334 {
335  puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
336  puts(" Specify tests by name, or using a prefix ending with '..'");
337  puts(" To skip a test, prefix its name with a colon.");
338  puts(" To enable a disabled test, prefix its name with a plus.");
339  puts(" Use --list-tests for a list of tests.");
340  if (list_groups) {
341  puts("Known tests are:");
342  tinytest_set_flag_(groups, "..", 1, 0);
343  }
344  exit(0);
345 }
346 
347 static int
348 process_test_alias(struct testgroup_t *groups, const char *test)
349 {
350  int i, j, n, r;
351  for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
352  if (!strcmp(cfg_aliases[i].name, test)) {
353  n = 0;
354  for (j = 0; cfg_aliases[i].tests[j]; ++j) {
355  r = process_test_option(groups, cfg_aliases[i].tests[j]);
356  if (r<0)
357  return -1;
358  n += r;
359  }
360  return n;
361  }
362  }
363  printf("No such test alias as @%s!",test);
364  return -1;
365 }
366 
367 static int
368 process_test_option(struct testgroup_t *groups, const char *test)
369 {
370  int flag = TT_ENABLED_;
371  int n = 0;
372  if (test[0] == '@') {
373  return process_test_alias(groups, test + 1);
374  } else if (test[0] == ':') {
375  ++test;
376  flag = TT_SKIP;
377  } else if (test[0] == '+') {
378  ++test;
379  ++n;
380  if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
381  printf("No such test as %s!\n", test);
382  return -1;
383  }
384  } else {
385  ++n;
386  }
387  if (!tinytest_set_flag_(groups, test, 1, flag)) {
388  printf("No such test as %s!\n", test);
389  return -1;
390  }
391  return n;
392 }
393 
394 void
395 tinytest_set_aliases(const struct testlist_alias_t *aliases)
396 {
397  cfg_aliases = aliases;
398 }
399 
400 int
401 tinytest_main(int c, const char **v, struct testgroup_t *groups)
402 {
403  int i, j, n=0;
404 
405 #ifdef _WIN32
406  const char *sp = strrchr(v[0], '.');
407  const char *extension = "";
408  if (!sp || stricmp(sp, ".exe"))
409  extension = ".exe"; /* Add an exe so CreateProcess will work */
410  snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
411  commandname[MAX_PATH]='\0';
412 #endif
413  for (i=1; i<c; ++i) {
414  if (v[i][0] == '-') {
415  if (!strcmp(v[i], "--RUNNING-FORKED")) {
416  opt_forked = 1;
417  } else if (!strcmp(v[i], "--no-fork")) {
418  opt_nofork = 1;
419  } else if (!strcmp(v[i], "--quiet")) {
420  opt_verbosity = -1;
421  verbosity_flag = "--quiet";
422  } else if (!strcmp(v[i], "--verbose")) {
423  opt_verbosity = 2;
424  verbosity_flag = "--verbose";
425  } else if (!strcmp(v[i], "--terse")) {
426  opt_verbosity = 0;
427  verbosity_flag = "--terse";
428  } else if (!strcmp(v[i], "--help")) {
429  usage(groups, 0);
430  } else if (!strcmp(v[i], "--list-tests")) {
431  usage(groups, 1);
432  } else {
433  printf("Unknown option %s. Try --help\n",v[i]);
434  return -1;
435  }
436  } else {
437  int r = process_test_option(groups, v[i]);
438  if (r<0)
439  return -1;
440  n += r;
441  }
442  }
443  if (!n)
444  tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
445 
446 #ifdef _IONBF
447  setvbuf(stdout, NULL, _IONBF, 0);
448 #endif
449 
450  ++in_tinytest_main;
451  for (i=0; groups[i].prefix; ++i)
452  for (j=0; groups[i].cases[j].name; ++j)
453  if (groups[i].cases[j].flags & TT_ENABLED_)
454  testcase_run_one(&groups[i],
455  &groups[i].cases[j]);
456 
457  --in_tinytest_main;
458 
459  if (opt_verbosity==0)
460  puts("");
461 
462  if (n_bad)
463  printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
464  n_bad+n_ok,n_skipped);
465  else if (opt_verbosity >= 1)
466  printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped);
467 
468  return (n_bad == 0) ? 0 : 1;
469 }
470 
471 int
472 tinytest_get_verbosity_(void)
473 {
474  return opt_verbosity;
475 }
476 
477 void
478 tinytest_set_test_failed_(void)
479 {
480  if (opt_verbosity <= 0 && cur_test_name) {
481  if (opt_verbosity==0) puts("");
482  printf("%s%s: ", cur_test_prefix, cur_test_name);
483  cur_test_name = NULL;
484  }
485  cur_test_outcome = 0;
486 }
487 
488 void
489 tinytest_set_test_skipped_(void)
490 {
491  if (cur_test_outcome==OK)
492  cur_test_outcome = SKIP;
493 }
494 
495 int
496 tinytest_cur_test_has_failed(void)
497 {
498  return (cur_test_outcome == FAIL);
499 }
500 
501 char *
502 tinytest_format_hex_(const void *val_, unsigned long len)
503 {
504  const unsigned char *val = val_;
505  char *result, *cp;
506  size_t i;
507  int ellipses = 0;
508 
509  if (!val)
510  return strdup("null");
511  if (len > 1024) {
512  ellipses = 3;
513  len = 1024;
514  }
515  if (!(result = malloc(len*2+4)))
516  return strdup("<allocation failure>");
517  cp = result;
518  for (i=0;i<len;++i) {
519  *cp++ = "0123456789ABCDEF"[(val[i] >> 4)&0x0f];
520  *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
521  }
522  while (ellipses--)
523  *cp++ = '.';
524  *cp = 0;
525  return result;
526 }
const char * name
Definition: config.c:2434
void *(* setup_fn)(const struct testcase_t *)
Definition: tinytest.h:47
int(* cleanup_fn)(const struct testcase_t *, void *)
Definition: tinytest.h:49
testcase_fn fn
Definition: tinytest.h:55
const char * name
Definition: tinytest.h:54
unsigned long flags
Definition: tinytest.h:56
const struct testcase_setup_t * setup
Definition: tinytest.h:57
const char * prefix
Definition: tinytest.h:64