Line data Source code
1 : /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2 : *
3 : * Redistribution and use in source and binary forms, with or without
4 : * modification, are permitted provided that the following conditions
5 : * are met:
6 : * 1. Redistributions of source code must retain the above copyright
7 : * notice, this list of conditions and the following disclaimer.
8 : * 2. Redistributions in binary form must reproduce the above copyright
9 : * notice, this list of conditions and the following disclaimer in the
10 : * documentation and/or other materials provided with the distribution.
11 : * 3. The name of the author may not be used to endorse or promote products
12 : * derived from this software without specific prior written permission.
13 : *
14 : * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 : * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 : * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 : * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18 : * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 : * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 : * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 : * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 : * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 : * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 : */
25 : #ifdef TINYTEST_LOCAL
26 : #include "tinytest_local.h"
27 : #endif
28 : #define TINYTEST_POSTFORK
29 :
30 : #include <stdio.h>
31 : #include <stdlib.h>
32 : #include <string.h>
33 : #include <assert.h>
34 :
35 : #ifndef NO_FORKING
36 :
37 : #ifdef _WIN32
38 : #include <windows.h>
39 : #else
40 : #include <sys/types.h>
41 : #include <sys/wait.h>
42 : #include <unistd.h>
43 : #endif
44 :
45 : #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
46 : #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
47 : __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
48 : /* Workaround for a stupid bug in OSX 10.6 */
49 : #define FORK_BREAKS_GCOV
50 : #include <vproc.h>
51 : #endif
52 : #endif
53 :
54 : #endif /* !NO_FORKING */
55 :
56 : #ifndef __GNUC__
57 : #define __attribute__(x)
58 : #endif
59 :
60 : #include "tinytest.h"
61 : #include "tinytest_macros.h"
62 :
63 : #define LONGEST_TEST_NAME 16384
64 :
65 : static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
66 : static int n_ok = 0; /**< Number of tests that have passed */
67 : static int n_bad = 0; /**< Number of tests that have failed. */
68 : static int n_skipped = 0; /**< Number of tests that have been skipped. */
69 :
70 : static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
71 : static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
72 : static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
73 : static const char *verbosity_flag = "";
74 :
75 : static const struct testlist_alias_t *cfg_aliases=NULL;
76 :
77 : enum outcome { SKIP=2, OK=1, FAIL=0 };
78 : static enum outcome cur_test_outcome = 0;
79 : /** prefix of the current test group */
80 : static const char *cur_test_prefix = NULL;
81 : /** Name of the current test, if we haven't logged is yet. Used for --quiet */
82 : static const char *cur_test_name = NULL;
83 :
84 : #ifdef _WIN32
85 : /* Copy of argv[0] for win32. */
86 : static char commandname[MAX_PATH+1];
87 : #endif
88 :
89 : static void usage(struct testgroup_t *groups, int list_groups)
90 : __attribute__((noreturn));
91 : static int process_test_option(struct testgroup_t *groups, const char *test);
92 :
93 : static enum outcome
94 1486 : testcase_run_bare_(const struct testcase_t *testcase)
95 : {
96 1486 : void *env = NULL;
97 1486 : int outcome;
98 1486 : if (testcase->setup) {
99 375 : env = testcase->setup->setup_fn(testcase);
100 375 : if (!env)
101 : return FAIL;
102 375 : else if (env == (void*)TT_SKIP)
103 : return SKIP;
104 : }
105 :
106 1486 : cur_test_outcome = OK;
107 1486 : testcase->fn(env);
108 1486 : outcome = cur_test_outcome;
109 :
110 1486 : if (testcase->setup) {
111 375 : if (testcase->setup->cleanup_fn(testcase, env) == 0)
112 0 : outcome = FAIL;
113 : }
114 :
115 1486 : return outcome;
116 : }
117 :
118 : #define MAGIC_EXITCODE 42
119 :
120 : #ifndef NO_FORKING
121 :
122 : #ifdef TINYTEST_POSTFORK
123 : void tinytest_prefork(void);
124 : void tinytest_postfork(void);
125 : #else
126 : static void tinytest_prefork(void) { }
127 : static void tinytest_postfork(void) { }
128 : #endif
129 :
130 : static enum outcome
131 927 : testcase_run_forked_(const struct testgroup_t *group,
132 : const struct testcase_t *testcase)
133 : {
134 : #ifdef _WIN32
135 : /* Fork? On Win32? How primitive! We'll do what the smart kids do:
136 : we'll invoke our own exe (whose name we recall from the command
137 : line) with a command line that tells it to run just the test we
138 : want, and this time without forking.
139 :
140 : (No, threads aren't an option. The whole point of forking is to
141 : share no state between tests.)
142 : */
143 : int ok;
144 : char buffer[LONGEST_TEST_NAME+256];
145 : STARTUPINFOA si;
146 : PROCESS_INFORMATION info;
147 : DWORD exitcode;
148 :
149 : if (!in_tinytest_main) {
150 : printf("\nERROR. On Windows, testcase_run_forked_ must be"
151 : " called from within tinytest_main.\n");
152 : abort();
153 : }
154 : if (opt_verbosity>0)
155 : printf("[forking] ");
156 :
157 : snprintf(buffer, sizeof(buffer), "\"%s\" --RUNNING-FORKED %s %s%s",
158 : commandname, verbosity_flag, group->prefix, testcase->name);
159 :
160 : memset(&si, 0, sizeof(si));
161 : memset(&info, 0, sizeof(info));
162 : si.cb = sizeof(si);
163 :
164 : ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
165 : 0, NULL, NULL, &si, &info);
166 : if (!ok) {
167 : printf("CreateProcess failed!\n");
168 : return 0;
169 : }
170 : WaitForSingleObject(info.hProcess, INFINITE);
171 : GetExitCodeProcess(info.hProcess, &exitcode);
172 : CloseHandle(info.hProcess);
173 : CloseHandle(info.hThread);
174 : if (exitcode == 0)
175 : return OK;
176 : else if (exitcode == MAGIC_EXITCODE)
177 : return SKIP;
178 : else
179 : return FAIL;
180 : #else
181 927 : int outcome_pipe[2];
182 927 : pid_t pid;
183 927 : (void)group;
184 :
185 927 : if (pipe(outcome_pipe))
186 0 : perror("opening pipe");
187 :
188 927 : if (opt_verbosity>0)
189 927 : printf("[forking] ");
190 927 : tinytest_prefork();
191 927 : pid = fork();
192 : #ifdef FORK_BREAKS_GCOV
193 : vproc_transaction_begin(0);
194 : #endif
195 1854 : tinytest_postfork();
196 1854 : if (!pid) {
197 : /* child. */
198 927 : int test_r, write_r;
199 927 : char b[1];
200 927 : close(outcome_pipe[0]);
201 927 : test_r = testcase_run_bare_(testcase);
202 927 : assert(0<=(int)test_r && (int)test_r<=2);
203 927 : b[0] = "NYS"[test_r];
204 927 : write_r = (int)write(outcome_pipe[1], b, 1);
205 927 : if (write_r != 1) {
206 0 : perror("write outcome to pipe");
207 0 : exit(1);
208 : }
209 927 : exit(0);
210 : return FAIL; /* unreachable */
211 : } else {
212 : /* parent */
213 927 : int status, r;
214 927 : char b[1];
215 : /* Close this now, so that if the other side closes it,
216 : * our read fails. */
217 927 : close(outcome_pipe[1]);
218 927 : r = (int)read(outcome_pipe[0], b, 1);
219 927 : if (r == 0) {
220 0 : printf("[Lost connection!] ");
221 0 : return FAIL;
222 927 : } else if (r != 1) {
223 0 : perror("read outcome from pipe");
224 : }
225 927 : r = waitpid(pid, &status, 0);
226 927 : close(outcome_pipe[0]);
227 927 : if (r == -1) {
228 0 : perror("waitpid");
229 0 : return FAIL;
230 : }
231 927 : if (! WIFEXITED(status) || WEXITSTATUS(status) != 0) {
232 0 : printf("[did not exit cleanly.]");
233 0 : return FAIL;
234 : }
235 927 : return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
236 : }
237 : #endif
238 : }
239 :
240 : #endif /* !NO_FORKING */
241 :
242 : int
243 11928 : testcase_run_one(const struct testgroup_t *group,
244 : const struct testcase_t *testcase)
245 : {
246 11928 : enum outcome outcome;
247 :
248 11928 : if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
249 10442 : if (opt_verbosity>0)
250 20884 : printf("%s%s: %s\n",
251 10442 : group->prefix, testcase->name,
252 10442 : (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
253 10442 : ++n_skipped;
254 10442 : return SKIP;
255 : }
256 :
257 1486 : if (opt_verbosity>0 && !opt_forked) {
258 1486 : printf("%s%s: ", group->prefix, testcase->name);
259 : } else {
260 0 : if (opt_verbosity==0) printf(".");
261 0 : cur_test_prefix = group->prefix;
262 0 : cur_test_name = testcase->name;
263 : }
264 :
265 : #ifndef NO_FORKING
266 1486 : if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
267 927 : outcome = testcase_run_forked_(group, testcase);
268 : } else {
269 : #else
270 : {
271 : #endif
272 559 : outcome = testcase_run_bare_(testcase);
273 : }
274 :
275 1486 : if (outcome == OK) {
276 1485 : ++n_ok;
277 1485 : if (opt_verbosity>0 && !opt_forked)
278 1485 : puts(opt_verbosity==1?"OK":"");
279 1 : } else if (outcome == SKIP) {
280 1 : ++n_skipped;
281 1 : if (opt_verbosity>0 && !opt_forked)
282 1 : puts("SKIPPED");
283 : } else {
284 0 : ++n_bad;
285 0 : if (!opt_forked)
286 0 : printf("\n [%s FAILED]\n", testcase->name);
287 : }
288 :
289 1486 : if (opt_forked) {
290 0 : exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
291 : return 1; /* unreachable */
292 : } else {
293 1486 : return (int)outcome;
294 : }
295 : }
296 :
297 : int
298 9 : tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
299 : {
300 9 : int i, j;
301 9 : size_t length = LONGEST_TEST_NAME;
302 9 : char fullname[LONGEST_TEST_NAME];
303 9 : int found=0;
304 9 : if (strstr(arg, ".."))
305 9 : length = strstr(arg,"..")-arg;
306 949 : for (i=0; groups[i].prefix; ++i) {
307 12868 : for (j=0; groups[i].cases[j].name; ++j) {
308 11928 : struct testcase_t *testcase = &groups[i].cases[j];
309 11928 : snprintf(fullname, sizeof(fullname), "%s%s",
310 : groups[i].prefix, testcase->name);
311 11928 : if (!flag) { /* Hack! */
312 0 : printf(" %s", fullname);
313 0 : if (testcase->flags & TT_OFF_BY_DEFAULT)
314 0 : puts(" (Off by default)");
315 0 : else if (testcase->flags & TT_SKIP)
316 0 : puts(" (DISABLED)");
317 : else
318 0 : puts("");
319 : }
320 11928 : if (!strncmp(fullname, arg, length)) {
321 11928 : if (set)
322 11928 : testcase->flags |= flag;
323 : else
324 0 : testcase->flags &= ~flag;
325 11928 : ++found;
326 : }
327 : }
328 : }
329 9 : return found;
330 : }
331 :
332 : static void
333 0 : usage(struct testgroup_t *groups, int list_groups)
334 : {
335 0 : puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
336 0 : puts(" Specify tests by name, or using a prefix ending with '..'");
337 0 : puts(" To skip a test, prefix its name with a colon.");
338 0 : puts(" To enable a disabled test, prefix its name with a plus.");
339 0 : puts(" Use --list-tests for a list of tests.");
340 0 : if (list_groups) {
341 0 : puts("Known tests are:");
342 0 : tinytest_set_flag_(groups, "..", 1, 0);
343 : }
344 0 : exit(0);
345 : }
346 :
347 : static int
348 0 : process_test_alias(struct testgroup_t *groups, const char *test)
349 : {
350 0 : int i, j, n, r;
351 0 : for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
352 0 : if (!strcmp(cfg_aliases[i].name, test)) {
353 : n = 0;
354 0 : for (j = 0; cfg_aliases[i].tests[j]; ++j) {
355 0 : r = process_test_option(groups, cfg_aliases[i].tests[j]);
356 0 : if (r<0)
357 : return -1;
358 0 : n += r;
359 : }
360 0 : return n;
361 : }
362 : }
363 0 : printf("No such test alias as @%s!",test);
364 0 : return -1;
365 : }
366 :
367 : static int
368 0 : process_test_option(struct testgroup_t *groups, const char *test)
369 : {
370 0 : int flag = TT_ENABLED_;
371 0 : int n = 0;
372 0 : if (test[0] == '@') {
373 0 : return process_test_alias(groups, test + 1);
374 0 : } else if (test[0] == ':') {
375 0 : ++test;
376 0 : flag = TT_SKIP;
377 0 : } else if (test[0] == '+') {
378 0 : ++test;
379 0 : ++n;
380 0 : if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
381 0 : printf("No such test as %s!\n", test);
382 0 : return -1;
383 : }
384 : } else {
385 : ++n;
386 : }
387 0 : if (!tinytest_set_flag_(groups, test, 1, flag)) {
388 0 : printf("No such test as %s!\n", test);
389 0 : return -1;
390 : }
391 : return n;
392 : }
393 :
394 : void
395 0 : tinytest_set_aliases(const struct testlist_alias_t *aliases)
396 : {
397 0 : cfg_aliases = aliases;
398 0 : }
399 :
400 : int
401 9 : tinytest_main(int c, const char **v, struct testgroup_t *groups)
402 : {
403 9 : int i, j, n=0;
404 :
405 : #ifdef _WIN32
406 : const char *sp = strrchr(v[0], '.');
407 : const char *extension = "";
408 : if (!sp || stricmp(sp, ".exe"))
409 : extension = ".exe"; /* Add an exe so CreateProcess will work */
410 : snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
411 : commandname[MAX_PATH]='\0';
412 : #endif
413 9 : for (i=1; i<c; ++i) {
414 0 : if (v[i][0] == '-') {
415 0 : if (!strcmp(v[i], "--RUNNING-FORKED")) {
416 0 : opt_forked = 1;
417 0 : } else if (!strcmp(v[i], "--no-fork")) {
418 0 : opt_nofork = 1;
419 0 : } else if (!strcmp(v[i], "--quiet")) {
420 0 : opt_verbosity = -1;
421 0 : verbosity_flag = "--quiet";
422 0 : } else if (!strcmp(v[i], "--verbose")) {
423 0 : opt_verbosity = 2;
424 0 : verbosity_flag = "--verbose";
425 0 : } else if (!strcmp(v[i], "--terse")) {
426 0 : opt_verbosity = 0;
427 0 : verbosity_flag = "--terse";
428 0 : } else if (!strcmp(v[i], "--help")) {
429 0 : usage(groups, 0);
430 0 : } else if (!strcmp(v[i], "--list-tests")) {
431 0 : usage(groups, 1);
432 : } else {
433 0 : printf("Unknown option %s. Try --help\n",v[i]);
434 0 : return -1;
435 : }
436 : } else {
437 0 : int r = process_test_option(groups, v[i]);
438 0 : if (r<0)
439 : return -1;
440 0 : n += r;
441 : }
442 : }
443 9 : if (!n)
444 9 : tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
445 :
446 : #ifdef _IONBF
447 9 : setvbuf(stdout, NULL, _IONBF, 0);
448 : #endif
449 :
450 9 : ++in_tinytest_main;
451 949 : for (i=0; groups[i].prefix; ++i)
452 12868 : for (j=0; groups[i].cases[j].name; ++j)
453 11928 : if (groups[i].cases[j].flags & TT_ENABLED_)
454 11928 : testcase_run_one(&groups[i],
455 : &groups[i].cases[j]);
456 :
457 9 : --in_tinytest_main;
458 :
459 9 : if (opt_verbosity==0)
460 0 : puts("");
461 :
462 9 : if (n_bad)
463 0 : printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
464 : n_bad+n_ok,n_skipped);
465 9 : else if (opt_verbosity >= 1)
466 9 : printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped);
467 :
468 9 : return (n_bad == 0) ? 0 : 1;
469 : }
470 :
471 : int
472 735695 : tinytest_get_verbosity_(void)
473 : {
474 735695 : return opt_verbosity;
475 : }
476 :
477 : void
478 0 : tinytest_set_test_failed_(void)
479 : {
480 0 : if (opt_verbosity <= 0 && cur_test_name) {
481 0 : if (opt_verbosity==0) puts("");
482 0 : printf("%s%s: ", cur_test_prefix, cur_test_name);
483 0 : cur_test_name = NULL;
484 : }
485 0 : cur_test_outcome = 0;
486 0 : }
487 :
488 : void
489 1 : tinytest_set_test_skipped_(void)
490 : {
491 1 : if (cur_test_outcome==OK)
492 1 : cur_test_outcome = SKIP;
493 1 : }
494 :
495 : int
496 21 : tinytest_cur_test_has_failed(void)
497 : {
498 21 : return (cur_test_outcome == FAIL);
499 : }
500 :
501 : char *
502 0 : tinytest_format_hex_(const void *val_, unsigned long len)
503 : {
504 0 : const unsigned char *val = val_;
505 0 : char *result, *cp;
506 0 : size_t i;
507 0 : int ellipses = 0;
508 :
509 0 : if (!val)
510 0 : return strdup("null");
511 0 : if (len > 1024) {
512 0 : ellipses = 3;
513 0 : len = 1024;
514 : }
515 0 : if (!(result = malloc(len*2+4)))
516 0 : return strdup("<allocation failure>");
517 : cp = result;
518 0 : for (i=0;i<len;++i) {
519 0 : *cp++ = "0123456789ABCDEF"[(val[i] >> 4)&0x0f];
520 0 : *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
521 : }
522 0 : while (ellipses--)
523 0 : *cp++ = '.';
524 0 : *cp = 0;
525 0 : return result;
526 : }
|
