diff --git a/afl-analyze.c b/afl-analyze.c index 2c27a51e..c2213822 100644 --- a/afl-analyze.c +++ b/afl-analyze.c @@ -89,21 +89,17 @@ static volatile u8 /* Classify tuple counts. This is a slow & naive version, but good enough here. */ -#define AREP4(_sym) (_sym), (_sym), (_sym), (_sym) -#define AREP8(_sym) AREP4(_sym), AREP4(_sym) -#define AREP16(_sym) AREP8(_sym), AREP8(_sym) -#define AREP32(_sym) AREP16(_sym), AREP16(_sym) -#define AREP64(_sym) AREP32(_sym), AREP32(_sym) -#define AREP128(_sym) AREP64(_sym), AREP64(_sym) - static u8 count_class_lookup[256] = { - /* 0 - 3: 4 */ 0, 1, 2, 4, - /* 4 - 7: +4 */ AREP4(8), - /* 8 - 15: +8 */ AREP8(16), - /* 16 - 31: +16 */ AREP16(32), - /* 32 - 127: +96 */ AREP64(64), AREP32(64), - /* 128+: +128 */ AREP128(128) + [0] = 0, + [1] = 1, + [2] = 2, + [3] = 4, + [4 ... 7] = 8, + [8 ... 15] = 16, + [16 ... 31] = 32, + [32 ... 127] = 64, + [128 ... 255] = 128 }; diff --git a/afl-fuzz.c b/afl-fuzz.c index 2d362933..044dc99b 100644 --- a/afl-fuzz.c +++ b/afl-fuzz.c @@ -60,7 +60,8 @@ # include #endif /* __APPLE__ || __FreeBSD__ || __OpenBSD__ */ -/* For supporting -Z on systems that have sched_setaffinity. */ +/* For systems that have sched_setaffinity; right now just Linux, but one + can hope... */ #ifdef __linux__ # define HAVE_AFFINITY 1 @@ -200,14 +201,11 @@ static u64 total_cal_us, /* Total calibration time (us) */ static u64 total_bitmap_size, /* Total bit count for all bitmaps */ total_bitmap_entries; /* Number of bitmaps counted */ -static u32 cpu_core_count; /* CPU core count */ +static s32 cpu_core_count; /* CPU core count */ #ifdef HAVE_AFFINITY -static u8 use_affinity; /* Using -Z */ - -static u32 cpu_aff_main, /* Affinity for main process */ - cpu_aff_child; /* Affinity for fuzzed child */ +static s32 cpu_aff = -1; /* Selected CPU core */ #endif /* HAVE_AFFINITY */ @@ -340,25 +338,6 @@ static u64 get_cur_time_us(void) { } -#ifdef HAVE_AFFINITY - -/* Set CPU affinity (on systems that support it). */ - -static void set_cpu_affinity(u32 cpu_id) { - - cpu_set_t c; - - CPU_ZERO(&c); - CPU_SET(cpu_id, &c); - - if (sched_setaffinity(0, sizeof(c), &c)) - PFATAL("sched_setaffinity failed"); - -} - -#endif /* HAVE_AFFINITY */ - - /* Generate a random number (from 0 to limit - 1). This may have slight bias. */ @@ -398,6 +377,122 @@ static void shuffle_ptrs(void** ptrs, u32 cnt) { } +#ifdef HAVE_AFFINITY + +/* Build a list of processes bound to specific cores. Returns -1 if nothing + can be found. Assumes an upper bound of 4k CPUs. */ + +static void bind_to_free_cpu(void) { + + DIR* d; + struct dirent* de; + cpu_set_t c; + + u8 cpu_used[4096] = { 0 }; + u32 i; + + if (!cpu_core_count) return; + + if (getenv("AFL_NO_AFFINITY")) { + + WARNF("Not binding to a CPU core (AFL_NO_AFFINITY set)."); + return; + + } + + d = opendir("/proc"); + + if (!d) { + + WARNF("Unable to access /proc - can't scan for free CPU cores."); + return; + + } + + ACTF("Checking CPU core loadout..."); + + /* Introduce some jitter, in case multiple AFL tasks are doing the same + thing at the same time... */ + + usleep(R(1000) * 250); + + /* Scan all /proc//status entries, checking for Cpus_allowed_list. + Flag all processes bound to a specific CPU using cpu_used[]. This will + fail for some exotic binding setups, but is likely good enough in almost + all real-world use cases. */ + + while ((de = readdir(d))) { + + u8* fn; + FILE* f; + u8 tmp[MAX_LINE]; + u8 has_vmsize = 0; + + if (!isdigit(de->d_name[0])) continue; + + fn = alloc_printf("/proc/%s/status", de->d_name); + + if (!(f = fopen(fn, "r"))) { + ck_free(fn); + continue; + } + + while (fgets(tmp, MAX_LINE, f)) { + + u32 hval; + + /* Processes without VmSize are probably kernel tasks. */ + + if (!strncmp(tmp, "VmSize:\t", 8)) has_vmsize = 1; + + if (!strncmp(tmp, "Cpus_allowed_list:\t", 19) && + !strchr(tmp, '-') && !strchr(tmp, ',') && + sscanf(tmp + 19, "%u", &hval) == 1 && hval < sizeof(cpu_used) && + has_vmsize) { + + cpu_used[hval] = 1; + break; + + } + + } + + ck_free(fn); + fclose(f); + + } + + closedir(d); + + for (i = 0; i < cpu_core_count; i++) if (!cpu_used[i]) break; + + if (i == cpu_core_count) { + + SAYF("\n" cLRD "[-] " cRST + "Uh-oh, looks like all %u CPU cores on your system are allocated to\n" + " other instances of afl-fuzz (or similar CPU-locked tasks). Starting\n" + " another fuzzer on this machine is probably a bad plan, but if you are\n" + " absolutely sure, you can set AFL_NO_AFFINITY and try again.\n", + cpu_core_count); + + FATAL("No more free CPU cores"); + + } + + OKF("Found a free CPU core, binding to #%u.", i); + + cpu_aff = i; + + CPU_ZERO(&c); + CPU_SET(i, &c); + + if (sched_setaffinity(0, sizeof(c), &c)) + PFATAL("sched_setaffinity failed"); + +} + +#endif /* HAVE_AFFINITY */ + #ifndef IGNORE_FINDS /* Helper function to compare buffers; returns first and last differing offset. We @@ -948,21 +1043,11 @@ static u32 count_non_255_bytes(u8* mem) { is hit or not. Called on every new crash or hang, should be reasonably fast. */ -#define AREP4(_sym) (_sym), (_sym), (_sym), (_sym) -#define AREP8(_sym) AREP4(_sym), AREP4(_sym) -#define AREP16(_sym) AREP8(_sym), AREP8(_sym) -#define AREP32(_sym) AREP16(_sym), AREP16(_sym) -#define AREP64(_sym) AREP32(_sym), AREP32(_sym) -#define AREP128(_sym) AREP64(_sym), AREP64(_sym) - -static u8 simplify_lookup[256] = { - /* 4 */ 1, 128, 128, 128, - /* +4 */ AREP4(128), - /* +8 */ AREP8(128), - /* +16 */ AREP16(128), - /* +32 */ AREP32(128), - /* +64 */ AREP64(128), - /* +128 */ AREP128(128) +static const u8 simplify_lookup[256] = { + + [0] = 1, + [1 ... 255] = 128 + }; #ifdef __x86_64__ @@ -1029,14 +1114,17 @@ static void simplify_trace(u32* mem) { preprocessing step for any newly acquired traces. Called on every exec, must be fast. */ -static u8 count_class_lookup[256] = { +static const u8 count_class_lookup[256] = { - /* 0 - 3: 4 */ 0, 1, 2, 4, - /* 4 - 7: +4 */ AREP4(8), - /* 8 - 15: +8 */ AREP8(16), - /* 16 - 31: +16 */ AREP16(32), - /* 32 - 127: +96 */ AREP64(64), AREP32(64), - /* 128+: +128 */ AREP128(128) + [0] = 0, + [1] = 1, + [2] = 2, + [3] = 4, + [4 ... 7] = 8, + [8 ... 15] = 16, + [16 ... 31] = 32, + [32 ... 127] = 64, + [128 ... 255] = 128 }; @@ -1897,10 +1985,6 @@ EXP_ST void init_forkserver(char** argv) { struct rlimit r; -#ifdef HAVE_AFFINITY - if (use_affinity) set_cpu_affinity(cpu_aff_child); -#endif /* HAVE_AFFINITY */ - /* Umpf. On OpenBSD, the default fd limit for root users is set to soft 128. Let's try to fix that... */ @@ -2199,10 +2283,6 @@ static u8 run_target(char** argv) { struct rlimit r; -#ifdef HAVE_AFFINITY - if (use_affinity) set_cpu_affinity(cpu_aff_child); -#endif /* HAVE_AFFINITY */ - if (mem_limit) { r.rlim_max = r.rlim_cur = ((rlim_t)mem_limit) << 20; @@ -4132,10 +4212,10 @@ static void show_stats(void) { #ifdef HAVE_AFFINITY - if (use_affinity) { + if (cpu_aff >= 0) { - SAYF(SP10 cGRA "[cpu@%02u:%s%3u%%" cGRA "]\r" cRST, - MIN(cpu_aff_child, 99), cpu_color, + SAYF(SP10 cGRA "[cpu%03u:%s%3u%%" cGRA "]\r" cRST, + MIN(cpu_aff, 999), cpu_color, MIN(cur_utilization, 999)); } else { @@ -4144,6 +4224,7 @@ static void show_stats(void) { cpu_color, MIN(cur_utilization, 999)); } + #else SAYF(SP10 cGRA " [cpu:%s%3u%%" cGRA "]\r" cRST, @@ -6838,9 +6919,6 @@ static void usage(u8* argv0) { " -T text - text banner to show on the screen\n" " -M / -S id - distributed mode (see parallel_fuzzing.txt)\n" -#ifdef HAVE_AFFINITY - " -Z core_id - set CPU affinity (see perf_tips.txt)\n" -#endif /* HAVE_AFFINITY */ " -C - crash exploration mode (the peruvian rabbit thing)\n\n" "For additional tips, please consult %s/README.\n\n", @@ -7138,27 +7216,27 @@ static void get_core_count(void) { #else - if (!cpu_core_count) { +#ifdef HAVE_AFFINITY + + cpu_core_count = sysconf(_SC_NPROCESSORS_ONLN); + +#else - /* On Linux, a simple way is to look at /proc/stat, especially since we'd - be parsing it anyway for other reasons later on. But do this only if - cpu_core_count hasn't been obtained before as a result of specifying - -Z. */ + FILE* f = fopen("/proc/stat", "r"); + u8 tmp[1024]; - FILE* f = fopen("/proc/stat", "r"); - u8 tmp[1024]; + if (!f) return; - if (!f) return; + while (fgets(tmp, sizeof(tmp), f)) + if (!strncmp(tmp, "cpu", 3) && isdigit(tmp[3])) cpu_core_count++; - while (fgets(tmp, sizeof(tmp), f)) - if (!strncmp(tmp, "cpu", 3) && isdigit(tmp[3])) cpu_core_count++; + fclose(f); - fclose(f); - } +#endif /* ^HAVE_AFFINITY */ #endif /* ^(__APPLE__ || __FreeBSD__ || __OpenBSD__) */ - if (cpu_core_count) { + if (cpu_core_count > 0) { cur_runnable = (u32)get_runnable_processes(); @@ -7187,17 +7265,12 @@ static void get_core_count(void) { } - } else WARNF("Unable to figure out the number of CPU cores."); - -#ifdef HAVE_AFFINITY + } else { - if (use_affinity) - OKF("Using specified CPU affinity: main = %u, child = %u", - cpu_aff_main, cpu_aff_child); - else if (cpu_core_count > 1) - OKF(cBRI "Try setting CPU affinity (-Z) for a performance boost!" cRST); + cpu_core_count = 0; + WARNF("Unable to figure out the number of CPU cores."); -#endif /* HAVE_AFFINITY */ + } } @@ -7490,7 +7563,7 @@ int main(int argc, char** argv) { doc_path = access(DOC_PATH, F_OK) ? "docs" : DOC_PATH; - while ((opt = getopt(argc, argv, "+i:o:f:m:t:T:dnCB:S:M:x:QZ:")) > 0) + while ((opt = getopt(argc, argv, "+i:o:f:m:t:T:dnCB:S:M:x:Q")) > 0) switch (opt) { @@ -7586,35 +7659,6 @@ int main(int argc, char** argv) { break; -#ifdef HAVE_AFFINITY - - case 'Z': { - - s32 i; - - if (use_affinity) FATAL("Multiple -Z options not supported"); - use_affinity = 1; - - cpu_core_count = sysconf(_SC_NPROCESSORS_ONLN); - - i = sscanf(optarg, "%u,%u", &cpu_aff_main, &cpu_aff_child); - - if (i < 1 || cpu_aff_main >= cpu_core_count) - FATAL("Bogus primary core ID passed to -Z (expected 0-%u)", - cpu_core_count - 1); - - if (i == 1) cpu_aff_child = cpu_aff_main; - - if (cpu_aff_child >= cpu_core_count) - FATAL("Bogus secondary core ID passed to -Z (expected 0-%u)", - cpu_core_count - 1); - - break; - - } - -#endif /* HAVE_AFFINITY */ - case 'd': if (skip_deterministic) FATAL("Multiple -d options not supported"); @@ -7680,10 +7724,6 @@ int main(int argc, char** argv) { setup_signal_handlers(); check_asan_opts(); -#ifdef HAVE_AFFINITY - if (use_affinity) set_cpu_affinity(cpu_aff_main); -#endif /* HAVE_AFFINITY */ - if (sync_id) fix_up_sync(); if (!strcmp(in_dir, out_dir)) @@ -7714,6 +7754,11 @@ int main(int argc, char** argv) { check_if_tty(); get_core_count(); + +#ifdef HAVE_AFFINITY + bind_to_free_cpu(); +#endif /* HAVE_AFFINITY */ + check_crash_handling(); check_cpu_governor(); diff --git a/afl-showmap.c b/afl-showmap.c index cd66772e..c03d67bf 100644 --- a/afl-showmap.c +++ b/afl-showmap.c @@ -73,21 +73,17 @@ static volatile u8 /* Classify tuple counts. Instead of mapping to individual bits, as in afl-fuzz.c, we map to more user-friendly numbers between 1 and 8. */ -#define AREP4(_sym) (_sym), (_sym), (_sym), (_sym) -#define AREP8(_sym) AREP4(_sym), AREP4(_sym) -#define AREP16(_sym) AREP8(_sym), AREP8(_sym) -#define AREP32(_sym) AREP16(_sym), AREP16(_sym) -#define AREP64(_sym) AREP32(_sym), AREP32(_sym) -#define AREP128(_sym) AREP64(_sym), AREP64(_sym) - -static u8 count_class_lookup[256] = { - - /* 0 - 3: 4 */ 0, 1, 2, 3, - /* 4 - 7: +4 */ AREP4(4), - /* 8 - 15: +8 */ AREP8(5), - /* 16 - 31: +16 */ AREP16(6), - /* 32 - 127: +96 */ AREP64(7), AREP32(7), - /* 128+: +128 */ AREP128(8) +static const u8 count_class_lookup[256] = { + + [0] = 0, + [1] = 1, + [2] = 2, + [3] = 3, + [4 ... 7] = 4, + [8 ... 15] = 5, + [16 ... 31] = 6, + [32 ... 127] = 7, + [128 ... 255] = 8 }; diff --git a/afl-tmin.c b/afl-tmin.c index 1a0cbb83..a42698ef 100644 --- a/afl-tmin.c +++ b/afl-tmin.c @@ -81,21 +81,17 @@ static volatile u8 /* Classify tuple counts. This is a slow & naive version, but good enough here. */ -#define AREP4(_sym) (_sym), (_sym), (_sym), (_sym) -#define AREP8(_sym) AREP4(_sym), AREP4(_sym) -#define AREP16(_sym) AREP8(_sym), AREP8(_sym) -#define AREP32(_sym) AREP16(_sym), AREP16(_sym) -#define AREP64(_sym) AREP32(_sym), AREP32(_sym) -#define AREP128(_sym) AREP64(_sym), AREP64(_sym) - -static u8 count_class_lookup[256] = { - - /* 0 - 3: 4 */ 0, 1, 2, 4, - /* 4 - 7: +4 */ AREP4(8), - /* 8 - 15: +8 */ AREP8(16), - /* 16 - 31: +16 */ AREP16(32), - /* 32 - 127: +96 */ AREP64(64), AREP32(64), - /* 128+: +128 */ AREP128(128) +static const u8 count_class_lookup[256] = { + + [0] = 0, + [1] = 1, + [2] = 2, + [3] = 4, + [4 ... 7] = 8, + [8 ... 15] = 16, + [16 ... 31] = 32, + [32 ... 127] = 64, + [128 ... 255] = 128 }; diff --git a/config.h b/config.h index 99d3c2f7..364e46c0 100644 --- a/config.h +++ b/config.h @@ -21,7 +21,7 @@ /* Version string: */ -#define VERSION "2.16b" +#define VERSION "2.17b" /****************************************************** * * diff --git a/docs/ChangeLog b/docs/ChangeLog index 17d8e59d..d5096a5d 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -16,6 +16,16 @@ Not sure if you should upgrade? The lowest currently recommended version is 2.07b. If you're stuck on an earlier release, it's strongly advisable to get on with the times. +-------------- +Version 2.17b: +-------------- + + - Killed the error-prone and manual -Z option. On Linux, AFL will now + automatically bind to the first free core (or complain if there are no + free cores left). + + - Made some doc updates along these lines. + -------------- Version 2.16b: -------------- diff --git a/docs/env_variables.txt b/docs/env_variables.txt index 16de0344..8bd2362a 100644 --- a/docs/env_variables.txt +++ b/docs/env_variables.txt @@ -109,6 +109,10 @@ checks or alter some of the more exotic semantics of the tool: normally indicated by the cycle counter in the UI turning green. May be convenient for some types of automated jobs. + - Setting AFL_NO_AFFINITY disables attempts to bind to a specific CPU core + on Linux systems. This slows things down, but lets you run more instances + of afl-fuzz than would be prudent (if you really want to). + - AFL_SKIP_CRASHES causes AFL to tolerate crashing files in the input queue. This can help with rare situations where a program crashes only intermittently, but it's not really recommended under normal operating diff --git a/docs/perf_tips.txt b/docs/perf_tips.txt index e05401d7..e7a8b13b 100644 --- a/docs/perf_tips.txt +++ b/docs/perf_tips.txt @@ -144,29 +144,7 @@ a fair amount of time allocating and initializing megabytes of memory when presented with pathological inputs. Low -m values can make them give up sooner and not waste CPU time. -8) Set CPU core affinity for AFL --------------------------------- - -Making sure that the fuzzer always runs on the same (idle) CPU core can offer -a significant speed bump and reduce scheduler jitter. The benefits can be even -more striking on true multiprocessor systems. - -On Linux, you can assign the fuzzer to a specific core by first running -afl-gotcpu to see which cores are idle, and then specifying the ID of a -preferred core via -Z, like so: - - $ ./afl-fuzz -Z core_id [...other parameters...] - -Note that this parameter needs to be used with care; accidentally forcing -multiple fuzzers to share the same core may result in performance that is -worse than what you would get without -Z. - -(It is also possible to specify two comma-delimited values for -Z, in which -case, the fuzzer will run on one designated core, and the target binary will -be banished to another. This can sometimes offer minor benefits, but isn't -recommended for general use.) - -9) Check OS configuration +8) Check OS configuration ------------------------- There are several OS-level factors that may affect fuzzing speed: @@ -200,8 +178,8 @@ There are several OS-level factors that may affect fuzzing speed: SCHED_RR - can usually speed things up, too, but needs to be done with care. -10) If all other options fail, use -d -------------------------------------- +9) If all other options fail, use -d +------------------------------------ For programs that are genuinely slow, in cases where you really can't escape using huge input files, or when you simply want to get quick and dirty results diff --git a/docs/sister_projects.txt b/docs/sister_projects.txt index 886c2f59..f8fe7312 100644 --- a/docs/sister_projects.txt +++ b/docs/sister_projects.txt @@ -6,9 +6,9 @@ Sister projects designed for, or meant to integrate with AFL. See README for the general instruction manual. ----------------------------- -Support for other languages: ----------------------------- +------------------------------------------- +Support for other languages / environments: +------------------------------------------- Python AFL (Jakub Wilk) ----------------------- @@ -70,6 +70,14 @@ AFL fixup shim (Ben Nagy) https://github.com/bnagy/aflfix +TriforceAFL (Tim Newsham and Jesse Hertz) +----------------------------------------- + + Leverages QEMU full system emulation mode to allow AFL to target operating + systems and other alien worlds: + + https://www.nccgroup.trust/us/about-us/newsroom-and-events/blog/2016/june/project-triforce-run-afl-on-everything/ + ---------------- Network fuzzing: ---------------- @@ -273,3 +281,4 @@ Kernel fuzzing (Dmitry Vyukov) https://github.com/google/syzkaller/wiki/Found-Bugs https://github.com/dvyukov/linux/commit/33787098ffaaa83b8a7ccf519913ac5fd6125931 + http://events.linuxfoundation.org/sites/events/files/slides/AFL%20filesystem%20fuzzing%2C%20Vault%202016_0.pdf