From 831a3a497b5418e513ec16f5dcf3dbd68362dd57 Mon Sep 17 00:00:00 2001 From: Steve Pinkham Date: Tue, 9 Aug 2011 16:01:39 -0400 Subject: [PATCH] 1.92b: Reading starting URLs from file now supported (@ prefix). --- ChangeLog | 5 ++++ Makefile | 6 ++--- README | 9 ++++--- alloc-inl.h | 2 +- analysis.c | 2 +- analysis.h | 2 +- assets/index.html | 2 +- config.h | 2 +- crawler.c | 2 +- crawler.h | 2 +- database.c | 2 +- database.h | 2 +- debug.h | 2 +- http_client.c | 2 +- http_client.h | 2 +- report.c | 2 +- report.h | 2 +- same_test.c | 2 +- skipfish.c | 65 ++++++++++++++++++++++++++++++++++++++++++++--- types.h | 2 +- 20 files changed, 91 insertions(+), 26 deletions(-) diff --git a/ChangeLog b/ChangeLog index 4a0f556..f530412 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,8 @@ +Version 1.92b: +-------------- + + - Reading starting URLs from file is now supported (@ prefix). + Version 1.90b / 1.91b: ---------------------- diff --git a/Makefile b/Makefile index b8d5a38..b83b837 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ # # Author: Michal Zalewski # -# Copyright 2009, 2010 by Google Inc. All Rights Reserved. +# Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -20,7 +20,7 @@ # PROGNAME = skipfish -VERSION = 1.91b +VERSION = 1.92b OBJFILES = http_client.c database.c crawler.c analysis.c report.c INCFILES = alloc-inl.h string-inl.h debug.h types.h http_client.h \ @@ -28,7 +28,7 @@ INCFILES = alloc-inl.h string-inl.h debug.h types.h http_client.h \ CFLAGS_GEN = -Wall -funsigned-char -g -ggdb -I/usr/local/include/ \ -I/opt/local/include/ $(CFLAGS) -DVERSION=\"$(VERSION)\" -CFLAGS_DBG = -DLOG_STDERR=1 -DDEBUG_ALLOCATOR=1 $(CFLAGS_GEN) +CFLAGS_DBG = -DLOG_STDERR=1 -DDEBUG_ALLOCATOR=1 $(CFLAGS_GEN) CFLAGS_OPT = -O3 -Wno-format $(CFLAGS_GEN) LDFLAGS += -L/usr/local/lib/ -L/opt/local/lib diff --git a/README b/README index b0de854..57abb62 100644 --- a/README +++ b/README @@ -5,7 +5,7 @@ skipfish - web application security scanner http://code.google.com/p/skipfish/ * Written and maintained by Michal Zalewski . - * Copyright 2009, 2010 Google Inc, rights reserved. + * Copyright 2009, 2010, 2011 Google Inc, rights reserved. * Released under terms and conditions of the Apache License, version 2.0. -------------------- @@ -237,8 +237,11 @@ Once you have the dictionary selected, you can try: $ ./skipfish -o output_dir http://www.example.com/some/starting/path.txt -Note that you can provide more than one starting URL if so desired; all of -them will be crawled. +Note that you can provide more than one starting URL if so desired; all of +them will be crawled. It is also possible to read URLs from file, using +the following syntax: + +$ ./skipfish -o output_dir @../path/to/url_list.txt The tool will display some helpful stats while the scan is in progress. You can also switch to a list of in-flight HTTP requests by pressing return. diff --git a/alloc-inl.h b/alloc-inl.h index 591ea8c..a37717e 100644 --- a/alloc-inl.h +++ b/alloc-inl.h @@ -7,7 +7,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/analysis.c b/analysis.c index 7fa272e..bed61e7 100644 --- a/analysis.c +++ b/analysis.c @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/analysis.h b/analysis.h index 056f4b5..f1716e0 100644 --- a/analysis.h +++ b/analysis.h @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/assets/index.html b/assets/index.html index ce1ebda..22e8c05 100644 --- a/assets/index.html +++ b/assets/index.html @@ -7,7 +7,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/config.h b/config.h index dd9b5ce..5bbf921 100644 --- a/config.h +++ b/config.h @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/crawler.c b/crawler.c index c3f69e4..fb2cded 100644 --- a/crawler.c +++ b/crawler.c @@ -6,7 +6,7 @@ Author: Michal Zalewski - Copyright 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/crawler.h b/crawler.h index 3879a12..3f62840 100644 --- a/crawler.h +++ b/crawler.h @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/database.c b/database.c index f41098e..63e84be 100644 --- a/database.c +++ b/database.c @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/database.h b/database.h index 506f514..dcc5e5a 100644 --- a/database.h +++ b/database.h @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/debug.h b/debug.h index 90d186d..e7b9586 100644 --- a/debug.h +++ b/debug.h @@ -5,7 +5,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/http_client.c b/http_client.c index 2f629a5..891ff2b 100644 --- a/http_client.c +++ b/http_client.c @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/http_client.h b/http_client.h index aa58737..a2a3a54 100644 --- a/http_client.h +++ b/http_client.h @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/report.c b/report.c index b1bc667..c4c0bb7 100644 --- a/report.c +++ b/report.c @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/report.h b/report.h index 291ed5d..97182a6 100644 --- a/report.h +++ b/report.h @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/same_test.c b/same_test.c index 946d804..367edf3 100644 --- a/same_test.c +++ b/same_test.c @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/skipfish.c b/skipfish.c index 8c27207..fcd52cc 100644 --- a/skipfish.c +++ b/skipfish.c @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -181,6 +181,54 @@ void splash_screen(void) { #endif /* SHOW_SPLASH */ +/* Load URLs from file. */ + +static void read_urls(u8* fn) { + FILE* f = fopen((char*)fn, "r"); + u8 tmp[MAX_URL_LEN]; + u32 loaded = 0; + + if (!f) FATAL("Unable to open '%s'.", fn); + + while (fgets((char*)tmp, MAX_URL_LEN, f)) { + struct http_request *req; + u8* url = tmp; + u32 l; + + while (isspace(*url)) url++; + + l = strlen((char*)url); + while (l && isspace(url[l-1])) l--; + url[l] = 0; + + if (*url == '#' || !*url) continue; + + req = ck_alloc(sizeof(struct http_request)); + + if (parse_url(url, req, NULL)) + FATAL("Scan target '%s' in file '%s' is not a valid absolute URL.", url, fn); + + if (!url_allowed_host(req)) + APPEND_FILTER(allow_domains, num_allow_domains, + __DFL_ck_strdup(req->host)); + + if (!url_allowed(req)) + FATAL("URL '%s' in file '%s' explicitly excluded by -I / -X rules.", + url, fn); + + maybe_add_pivot(req, NULL, 2); + destroy_request(req); + loaded++; + + } + + fclose(f); + + if (!loaded) FATAL("No valid URLs found in '%s'.", fn); + +} + + /* Main entry point */ int main(int argc, char** argv) { @@ -458,14 +506,23 @@ int main(int argc, char** argv) { load_keywords((u8*)wordlist, purge_age); - /* Schedule all URLs in the command line for scanning */ + /* Schedule all URLs in the command line for scanning. */ while (optind < argc) { - struct http_request *req = ck_alloc(sizeof(struct http_request)); + struct http_request *req; + + /* Support @ notation for reading URL lists from files. */ + + if (argv[optind][0] == '@') { + read_urls((u8*)argv[optind++] + 1); + continue; + } + + req = ck_alloc(sizeof(struct http_request)); if (parse_url((u8*)argv[optind], req, NULL)) - FATAL("One of specified scan targets is not a valid absolute URL."); + FATAL("Scan target '%s' is not a valid absolute URL.", argv[optind]); if (!url_allowed_host(req)) APPEND_FILTER(allow_domains, num_allow_domains, diff --git a/types.h b/types.h index fdf05d3..2e5f1d7 100644 --- a/types.h +++ b/types.h @@ -4,7 +4,7 @@ Author: Michal Zalewski - Copyright 2009, 2010 by Google Inc. All Rights Reserved. + Copyright 2009, 2010, 2011 by Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.