Version 1.66b: Dir detection and dictionary updates

ChangeLog

@@ -1,3 +1,10 @@
+Version 1.66b:
+--------------
+
+  - Improved directory detection logic.
+
+  - Some dictionary updates.
+
 Version 1.65b:
 --------------
 

Makefile

@@ -20,7 +20,7 @@
 #
 
 PROGNAME   = skipfish
-VERSION    = 1.65b
+VERSION    = 1.66b
 
 OBJFILES   = http_client.c database.c crawler.c analysis.c report.c
 INCFILES   = alloc-inl.h string-inl.h debug.h types.h http_client.h \

crawler.c

@@ -2278,9 +2278,11 @@ static u8 dir_404_callback(struct http_request* req,
 
   /* If the first 404 probe returned something that looks like the
      "root" page for the currently tested directory, panic. But don't
-     do that check on server pivots. */
+     do that check on server pivots, or if valid redirect detected
+     earlier. */
 
-  if (!req->user_val && req->pivot->type != PIVOT_SERV && RPRES(req) && 
+  if (!req->user_val && !req->pivot->sure_dir && 
+      req->pivot->type != PIVOT_SERV && RPRES(req) && 
       same_page(&res->sig, &RPRES(req)->sig)) {
     DEBUG("* First 404 probe identical with parent!\n");
     goto schedule_next;
@@ -2926,7 +2928,7 @@ static u8 unknown_check_callback(struct http_request* req,
      checks).
 
      If pivot != res, and res is not a 404 response, assume dir;
-     and if it is 404, assume file.
+     and if it is 404, assume file, except if pivot redirected to res.
 
      We also have a special case if the original request returned a
      non-empty 2xx response, but the new one returned 3xx-5xx - this is
@@ -2936,6 +2938,24 @@ static u8 unknown_check_callback(struct http_request* req,
     u32 i = 0;
     struct pivot_desc* par = dir_parent(req->pivot);
 
+    if (res->code == 404 && RPRES(req)->code >= 300 && RPRES(req)->code < 400) {
+      u8 *loc = GET_HDR((u8*)"Location", &RPRES(req)->hdr);
+
+      if (loc) {
+        u8* path = serialize_path(req, 1, 0);
+
+        if (!strcasecmp((char*)path, (char*)loc)) {
+          ck_free(path);
+          req->pivot->sure_dir = 1;
+          goto assume_dir;
+        }
+
+        ck_free(path);
+
+      }
+
+    }
+
     if (par)
       for (i=0;i<par->r404_cnt;i++)
         if (same_page(&res->sig, &par->r404[i])) break;

database.h

@@ -109,6 +109,7 @@ struct pivot_desc {
   /* Fuzzer and probe state data: */
 
   u8 no_fuzz;                                   /* Do not attepmt fuzzing.   */
+  u8 sure_dir;                                  /* Very sure it's a dir?     */
 
   u8  uses_ips;                                 /* Uses IPS filtering?       */
 

dictionaries/complete.wl

@@ -17,6 +17,7 @@ e 1 1 1 conf
 e 1 1 1 config
 e 1 1 1 cpp
 e 1 1 1 cs
+e 1 1 1 csproj
 e 1 1 1 csv
 e 1 1 1 dat
 e 1 1 1 db
@@ -28,6 +29,7 @@ e 1 1 1 ep
 e 1 1 1 err
 e 1 1 1 error
 e 1 1 1 exe
+e 1 1 1 fcgi
 e 1 1 1 gif
 e 1 1 1 gz
 e 1 1 1 htm
@@ -56,6 +58,8 @@ e 1 1 1 orig
 e 1 1 1 out
 e 1 1 1 part
 e 1 1 1 pdf
+e 1 1 1 pem
+e 1 1 1 pfx
 e 1 1 1 php
 e 1 1 1 php3
 e 1 1 1 phtml
@@ -66,6 +70,8 @@ e 1 1 1 ppt
 e 1 1 1 properties
 e 1 1 1 py
 e 1 1 1 rar
+e 1 1 1 rb
+e 1 1 1 rhtml
 e 1 1 1 rss
 e 1 1 1 rtf
 e 1 1 1 save
@@ -74,6 +80,7 @@ e 1 1 1 shtml
 e 1 1 1 so
 e 1 1 1 sql
 e 1 1 1 stackdump
+e 1 1 1 svn-base
 e 1 1 1 swf
 e 1 1 1 tar
 e 1 1 1 tar.bz2
@@ -90,6 +97,7 @@ e 1 1 1 ws
 e 1 1 1 xls
 e 1 1 1 xml
 e 1 1 1 xsl
+e 1 1 1 yml
 e 1 1 1 zip
 w 1 1 1 .bash_history
 w 1 1 1 .bashrc
@@ -192,6 +200,7 @@ w 1 1 1 OAErrorDetailPage
 w 1 1 1 OA_HTML
 w 1 1 1 Program Files
 w 1 1 1 README
+w 1 1 1 Rakefile
 w 1 1 1 Readme
 w 1 1 1 Recycled
 w 1 1 1 Root
@@ -278,6 +287,7 @@ w 1 1 1 alerts
 w 1 1 1 alias
 w 1 1 1 aliases
 w 1 1 1 all
+w 1 1 1 all-wcprops
 w 1 1 1 alpha
 w 1 1 1 alumni
 w 1 1 1 amazon
@@ -489,7 +499,6 @@ w 1 1 1 cluster
 w 1 1 1 clusters
 w 1 1 1 cmd
 w 1 1 1 cms
-w 1 1 1 cnf
 w 1 1 1 cnt
 w 1 1 1 cocoon
 w 1 1 1 code
@@ -637,6 +646,7 @@ w 1 1 1 diffs
 w 1 1 1 digest
 w 1 1 1 digg
 w 1 1 1 dir
+w 1 1 1 dir-prop-base
 w 1 1 1 directories
 w 1 1 1 directory
 w 1 1 1 dirs
@@ -718,7 +728,6 @@ w 1 1 1 entry
 w 1 1 1 env
 w 1 1 1 environ
 w 1 1 1 environment
-w 1 1 1 error
 w 1 1 1 error-log
 w 1 1 1 error_log
 w 1 1 1 errors
@@ -769,6 +778,7 @@ w 1 1 1 field
 w 1 1 1 fields
 w 1 1 1 file
 w 1 1 1 fileadmin
+w 1 1 1 filelist
 w 1 1 1 files
 w 1 1 1 filez
 w 1 1 1 finance
@@ -793,6 +803,7 @@ w 1 1 1 foo
 w 1 1 1 footer
 w 1 1 1 footers
 w 1 1 1 form
+w 1 1 1 format
 w 1 1 1 formatting
 w 1 1 1 formmail
 w 1 1 1 forms
@@ -1326,7 +1337,6 @@ w 1 1 1 playlist
 w 1 1 1 pls
 w 1 1 1 plugin
 w 1 1 1 plugins
-w 1 1 1 pm
 w 1 1 1 poc
 w 1 1 1 poi
 w 1 1 1 policies
@@ -1389,7 +1399,7 @@ w 1 1 1 project
 w 1 1 1 projects
 w 1 1 1 promo
 w 1 1 1 prop
-w 1 1 1 properties
+w 1 1 1 prop-base
 w 1 1 1 property
 w 1 1 1 props
 w 1 1 1 prot
@@ -1747,6 +1757,7 @@ w 1 1 1 testing
 w 1 1 1 tests
 w 1 1 1 texis
 w 1 1 1 text
+w 1 1 1 text-base
 w 1 1 1 texts
 w 1 1 1 theme
 w 1 1 1 themes
@@ -1952,7 +1963,6 @@ w 1 1 1 wp-includes
 w 1 1 1 wp-login
 w 1 1 1 wp-syntax
 w 1 1 1 wrap
-w 1 1 1 ws
 w 1 1 1 ws-client
 w 1 1 1 ws_ftp
 w 1 1 1 wtai

dictionaries/default.wl

@@ -12,6 +12,7 @@ e 1 1 1 cnf
 e 1 1 1 conf
 e 1 1 1 config
 e 1 1 1 cpp
+e 1 1 1 csproj
 e 1 1 1 csv
 e 1 1 1 dat
 e 1 1 1 db
@@ -19,6 +20,7 @@ e 1 1 1 dll
 e 1 1 1 err
 e 1 1 1 error
 e 1 1 1 exe
+e 1 1 1 fcgi
 e 1 1 1 gz
 e 1 1 1 htm
 e 1 1 1 html
@@ -44,11 +46,13 @@ e 1 1 1 phtml
 e 1 1 1 pl
 e 1 1 1 pm
 e 1 1 1 py
+e 1 1 1 rb
 e 1 1 1 rss
 e 1 1 1 sh
 e 1 1 1 shtml
 e 1 1 1 sql
 e 1 1 1 stackdump
+e 1 1 1 svn-base
 e 1 1 1 tar.gz
 e 1 1 1 temp
 e 1 1 1 test
@@ -164,6 +168,7 @@ w 1 1 1 OAErrorDetailPage
 w 1 1 1 OA_HTML
 w 1 1 1 Program Files
 w 1 1 1 README
+w 1 1 1 Rakefile
 w 1 1 1 Readme
 w 1 1 1 Recycled
 w 1 1 1 Root
@@ -250,6 +255,7 @@ w 1 1 1 alerts
 w 1 1 1 alias
 w 1 1 1 aliases
 w 1 1 1 all
+w 1 1 1 all-wcprops
 w 1 1 1 alpha
 w 1 1 1 alumni
 w 1 1 1 amazon
@@ -464,7 +470,6 @@ w 1 1 1 cluster
 w 1 1 1 clusters
 w 1 1 1 cmd
 w 1 1 1 cms
-w 1 1 1 cnf
 w 1 1 1 cnt
 w 1 1 1 cocoon
 w 1 1 1 code
@@ -613,6 +618,7 @@ w 1 1 1 diffs
 w 1 1 1 digest
 w 1 1 1 digg
 w 1 1 1 dir
+w 1 1 1 dir-prop-base
 w 1 1 1 directories
 w 1 1 1 directory
 w 1 1 1 dirs
@@ -698,7 +704,6 @@ w 1 1 1 env
 w 1 1 1 environ
 w 1 1 1 environment
 w 1 1 1 ep
-w 1 1 1 error
 w 1 1 1 error-log
 w 1 1 1 error_log
 w 1 1 1 errors
@@ -749,6 +754,7 @@ w 1 1 1 field
 w 1 1 1 fields
 w 1 1 1 file
 w 1 1 1 fileadmin
+w 1 1 1 filelist
 w 1 1 1 files
 w 1 1 1 filez
 w 1 1 1 finance
@@ -773,6 +779,7 @@ w 1 1 1 foo
 w 1 1 1 footer
 w 1 1 1 footers
 w 1 1 1 form
+w 1 1 1 format
 w 1 1 1 formatting
 w 1 1 1 formmail
 w 1 1 1 forms
@@ -1273,12 +1280,14 @@ w 1 1 1 pdf
 w 1 1 1 pdfs
 w 1 1 1 pear
 w 1 1 1 peek
+w 1 1 1 pem
 w 1 1 1 pending
 w 1 1 1 people
 w 1 1 1 perf
 w 1 1 1 performance
 w 1 1 1 perl
 w 1 1 1 personal
+w 1 1 1 pfx
 w 1 1 1 pg
 w 1 1 1 phf
 w 1 1 1 phone
@@ -1315,7 +1324,6 @@ w 1 1 1 playlist
 w 1 1 1 pls
 w 1 1 1 plugin
 w 1 1 1 plugins
-w 1 1 1 pm
 w 1 1 1 png
 w 1 1 1 poc
 w 1 1 1 poi
@@ -1380,6 +1388,7 @@ w 1 1 1 project
 w 1 1 1 projects
 w 1 1 1 promo
 w 1 1 1 prop
+w 1 1 1 prop-base
 w 1 1 1 properties
 w 1 1 1 property
 w 1 1 1 props
@@ -1488,6 +1497,7 @@ w 1 1 1 revert
 w 1 1 1 reverted
 w 1 1 1 review
 w 1 1 1 reviews
+w 1 1 1 rhtml
 w 1 1 1 right
 w 1 1 1 roam
 w 1 1 1 roaming
@@ -1745,6 +1755,7 @@ w 1 1 1 testing
 w 1 1 1 tests
 w 1 1 1 texis
 w 1 1 1 text
+w 1 1 1 text-base
 w 1 1 1 texts
 w 1 1 1 theme
 w 1 1 1 themes
@@ -1951,7 +1962,6 @@ w 1 1 1 wp-includes
 w 1 1 1 wp-login
 w 1 1 1 wp-syntax
 w 1 1 1 wrap
-w 1 1 1 ws
 w 1 1 1 ws-client
 w 1 1 1 ws_ftp
 w 1 1 1 wtai
@@ -1979,6 +1989,7 @@ w 1 1 1 y
 w 1 1 1 yahoo
 w 1 1 1 year
 w 1 1 1 yearly
+w 1 1 1 yml
 w 1 1 1 youtube
 w 1 1 1 yt
 w 1 1 1 z

dictionaries/extensions-only.wl

@@ -1,3 +1,4 @@
+e 1 1 1 7z
 e 1 1 1 asmx
 e 1 1 1 asp
 e 1 1 1 aspx
@@ -14,6 +15,7 @@ e 1 1 1 conf
 e 1 1 1 config
 e 1 1 1 cpp
 e 1 1 1 cs
+e 1 1 1 csproj
 e 1 1 1 csv
 e 1 1 1 dat
 e 1 1 1 db
@@ -25,6 +27,7 @@ e 1 1 1 ep
 e 1 1 1 err
 e 1 1 1 error
 e 1 1 1 exe
+e 1 1 1 fcgi
 e 1 1 1 gif
 e 1 1 1 gz
 e 1 1 1 htm
@@ -53,6 +56,8 @@ e 1 1 1 orig
 e 1 1 1 out
 e 1 1 1 part
 e 1 1 1 pdf
+e 1 1 1 pem
+e 1 1 1 pfx
 e 1 1 1 php
 e 1 1 1 php3
 e 1 1 1 phtml
@@ -63,6 +68,8 @@ e 1 1 1 ppt
 e 1 1 1 properties
 e 1 1 1 py
 e 1 1 1 rar
+e 1 1 1 rb
+e 1 1 1 rhtml
 e 1 1 1 rss
 e 1 1 1 rtf
 e 1 1 1 save
@@ -71,6 +78,7 @@ e 1 1 1 shtml
 e 1 1 1 so
 e 1 1 1 sql
 e 1 1 1 stackdump
+e 1 1 1 svn-base
 e 1 1 1 swf
 e 1 1 1 tar
 e 1 1 1 tar.bz2
@@ -87,5 +95,5 @@ e 1 1 1 ws
 e 1 1 1 xls
 e 1 1 1 xml
 e 1 1 1 xsl
+e 1 1 1 yml
 e 1 1 1 zip
-e 1 1 1 7z

dictionaries/minimal.wl

@@ -4,6 +4,7 @@ e 1 1 1 class
 e 1 1 1 cnf
 e 1 1 1 conf
 e 1 1 1 config
+e 1 1 1 csproj
 e 1 1 1 csv
 e 1 1 1 err
 e 1 1 1 error
@@ -18,7 +19,9 @@ e 1 1 1 orig
 e 1 1 1 out
 e 1 1 1 part
 e 1 1 1 pl
+e 1 1 1 rb
 e 1 1 1 sql
+e 1 1 1 svn-base
 e 1 1 1 temp
 e 1 1 1 test
 e 1 1 1 tmp
@@ -127,6 +130,7 @@ w 1 1 1 OAErrorDetailPage
 w 1 1 1 OA_HTML
 w 1 1 1 Program Files
 w 1 1 1 README
+w 1 1 1 Rakefile
 w 1 1 1 Readme
 w 1 1 1 Recycled
 w 1 1 1 Root
@@ -213,6 +217,7 @@ w 1 1 1 alerts
 w 1 1 1 alias
 w 1 1 1 aliases
 w 1 1 1 all
+w 1 1 1 all-wcprops
 w 1 1 1 alpha
 w 1 1 1 alumni
 w 1 1 1 amazon
@@ -434,7 +439,6 @@ w 1 1 1 cluster
 w 1 1 1 clusters
 w 1 1 1 cmd
 w 1 1 1 cms
-w 1 1 1 cnf
 w 1 1 1 cnt
 w 1 1 1 cocoon
 w 1 1 1 code
@@ -586,6 +590,7 @@ w 1 1 1 diffs
 w 1 1 1 digest
 w 1 1 1 digg
 w 1 1 1 dir
+w 1 1 1 dir-prop-base
 w 1 1 1 directories
 w 1 1 1 directory
 w 1 1 1 dirs
@@ -672,7 +677,6 @@ w 1 1 1 env
 w 1 1 1 environ
 w 1 1 1 environment
 w 1 1 1 ep
-w 1 1 1 error
 w 1 1 1 error-log
 w 1 1 1 error_log
 w 1 1 1 errors
@@ -712,6 +716,7 @@ w 1 1 1 faq
 w 1 1 1 faqs
 w 1 1 1 favorite
 w 1 1 1 favorites
+w 1 1 1 fcgi
 w 1 1 1 fcgi-bin
 w 1 1 1 feature
 w 1 1 1 features
@@ -724,6 +729,7 @@ w 1 1 1 field
 w 1 1 1 fields
 w 1 1 1 file
 w 1 1 1 fileadmin
+w 1 1 1 filelist
 w 1 1 1 files
 w 1 1 1 filez
 w 1 1 1 finance
@@ -748,6 +754,7 @@ w 1 1 1 foo
 w 1 1 1 footer
 w 1 1 1 footers
 w 1 1 1 form
+w 1 1 1 format
 w 1 1 1 formatting
 w 1 1 1 formmail
 w 1 1 1 forms
@@ -1257,12 +1264,14 @@ w 1 1 1 pdf
 w 1 1 1 pdfs
 w 1 1 1 pear
 w 1 1 1 peek
+w 1 1 1 pem
 w 1 1 1 pending
 w 1 1 1 people
 w 1 1 1 perf
 w 1 1 1 performance
 w 1 1 1 perl
 w 1 1 1 personal
+w 1 1 1 pfx
 w 1 1 1 pg
 w 1 1 1 phf
 w 1 1 1 phone
@@ -1280,6 +1289,7 @@ w 1 1 1 phpbb
 w 1 1 1 phpmyadmin
 w 1 1 1 phpnuke
 w 1 1 1 phps
+w 1 1 1 phtml
 w 1 1 1 pic
 w 1 1 1 pics
 w 1 1 1 pictures
@@ -1366,6 +1376,7 @@ w 1 1 1 project
 w 1 1 1 projects
 w 1 1 1 promo
 w 1 1 1 prop
+w 1 1 1 prop-base
 w 1 1 1 properties
 w 1 1 1 property
 w 1 1 1 props
@@ -1475,6 +1486,7 @@ w 1 1 1 revert
 w 1 1 1 reverted
 w 1 1 1 review
 w 1 1 1 reviews
+w 1 1 1 rhtml
 w 1 1 1 right
 w 1 1 1 roam
 w 1 1 1 roaming
@@ -1737,6 +1749,7 @@ w 1 1 1 testing
 w 1 1 1 tests
 w 1 1 1 texis
 w 1 1 1 text
+w 1 1 1 text-base
 w 1 1 1 texts
 w 1 1 1 tgz
 w 1 1 1 theme
@@ -1976,6 +1989,7 @@ w 1 1 1 y
 w 1 1 1 yahoo
 w 1 1 1 year
 w 1 1 1 yearly
+w 1 1 1 yml
 w 1 1 1 youtube
 w 1 1 1 yt
 w 1 1 1 z