sitemap: sitemap.xml
User-agent: *
#crawl delay not supported by Google, can be configured via Google webmaster tools. ASK, Yahoo, & Bing support.
Disallow: /*/ctl/
Disallow: /AdCreator #removes all helper files from the Contextual Ad Product
Disallow: /admin/
Disallow: /App_Browsers/
Disallow: /App_Code/
Disallow: /App_Data/
Disallow: /App_GlobalResources/
Disallow: /bin/
Disallow: /Components/
Disallow: /Config/
Disallow: /contest/
Disallow: /controls/
Disallow: /DesktopModules/ #will hide all print versions of pages, most css, 
Disallow: /Documentation/
Disallow: /HttpModules/
Disallow: /images/
Disallow: /Install/
Disallow: /js/
Disallow: /Portals/ #will block whitepapers
Disallow: /Providers/
Disallow: /Resources/
Disallow: /MyAccount/
# * and $ pattern matching is not supported by ASK.com
Disallow: /*.axd$ #remove all axd files (.net resource files) - may not be supported by Yahoo
Disallow: /*.js$ #remove all javascript files - may not be supported by Yahoo, most .js files are in /js directory