# All robots will spider the domain
User-agent: *
Disallow: /scripts
Disallow: /admin
Disallow: /download
Disallow: /images
Disallow: /Editable
Disallow: /includes
Disallow: /pdf
Disallow: /pub
Disallow: /temp