User-agent:  *
Disallow:  /cgi-bin
Disallow:  /errors
Disallow:  /private
Disallow:  /scripts
Disallow:  /test

User-agent: *
Disallow: /*.gif$

User-agent: *
Disallow: /*.jpg$

User-agent: *
Disallow: /*.pdf$

User-agent: *
Disallow: /*.rtf$

# disallow collectors and spam
User-agent: atSpider
Disallow: /

User-agent: cherrypicker
Disallow: /

User-agent: DSurf
Disallow: /

User-agent: EliteSys Entry
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /