User-agent: * # directed to all robots
Disallow: /cgi-bin/
Disallow: /bin/
Disallow: /custom/
Disallow: /dev/
Disallow: /etc/
Disallow: /lib/
Disallow: /logs/