# /robots.txt file 

User-agent: *
Allow: / 
Disallow: /tmp
Disallow: /logs
Disallow: /stats
Disallow: /usr
Disallow: /var
Disallow: /bin
Disallow: /dev
Disallow: /etc
Disallow: /pd