# Crawlers Setup
User-agent: *
Crawl-delay: 30

Sitemap: https://www.diamondscamera.com.au/feeds/dia_google_sitemap.xml
 
## DEVELOPMENT RELATED SETTINGS
 
## Do not crawl development files and folders: CVS, svn directories and dump files
Disallow:/*.CVS
Disallow:/*.svn$
Disallow:/*.idea$
Disallow:/*.sql$
Disallow:/*.tgz$
 
## GENERAL MAGENTO SETTINGS
 
## Do not crawl Magento admin page
Disallow:/admin/
 
## Do not crawl common Magento technical folders
Disallow:/app/
Disallow:/downloader/
Disallow:/errors/
Disallow:/includes/
Disallow:/lib/
Disallow:/pkginfo/
Disallow:/shell/
Disallow:/var/
 
## Do not crawl common Magento files
Disallow:/api.php
Disallow:/cron.php
Disallow:/cron.sh
Disallow:/error_log
Disallow:/get.php
Disallow:/install.php
Disallow:/LICENSE.html
Disallow:/LICENSE.txt
Disallow:/LICENSE_AFL.txt
Disallow:/README.txt
Disallow:/RELEASE_NOTES.txt
 
## MAGENTO SEO IMPROVEMENTS
 
## Do not crawl sub category pages that are sorted or filtered.
Disallow:/*?dir*
Disallow:/*?dir=desc
Disallow:/*?dir=asc
Disallow:/*?limit=all
Disallow:/*?mode*
 
## Do not crawl 2-nd home page copy (example.com/index.php/). Uncomment it only if you activated Magento SEO URLs.
## Disallow:/index.php/
 
## Do not crawl links with session IDs
Disallow:/*?SID=
 
## Do not crawl checkout and user account pages
Disallow:/checkout/
Disallow:/onestepcheckout/
Disallow:/customer/
Disallow:/customer/account/
Disallow:/customer/account/login/
 
## Do not crawl seach pages and not-SEO optimized catalog links
Disallow:/catalogsearch/
Disallow:/catalog/product_compare/
Disallow:/catalog/category/view/
Disallow:/catalog/product/view/