# All robots will spider the domain User-agent: * Disallow: # Disallow directory /catalog/ User-agent: * Disallow: /catalog/ # Disallow directory /admin/ User-agent: * Disallow: /admin/ # Disallow directory /cache/ User-agent: * Disallow: /cache/ # Disallow directory /cgi-bin/ User-agent: * Disallow: /cgi-bin/ # Disallow directory /new/ User-agent: * Disallow: /new/ # Don't need to index these User-agent: * Disallow: /account.php Disallow: /checkout_shipping.php Disallow: /create_account.php Disallow: /password_forgotten.php Disallow: /popup_image.php Disallow: /popup_image_2.php Disallow: /popup_image_3.php Disallow: /product_reviews.php Disallow: /product_reviews_write.php Disallow: /product_thumb.php Disallow: /shopping_cart.php