#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html
#
# For syntax checking, see:
# http://www.frobee.com/robots-txt-check

User-agent: *
Crawl-delay: 10
# CSS, JS, Images
Allow: /misc/*.css$
Allow: /misc/*.css?
Allow: /misc/*.js$
Allow: /misc/*.js?
Allow: /misc/*.gif
Allow: /misc/*.jpg
Allow: /misc/*.jpeg
Allow: /misc/*.png
Allow: /modules/*.css$
Allow: /modules/*.css?
Allow: /modules/*.js$
Allow: /modules/*.js?
Allow: /modules/*.gif
Allow: /modules/*.jpg
Allow: /modules/*.jpeg
Allow: /modules/*.png
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /themes/*.css$
Allow: /themes/*.css?
Allow: /themes/*.js$
Allow: /themes/*.js?
Allow: /themes/*.gif
Allow: /themes/*.jpg
Allow: /themes/*.jpeg
Allow: /themes/*.png
# Directories
Disallow: /includes/
Disallow: /misc/
Disallow: /modules/
Disallow: /profiles/
Disallow: /scripts/
Disallow: /themes/

# Files
Disallow: /CHANGELOG.txt
Disallow: /cron.php
Disallow: /INSTALL.mysql.txt
Disallow: /INSTALL.pgsql.txt
Disallow: /INSTALL.sqlite.txt
Disallow: /install.php
Disallow: /INSTALL.txt
Disallow: /LICENSE.txt
Disallow: /MAINTAINERS.txt
Disallow: /update.php
Disallow: /UPGRADE.txt
Disallow: /xmlrpc.php

# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips/
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register/
Disallow: /user/password/
Disallow: /user/login/
Disallow: /user/logout/
Disallow: /loveisdin-med/
Disallow: /loveisdin-med-thanks/
Disallow: /*/user/register/
Disallow: /*/user/password/
Disallow: /blog/editar-perfil$
Disallow: /blog/disfruta-tu-embarazo$
Disallow: /blog/gracias-por-registrarte$
Disallow: /blog/tu-calendario$
Disallow: /blog/registro$
Disallow: /blog/editar-perfil/
Disallow: /blog/disfruta-tu-embarazo/
Disallow: /blog/gracias-por-registrarte/
Disallow: /blog/tu-calendario/
Disallow: /blog/registro/
Disallow: /blog/cat/
Disallow: /blog/tag/
Disallow: /blog/author/
Disallow: /blog/sign-up/
Disallow: /blog/politica-de-privacidad/
Disallow: /blog/login/
Disallow: /blog/logout/
Disallow: /blog/rm_submissions/
Disallow: /blog/register/
Disallow: /blog/reset-password/
Disallow: /blog/gracias-por-registrate/
Disallow: /blog/httpswww-instagram-comisdinofficialhles/
Disallow: /farmacias/
# Paths excluded 17/09/2019
Disallow: /love-isdin-inicio-sesion/  
Disallow: /users/login/  
Disallow: /user    
Disallow: /users/new/
Disallow: /icnbarcelona2017/
Disallow: /icnbarcelona2017-thanks/  
Disallow: /login-registro-temp/
# Paths excluded 16/10/2019
Disallow: /pdf/NOTA-INFORMATIVA-DE-ISDIN-SOBRE-LA-COMUNICACION-DE-LA-AEMPS-EN-REFERENCIA-A-LA-FOTOPROTECCION-DE-UNO-DE-SUS-PRODUCTOS.pdf
Disallow: /pdf/Stellungnahme-ISDIN-OCU.pdf
Disallow: /pdf/Comunicacao-ISDIN-DECO.pdf
Disallow: /pdf/Comunicacao-ISDIN-OCU-BR.pdf
Disallow: /pdf/Communication-ISDIN-OCU.pdf
Disallow: /pdf/Comunicazione-ISDIN-OCU.pdf
Disallow: /pdf/Comunicacao-ISDIN-OCU.pdf
Disallow: /pdf/Comunicado-ISDIN-OCU.pdf
Disallow: /pdf/20190530_NOTA_INFORMATIVA_POSICIONAMIENTO_ISDIN_OCU.pdf

# Languages paths excluded 08/01/2015
Disallow: /es-PA/
# Languages paths exclude 21/01/2015
Disallow: /lb/
# Languages paths exclude 25/09/2018
Disallow: /fr-TN/
# Languages paths exclude 09/07/2019
Disallow: /uk-UA/
# Languages paths exclude 24/09/2019
Disallow: /el-GR/

# Languages /de/ paths exclude 30/04/2015. Updated 30/07/2015
Allow: /de/sonnenallergie-fotoultra-solar-allergy
Allow: /de/pigmentflecken-entfernen-hautunreinhetin
Allow: /de/eryfotona-aktinischer-keratose
Allow: /de/sonnenschutz-reparatur-aktinische-keratose
Allow: /de/isdin-ihre-haut-spricht
Allow: /de/kontakt
Allow: /de/cookie-richtlinie
Allow: /de/datenschutzrichtlinie
Allow: /de/nutzungsbedingungen
Allow: /de/allgemeine-verkaufs-und-lieferbedingungen
Allow: /de/impressum
Disallow: /us/brand*

# No follow Node without alias
Disallow: */node

# No folow urls pages
Disallow: /urls/$
Disallow: /en/urls/$
Disallow: /es/urls/$
Disallow: /pt-PT/urls/$
Disallow: /it/urls/$
Disallow: /es-AR/urls/$
Disallow: /es-CL/urls/$
Disallow: /eng/urls/$
Disallow: /de/urls/$
Disallow: /lb/urls/$
Disallow: /es-CO/urls/$
Disallow: /fr-FR/urls/$
Disallow: /en-CA/urls/$
Disallow: /fr-CA/urls/$
Disallow: /kr/urls/$
Disallow: /es-MX/urls/$
Disallow: /es-PA/urls/$
Disallow: /es-PE/urls/$
Disallow: /pt-BR/urls/$
Disallow: /en-IN/urls/$
Disallow: /fr-BE/urls/$
Disallow: /nl-BE/urls/$
Disallow: /es-PE/urls/$
Disallow: /en-ZA/urls/$
Disallow: /en-AE/urls/$
Disallow: /uk-UA/urls/$
Disallow: /el-GR/urls/$

# Paths (no clean URLs)
Disallow: /?q=admin/
Disallow: /?q=comment/reply/
Disallow: /?q=filter/tips/
Disallow: /?q=node/add/
Disallow: /?q=search/
Disallow: /?q=user/password/
Disallow: /?q=user/register/
Disallow: /?q=user/login/
Disallow: /?q=user/logout/
# Disallow new paths 01/04/2019
Disallow: /us/*?cgid=*
Disallow: /us/*?q=*
Disallow: /us/cart
Disallow: /us/sign-in
Disallow: /us/my-wishlist
Disallow: /us/login
Disallow: /us/my-orders
Disallow: /us/my-profile
Disallow: /us/aboutus
Disallow: /us/contact-us
Disallow: /us/privacy-policy
Disallow: /us/return-policy
Disallow: /us/terms-conditions
# Disallow demandware paths 17/06/2019
Disallow: /on/*
Disallow: */demandware*

User-agent: Googlebot-Image
Disallow: /assets/img/noindex/*

# Disallow annoying bots 01/04/2019
User-agent: MJ12bot
Disallow: /

User-agent: larbin
Disallow: /

User-agent: b2w/0.1
Disallow: /

User-agent: Copernic
Disallow: /

User-agent: psbot
Disallow: /

User-agent: Python-urllib
Disallow: /

User-agent: NetMechanic
Disallow: /

User-agent: URL_Spider_Pro
Disallow: /

User-agent: CherryPicker
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: WebBandit
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: ExtractorPro
Disallow: /

User-agent: CopyRightCheck
Disallow: /

User-agent: Crescent
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: ProWebWalker
Disallow: /

User-agent: CheeseBot
Disallow: /

User-agent: LNSpiderguy
Disallow: /

User-agent: Alexibot
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: MIIxpc
Disallow: /

User-agent: Telesoft
Disallow: /

User-agent: Website Quester
Disallow: /

User-agent: WebZip
Disallow: /

User-agent: moget/2.1
Disallow: /

User-agent: WebZip/4.0
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: WebSauger
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: NetAnts
Disallow: /

User-agent: Mister PiX
Disallow: /

User-agent: WebAuto
Disallow: /

User-agent: TheNomad
Disallow: /

User-agent: WWW-Collector-E
Disallow: /

User-agent: RMA
Disallow: /

User-agent: libWeb/clsHTTP
Disallow: /

User-agent: asterias
Disallow: /

User-agent: httplib
Disallow: /

User-agent: turingos
Disallow: /

User-agent: spanner
Disallow: /

User-agent: InfoNaviRobot
Disallow: /

User-agent: Harvest/1.5
Disallow: /

User-agent: Bullseye/1.0
Disallow: /

User-agent: Crescent Internet ToolPak HTTP OLE Control v.1.0
Disallow: /

User-agent: CherryPickerSE/1.0
Disallow: /

User-agent: CherryPickerElite/1.0
Disallow: /

User-agent: WebBandit/3.50
Disallow: /

User-agent: NICErsPRO
Disallow: /

User-agent: DittoSpyder
Disallow: /

User-agent: Foobot
Disallow: /

User-agent: WebmasterWorldForumBot
Disallow: /

User-agent: SpankBot
Disallow: /

User-agent: BotALot
Disallow: /

User-agent: lwp-trivial/1.34
Disallow: /

User-agent: lwp-trivial
Disallow: /

User-agent: BunnySlippers
Disallow: /

User-agent: Microsoft URL Control - 6.00.8169
Disallow: /

User-agent: URLy Warning
Disallow: /

User-agent: Wget/1.6
Disallow: /

User-agent: Wget/1.5.3
Disallow: /

User-agent: Wget
Disallow: /

User-agent: LinkWalker
Disallow: /

User-agent: cosmos
Disallow: /

User-agent: moget
Disallow: /

User-agent: hloader
Disallow: /

User-agent: humanlinks
Disallow: /

User-agent: LinkextractorPro
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Mata Hari
Disallow: /

User-agent: LexiBot
Disallow: /

User-agent: Web Image Collector
Disallow: /

User-agent: The Intraformant
Disallow: /

User-agent: True_Robot/1.0
Disallow: /

User-agent: True_Robot
Disallow: /

User-agent: BlowFish/1.0
Disallow: /

User-agent: JennyBot
Disallow: /

User-agent: MIIxpc/4.2
Disallow: /

User-agent: BuiltBotTough
Disallow: /

User-agent: ProPowerBot/2.14
Disallow: /

User-agent: BackDoorBot/1.0
Disallow: /

User-agent: toCrawl/UrlDispatcher
Disallow: /

User-agent: WebEnhancer
Disallow: /

User-agent: suzuran
Disallow: /

User-agent: TightTwatBot
Disallow: /

User-agent: VCI WebViewer VCI WebViewer Win32
Disallow: /

User-agent: VCI
Disallow: /

User-agent: Szukacz/1.4
Disallow: /

User-agent: QueryN Metasearch
Disallow: /

User-agent: Openfind data gathere
Disallow: /

User-agent: Openfind
Disallow: /

User-agent: RepoMonkey Bait & Tackle/v1.01
Disallow: /

User-agent: RepoMonkey
Disallow: /

User-agent: Openbot
Disallow: /

User-agent: URL Control
Disallow: /

User-agent: Zeus Link Scout
Disallow: /

User-agent: Webster Pro
Disallow: /

User-agent: EroCrawler
Disallow: /

User-agent: LinkScan/8.1a Unix
Disallow: /

User-agent: Keyword Density/0.9
Disallow: /

User-agent: Kenjin Spider
Disallow: /

User-agent: Iron33/1.0.2
Disallow: /

User-agent: Bookmark search tool
Disallow: /

User-agent: GetRight/4.2
Disallow: /

User-agent: FairAd Client
Disallow: /

User-agent: Gaisbot
Disallow: /

User-agent: Aqua_Products
Disallow: /

User-agent: Radiation Retriever 1.1
Disallow: /

User-agent: Flaming AttackBot
Disallow: /

User-agent: Oracle Ultra Search
Disallow: /

User-agent: PerMan
Disallow: /

User-agent: searchpreview
Disallow: /

User-agent: SQLMAP
Disallow: /

# 24/05/2019
Sitemap: https://www.isdin.com/sitemap.xml
Sitemap: https://www.isdin.com/blog/sitemap_index.xml
Sitemap: https://www.isdin.com/en-US/blog/sitemap_index.xml
Sitemap: https://www.isdin.com/en/sitemap_int_en.xml
Sitemap: https://www.isdin.com/sitemap_es_es.xml
Sitemap: https://www.isdin.com/de/sitemap_de_de.xml
Sitemap: https://www.isdin.com/es-AR/sitemap_es_ar.xml
Sitemap: https://www.isdin.com/es-MX/sitemap_es_mx.xml
Sitemap: https://www.isdin.com/es-PE/sitemap_es_pe.xml
Sitemap: https://www.isdin.com/pt-BR/sitemap_pt_br.xml
Sitemap: https://www.isdin.com/en-CA/sitemap_en_ca.xml
Sitemap: https://www.isdin.com/fr-CA/sitemap_fr_ca.xml
Sitemap: https://www.isdin.com/fr-FR/sitemap_fr_fr.xml
Sitemap: https://www.isdin.com/es-CO/sitemap_es_co.xml
Sitemap: https://www.isdin.com/es-CL/sitemap_es_cl.xml
Sitemap: https://www.isdin.com/it/sitemap_it.xml
Sitemap: https://www.isdin.com/pt-PT/sitemap_pt_pt.xml
Sitemap: https://www.isdin.com/fr-BE/sitemap_fr_be.xml
Sitemap: https://www.isdin.com/en-AE/sitemap_en_ae.xml
Sitemap: https://www.isdin.com/en-ZA/sitemap_en_za.xml
Sitemap: https://www.isdin.com/nl-BE/sitemap_nl_be.xml
Sitemap: https://www.isdin.com/en-IN/sitemap_en_in.xml
Sitemap: https://www.isdin.com/ru-RU/sitemap_ru_ru.xml
Sitemap: https://www.isdin.com/ru-BY/sitemap_ru_by.xml
Sitemap: https://www.isdin.com/us/sitemap_us.xml