#### Robots.txt 
# ========================================================
# This file should not be used to remove pages that are already in google's index.
# In general, only use it for files or pages that search engines should never see, or can significantly impact crawling

## Documentation
# Different search engines interpret directives differently. By default, the first matching directive always wins. But, with Google and Bing, specificity wins.
# Google docs - https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt



### Every bot that might possibly read and respect this file
# ========================================================
User-agent: *

### No SEO value
Disallow: /*/partners/*
Disallow: /partners/*
Disallow: /sgtm/*

### Query param management
# https://www.peakhour.io/blog/how-to-exclude-query-string-parameters-from-search-engines-using-robots-txt/
Disallow: /*/account/completeResetPassword*?*
Disallow: /*/travel-review*?*


### Old static files that are still live
# Directory where humans upload one-off sitemaps, generally to get google to fix an issue faster
Allow: /guides/sitemap-manual*
Disallow: /guides/


### Legacy files/pages that are thought to no longer be live
Disallow: /images/
Disallow: /bus-travel-review/wp-admin/
Disallow: /bus-travel-review/wp-content/plugins/
Disallow: /bus-travel-review/wp-comments-post.php
Disallow: /check/
Disallow: /static/
Disallow: /downtime.php
Disallow: /search/
Disallow: /cheap-unicorn-tickets/
Disallow: /maps/stationMap
Disallow: /popups/
Disallow: /app/css/
Disallow: /fr/billets-de-autobus/
Disallow: /fr/billets-de-train/
Disallow: /fr/stations/
Disallow: /fr/autobuses-et-trains/


### Wordpress
# Required by WP for ajax. https://developer.wordpress.org/plugins/javascript/ajax/#url
Allow: /blog/wp-admin/admin-ajax.php
Disallow: /blog/wp-admin/


### Internet Standards
# https://en.wikipedia.org/wiki/Well-known_URI
Disallow: /.well-known/*
# https://support.google.com/adsense/answer/12171612?hl=en
Disallow: /ads.txt



# Bad or Unhelpful Bots
# ==========================
User-agent: YandexBot
User-agent: Sogou blog
User-agent: Sogou inst spider
User-agent: Sogou News Spider
User-agent: Sogou Orion spider
User-agent: Sogou spider2
User-agent: Sogou web spider
User-agent: AspiegelBot
User-agent: BLEXBot
User-agent: MJ12bot
User-agent: serpstatbot
User-agent: HubSpot Crawler
Disallow: /


# Independent of user agent. Links in the sitemap are full URLs using https:// and need to match
# the protocol of the sitemap.
Sitemap: https://www.wanderu.com/sitemap.xml
Sitemap: https://www.wanderu.com/blog/sitemap_index.xml