# START YOAST BLOCK
# ---------------------------
User-agent: *
Disallow:

Sitemap: https://swiss-instasolar.ch/sitemap_index.xml
# ---------------------------
# END YOAST BLOCK

User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Disallow access to sensitive files
Disallow: /readme.html
Disallow: /license.txt
Disallow: /wp-login.php
Disallow: /wp-register.php

# Disallow specific directories that should not be crawled
Disallow: /cgi-bin/
Disallow: /wp-content/plugins/
Disallow: /wp-content/cache/
Disallow: /wp-content/themes/
Disallow: /wp-includes/
Disallow: /wp-content/uploads/
Disallow: /private/
Disallow: /tmp/
Disallow: /backup/

# Block certain file types from being indexed
Disallow: /*.php$
Disallow: /*.cgi$
Disallow: /*.asp$
Disallow: /*.aspx$
Disallow: /*.js$
Disallow: /*.css$


# Allow only specific search engine crawlers full access
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Block known bad bots (example bots, adjust as necessary)
User-agent: Baiduspider
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Baiduspider
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: spbot
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: megaindex
Disallow: /

User-agent: linkdexbot
Disallow: /

User-agent: siteexplorer.info
Disallow: /

User-agent: VelenPublicWebCrawler
Disallow: /

User-agent: Mail.RU_Bot
Disallow: /

User-agent: archive.org_bot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BDCbot
Disallow: /

User-agent: wotbox
Disallow: /

User-agent: BUbiNG
Disallow: /

User-agent: Swiftbot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Cliqzbot
Disallow: /

User-agent: SearchmetricsBot
Disallow: /

User-agent: Dataprovider
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: 360Spider
Disallow: /

User-agent: CensysInspect
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: Yeti
Disallow: /


# Prevent indexing of search results pages
Disallow: /?s=

# Prevent indexing of archives, categories, tags pages
Disallow: /category/
Disallow: /tag/
Disallow: /author/
Disallow: /archive/
Disallow: /page/