#Crawlers Setup
User-agent: *

#Directories
Disallow: /author/
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

#Paths (clean URLs)
#Disallow: 

#Files
Disallow: /wp-cron.php
Disallow: /403.shtml

#Paths (no clean URLs)
#Disallow: /*.js$
#Disallow: /*.css$
Disallow: /*.aspx$
Disallow: /*.ashx$

#old filters
#Disallow: /*price=
#Disallow: /*sizefilter=
Disallow: /*page_id=

Disallow: 
Allow: /

#User-agent: Googlebot-Image
#Disallow: /
#Allow: 

User-agent: Twitterbot
Disallow:
Allow: /

User-agent: Ahrefsbot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: MauiBot
Disallow: /

User-agent: Majestic
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: SEMrushBot
Disallow: /