User-agent: * disallow: /wp-admin/ allow: /wp-admin/admin-ajax.php disallow: /?s= disallow: /author/*/feed/$ disallow: /recruit/?s= disallow: /recruit/search/{search_term_string}/ disallow: /r/index.php?url= disallow: /*.pdf$ User-agent: Megalodon disallow: / User-agent: ia_archiver disallow: / User-agent: Updownerbot disallow: /