User-agent: *
# Disallow listed BEFORE Allow on purpose. Google resolves conflicts by longest
# path match (so order is irrelevant to it), but simpler parsers take the first
# matching rule — with `Allow: /` first those would hand out /admin.html.
Disallow: /admin.html
Allow: /

# /job.html, /company.html, and the other SPA fallbacks are intentionally
# noindexed via meta tags, NOT Disallowed here. Per Google's docs, combining
# robots.txt Disallow with a noindex meta is counterproductive: the page can't
# be crawled, so the noindex is never seen. Letting Google crawl them and read
# the noindex,follow meta is the cleanest signal — it routes link equity
# through to the canonical prerendered URLs (/jobs/{domain}/{id}/, etc).
#
# The same reasoning applies to the ~88K expired job pages: they carry
# `noindex, follow` and must stay crawlable so search engines actually SEE
# that. Never Disallow /jobs/.

# ---------------------------------------------------------------------------
# Blocked: commercial SEO backlink/rank crawlers  (added 2026-07-29)
#
# Measured from CloudFront access logs over a 4h window: AhrefsBot 1,479
# requests (17% of ALL site traffic, 46% of them 404s) and SemrushBot 227 —
# together ~20% of traffic for zero return. We are not a customer of these
# tools, they send no referral traffic, and they crawl us mainly so other
# people can research us. All of them honor robots.txt.
#
# NOTE: the site-audit crawlers stay allowed, so we can still audit our own
# property. AhrefsSiteAudit falls through to the * group on its own, but
# SemrushBot-SA does NOT — robots.txt group matching is longest-prefix, so
# `SemrushBot` would capture `SemrushBot-SA` too. It therefore needs its own
# explicit Allow group below. (Verified with a parser; without that group it
# was blocked.)
# ---------------------------------------------------------------------------
User-agent: AhrefsBot
Disallow: /

# SemrushBot-SA listed BEFORE SemrushBot: Google picks the longest-matching
# group (so -SA wins either way), but first-match parsers would otherwise see
# `SemrushBot` as a substring of `SemrushBot-SA` and block our own site audit.
User-agent: SemrushBot-SA
Allow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: rogerbot
Disallow: /

User-agent: Barkrowler
Disallow: /

User-agent: serpstatbot
Disallow: /

User-agent: SEOkicks
Disallow: /

# ---------------------------------------------------------------------------
# Blocked: AI *training* crawlers  (added 2026-07-29)
#
# meta-externalagent was our single largest consumer at 2,318 requests in the
# same 4h window — 26% of all traffic, more than Bing and 40x Googlebot,
# almost entirely on /jobs/ pages. These crawlers harvest the corpus for model
# training and return nothing. The sanctioned way to consume HireJack data
# programmatically is the MCP server (hirejack.com/claude): authenticated,
# rate-limited, attributed. Scraping the corpus for training is not.
#
# Google-Extended and Applebot-Extended are the AI-training opt-out tokens for
# Google and Apple. Blocking them does NOT affect Googlebot or Applebot —
# Search and Siri indexing are untouched.
#
# ClaudeBot is listed on purpose even though we promote the Claude integration:
# MCP is an API, not a crawl, so the integration is unaffected. The position is
# "use our MCP, don't scrape us." Reverse this one line if that ever changes.
# ---------------------------------------------------------------------------
User-agent: meta-externalagent
Disallow: /

User-agent: FacebookBot
Disallow: /

User-agent: GPTBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: anthropic-ai
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgilibot
Disallow: /

User-agent: omgili
Disallow: /

User-agent: Timpibot
Disallow: /

User-agent: PanguBot
Disallow: /

# ---------------------------------------------------------------------------
# DELIBERATELY STILL ALLOWED — do not "helpfully" add these to the block list.
#
#   Googlebot, bingbot, YandexBot, DuckDuckBot, Applebot, Amazonbot
#     search/assistant indexing. bingbot is currently our BEST crawler by a
#     wide margin — 1,972 requests vs Googlebot's 57 in the same window.
#
#   ChatGPT-User, Claude-User, PerplexityBot, OAI-SearchBot
#     these fire when a real person asks an assistant about one of our pages,
#     or power cited answers. Potential traffic, not harvesting — and blocking
#     them would contradict the entire "HireJack for Claude" pitch.
#
#   facebookexternalhit, Twitterbot, LinkedInBot, Slackbot, Discordbot
#     link-preview unfurlers. Blocking these breaks the share cards on every
#     social post and every OG image we generate.
# ---------------------------------------------------------------------------

Sitemap: https://hirejack.com/sitemap.xml