Sitemap: https://www.theinformation.com/sitemap.xml
Sitemap: https://www.theinformation.com/sitemap-news.xml
User-agent: *
Disallow: */1000$
Disallow: /ahoy/*
Disallow: /search*
Disallow: /api*
Disallow: /api_react*
Disallow: /cdn-cgi/rum
Disallow: /company-landing/
Disallow: *?commentId=*
Allow: /api/v1/subscriber_directory/*

# Topic pages
Disallow: /topics/
Allow: /topics/adobe
Allow: /topics/ai
Allow: /topics/airbnb
Allow: /topics/albemarle
Allow: /topics/alphabet
Allow: /topics/amazon
Allow: /topics/andreessen-horowitz
Allow: /topics/anthropic
Allow: /topics/apple
Allow: /topics/aws
Allow: /topics/binance
Allow: /topics/brex
Allow: /topics/bytedance
Allow: /topics/canva
Allow: /topics/chime
Allow: /topics/china
Allow: /topics/circle
Allow: /topics/cisco
Allow: /topics/cloud
Allow: /topics/cohere
Allow: /topics/cohesity
Allow: /topics/coinbase
Allow: /topics/coreweave
Allow: /topics/crypto
Allow: /topics/databricks
Allow: /topics/deel
Allow: /topics/deepmind
Allow: /topics/deepseek
Allow: /topics/didi-chuxing
Allow: /topics/doordash
Allow: /topics/e-commerce
Allow: /topics/enterprise
Allow: /topics/facebook
Allow: /topics/figma
Allow: /topics/flexport
Allow: /topics/google
Allow: /topics/instagram
Allow: /topics/intel
Allow: /topics/klarna
Allow: /topics/lambda
Allow: /topics/lyft
Allow: /topics/markets
Allow: /topics/meta
Allow: /topics/meta-generative-ai
Allow: /topics/microsoft
Allow: /topics/mongodb
Allow: /topics/netflix
Allow: /topics/notion
Allow: /topics/nvidia
Allow: /topics/one
Allow: /topics/openai
Allow: /topics/opendoor
Allow: /topics/oracle
Allow: /topics/palantir
Allow: /topics/patreon
Allow: /topics/pinterest
Allow: /topics/plaid
Allow: /topics/reddit
Allow: /topics/rippling
Allow: /topics/robinhood
Allow: /topics/salesforce
Allow: /topics/sap
Allow: /topics/scale-ai
Allow: /topics/servicenow
Allow: /topics/ses
Allow: /topics/shopify
Allow: /topics/snap
Allow: /topics/snowflake
Allow: /topics/softbank
Allow: /topics/spacex
Allow: /topics/spotify
Allow: /topics/startups
Allow: /topics/stripe
Allow: /topics/temu
Allow: /topics/tencent
Allow: /topics/tesla
Allow: /topics/tiktok
Allow: /topics/twitch
Allow: /topics/uber
Allow: /topics/venture-capital
Allow: /topics/walmart
Allow: /topics/warner-bros-discovery
Allow: /topics/waymo
Allow: /topics/wework
Allow: /topics/xai
Allow: /topics/youtube

# AI Training Bots
User-agent: Google-Extended
User-agent: GPTBot
Disallow: /
Allow: /$
Allow: /pro
Allow: /partners
Allow: /corporate
Allow: /newsletters
Allow: /about
Allow: /jobs
Allow: /events*

# Unknown Crawlers
User-agent: MauiBot
Disallow: /