6cd321552c
- Added comprehensive meta tags with 40+ keywords - Created sitemap.ts with all routes - Updated robots.txt for search engine crawlers - Added manifest.json for PWA support - Added JSON-LD structured data for SEO - Enhanced README with: - Table of contents - Star history chart - Better keyword density - Social media links - Tech stack tables - API documentation tables - Added 20 GitHub topics for discoverability - Optimized Open Graph and Twitter cards
47 lines
727 B
Plaintext
47 lines
727 B
Plaintext
# https://www.robotstxt.org/robotstxt.html
|
|
# Sitemap
|
|
Sitemap: https://github.com/roman-ryzenadvanced/mantle-ai-trader/sitemap.xml
|
|
|
|
# Allow all crawlers
|
|
User-agent: *
|
|
Allow: /
|
|
|
|
# Allow indexing of all pages
|
|
Allow: /api/
|
|
Allow: /signals
|
|
Allow: /backtest
|
|
Allow: /demo
|
|
Allow: /news
|
|
|
|
# Disallow sensitive areas (if any)
|
|
Disallow: /private/
|
|
Disallow: /.env
|
|
|
|
# Crawl-delay for respectful crawling
|
|
Crawl-delay: 1
|
|
|
|
# Specific rules for major search engines
|
|
User-agent: Googlebot
|
|
Allow: /
|
|
Crawl-delay: 0
|
|
|
|
User-agent: Bingbot
|
|
Allow: /
|
|
Crawl-delay: 1
|
|
|
|
User-agent: DuckDuckBot
|
|
Allow: /
|
|
Crawl-delay: 1
|
|
|
|
User-agent: Slurp
|
|
Allow: /
|
|
Crawl-delay: 1
|
|
|
|
User-agent: Baiduspider
|
|
Allow: /
|
|
Crawl-delay: 2
|
|
|
|
User-agent: YandexBot
|
|
Allow: /
|
|
Crawl-delay: 2
|