Add in robots.txt for Pocket mode, based on current state of https://getpocket.com/robots.txt

bedrock/pocket/templates/pocket/robots.txt

@@ -0,0 +1,42 @@
+User-agent: *
+Disallow: /v2/*
+
+User-agent: *
+Disallow: /v3/*
+
+User-agent: *
+Disallow: /create*
+
+User-agent: *
+Disallow: /mini_login*
+
+User-agent: *
+Disallow: /button*
+
+User-agent: *
+Disallow: /addemail*
+
+User-agent: *
+Disallow: /redirect*
+
+User-agent: *
+Disallow: /email_unsubscribe*
+
+User-agent: *
+Disallow: /firefox/new_tab_learn_more*
+
+User-agent: *
+Disallow: /s/*
+
+User-agent: *
+Disallow: /@*
+
+User-agent: *
+Disallow: /read/*
+
+User-agent: *
+Disallow: /edit*
+Disallow: /save*
+# Crawl-delay is non-standard and is interpreted differently between different
+# search engines. 2 *should* be a low enough value to not disrupt our current SEO
+Crawl-delay: 2

bedrock/urls/pocket_mode.py

@@ -5,6 +5,7 @@
 from django.conf import settings
 from django.urls import include, path
 from django.utils.module_loading import import_string
+from django.views.generic import TemplateView
 
 from watchman import views as watchman_views
 
@@ -21,6 +22,14 @@ urlpatterns = (
     path("healthz/", watchman_views.ping, name="watchman.ping"),
     path("readiness/", watchman_views.status, name="watchman.status"),
     path("healthz-cron/", base_views.cron_health_check),
+    path(
+        "robots.txt",
+        TemplateView.as_view(
+            content_type="text/plain",
+            template_name="pocket/robots.txt",
+        ),
+        name="robots.txt",
+    ),
 )
 
 if settings.DEBUG: