From 621c7c38ac29b900a5afab25cfee47e0a0d460c6 Mon Sep 17 00:00:00 2001 From: "endico%mozilla.org" Date: Fri, 28 Jul 2000 21:28:41 +0000 Subject: [PATCH] If the application is set up at the server root (e.g. bugzilla.mozilla.org) then this robots.txt disallows robots from scanning the site except for the top level file. --- webtools/bonsai/robots.txt | 4 ++++ webtools/bugzilla/robots.txt | 3 +++ webtools/lxr/robots.txt | 1 + webtools/tinderbox/robots.txt | 3 +++ 4 files changed, 11 insertions(+) create mode 100644 webtools/bonsai/robots.txt create mode 100644 webtools/bugzilla/robots.txt create mode 100644 webtools/tinderbox/robots.txt diff --git a/webtools/bonsai/robots.txt b/webtools/bonsai/robots.txt new file mode 100644 index 00000000000..60eabc1713a --- /dev/null +++ b/webtools/bonsai/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: /index.html +Allow: /cvsqueryform.cgi +Disallow: / diff --git a/webtools/bugzilla/robots.txt b/webtools/bugzilla/robots.txt new file mode 100644 index 00000000000..c364fe62e9c --- /dev/null +++ b/webtools/bugzilla/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Allow: /index.html +Disallow: / diff --git a/webtools/lxr/robots.txt b/webtools/lxr/robots.txt index 1f53798bb4f..c364fe62e9c 100644 --- a/webtools/lxr/robots.txt +++ b/webtools/lxr/robots.txt @@ -1,2 +1,3 @@ User-agent: * +Allow: /index.html Disallow: / diff --git a/webtools/tinderbox/robots.txt b/webtools/tinderbox/robots.txt new file mode 100644 index 00000000000..c364fe62e9c --- /dev/null +++ b/webtools/tinderbox/robots.txt @@ -0,0 +1,3 @@ +User-agent: * +Allow: /index.html +Disallow: /