public/robots.txt

@@ -0,0 +1,32 @@
+User-agent: *
+Crawl-delay: 1
+Disallow: /admin/
+Disallow: /api/
+Disallow: /*/*/commit/*
+Disallow: /*/*/tree/*
+Disallow: /*/*/blame/*
+Disallow: /*/*/raw/*
+Disallow: /*/branches
+Disallow: /*/tree/
+Disallow: /*/forks
+Disallow: /*/stars
+Disallow: *download*
+Disallow: /*/revisions
+Disallow: /*/issues/new
+Disallow: /*/issues/search
+Disallow: /*/commits/
+Disallow: /*/tags
+Disallow: /*/contributors/
+Disallow: /*/comments
+Disallow: /*/archive/
+Disallow: /*/blame/
+Disallow: /*/watchers
+Disallow: /*/graphs
+Disallow: /*/raw/
+Disallow: /*/compare/
+Disallow: /.git/
+Disallow: */.git/
+Disallow: /*.git$
+Disallow: /search
+Disallow: */search
+Disallow: /*q=