Details |
Last modification |
View Log
| Rev |
Author |
Line No. |
Line |
| 1 |
root |
1 |
# exclude help system from robots
|
|
|
2 |
User-agent: *
|
|
|
3 |
Disallow: /manual/
|
|
|
4 |
Disallow: /manual-1.3/
|
|
|
5 |
Disallow: /manual-2.0/
|
|
|
6 |
Disallow: /manual-2.2/
|
|
|
7 |
Disallow: /addon-modules/
|
|
|
8 |
Disallow: /doc/
|
|
|
9 |
Disallow: /images/
|
|
|
10 |
# the next line is a spam bot trap, for grepping the logs. you should _really_ change this to something else...
|
|
|
11 |
Disallow: /all_our_e-mail_addresses
|
|
|
12 |
# same idea here...
|
|
|
13 |
Disallow: /admin/
|
|
|
14 |
# but allow htdig to index our doc-tree
|
|
|
15 |
#User-agent: htdig
|
|
|
16 |
#Disallow:
|
|
|
17 |
# disallow stress test
|
|
|
18 |
user-agent: stress-agent
|
|
|
19 |
Disallow: /
|