# robots.txt!

User-agent: *

# Less stuff!
Crawl-Delay: 5
Request-rate: 1/5             # one document every 2 seconds

Disallow: /jvs     # testing version of jbovlaste, no reason to index.

Disallow: /jbovlaste/natlang # the really big, non-static parts of
Disallow: /jbovlaste/dict    # jbovlaste. the rest of it ought to
                             # be OK (and in fact, good, to expose).
# jbovlaste.lojban.org versions.
Disallow: /dict
Disallow: /natlang			

Disallow: /phpbb
Disallow: /bpfk

Disallow: /irc

Disallow: /cgi-bin/aegis.cgi
Disallow: /cgi-bin/htsearch
Disallow: /cgi-bin/info2www
Disallow: /cgi-bin/lconline/lc.cgi
Disallow: /cgi-bin/listargate.cgi
Disallow: /cgi-bin/lsg2.cgi
Disallow: /cgi-bin/mserv.cgi
Disallow: /cgi-bin/mserv_main.cgi
Disallow: /cgi-bin/mserv_margin.cgi
Disallow: /cgi-bin/viewcvs-query.cgi
Disallow: /cgi-bin/viewcvs.cgi

Disallow: /lsg2
Disallow: /bugzilla
Disallow: /rt

Disallow: /htdig
Disallow: /story
Disallow: /lists
Disallow: /cvsweb
Disallow: /viewcvs

# Disallow all of the non-"static" parts of twiki
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/poll
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/search
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/testenv
Disallow: /twiki/bin/upload

User-agent: lojban-org-htdig
Disallow: /jvs     # testing version of jbovlaste, no reason to index.

Disallow: /phpbb
Disallow: /bpfk

Disallow: /irc

Disallow: /cgi-bin/aegis.cgi
Disallow: /cgi-bin/htsearch
Disallow: /cgi-bin/info2www
Disallow: /cgi-bin/lconline/lc.cgi
Disallow: /cgi-bin/listargate.cgi
Disallow: /cgi-bin/lsg2.cgi
Disallow: /cgi-bin/mserv.cgi
Disallow: /cgi-bin/mserv_main.cgi
Disallow: /cgi-bin/mserv_margin.cgi
Disallow: /cgi-bin/viewcvs-query.cgi
Disallow: /cgi-bin/viewcvs.cgi

Disallow: /lsg2
Disallow: /bugzilla
Disallow: /rt

Disallow: /htdig
Disallow: /cvsweb
Disallow: /viewcvs

# Disallow all of the non-"static" parts of twiki
Disallow: /twiki/bin/attach
Disallow: /twiki/bin/changes
Disallow: /twiki/bin/edit
Disallow: /twiki/bin/geturl
Disallow: /twiki/bin/installpasswd
Disallow: /twiki/bin/mailnotify
Disallow: /twiki/bin/manage
Disallow: /twiki/bin/oops
Disallow: /twiki/bin/passwd
Disallow: /twiki/bin/poll
Disallow: /twiki/bin/preview
Disallow: /twiki/bin/rdiff
Disallow: /twiki/bin/register
Disallow: /twiki/bin/rename
Disallow: /twiki/bin/save
Disallow: /twiki/bin/search
Disallow: /twiki/bin/statistics
Disallow: /twiki/bin/testenv
Disallow: /twiki/bin/upload

# The rest of this is from tiki's robots.txt verbatim
User-agent: *
Crawl-Delay: 5
Disallow: /tiki/CVS/
Disallow: /tiki/lib/
Disallow: /tiki/doc/
Disallow: /tiki/dump/
Disallow: /tiki/img/
Disallow: /tiki/templates/
Disallow: /tiki/templates_c/
Disallow: /tiki/var/
Disallow: /tiki/lang/
Disallow: /tiki/backups/
Disallow: /tiki/bin/
Disallow: /tiki/db/
Disallow: /tiki/games/
Disallow: /tiki/images/
Disallow: /tiki/maps/
Disallow: /tiki/modules/
Disallow: /tiki/popups/
Disallow: /tiki/temp/
Disallow: /tiki/tests/
Disallow: /tiki/tikimovies/
Disallow: /tiki/tiki-install.php
Disallow: /tiki/tiki-admin.php
Disallow: /tiki/tiki-admin
Disallow: /tiki/tiki-editpage.php
Disallow: /tiki/tiki-print.php
Disallow: /tiki/tiki-print_article.php
Disallow: /tiki/tiki-print_blog_post.php
Disallow: /tiki/tiki-print_multi_pages.php
Disallow: /tiki/tiki-print_pages.php
Disallow: /tiki/tiki-pagehistory.php
Disallow: /tiki/tiki-backlinks.php
Disallow: /tiki/tiki-likepages.php
Disallow: /tiki/tiki-listpages.php
Disallow: /tiki/tiki-lastchanges.php
Disallow: /tiki/tiki-orphan_pages.php
Disallow: /tiki/tiki-register.php
Disallow: /tiki/tiki-login_scr.php
Disallow: /tiki/tiki-view_cache.php
Disallow: /tiki/tiki-user_information.php
Disallow: /tiki/get_strings.php
Disallow: /tiki/tiki-index.php?page=SandBox
Disallow: tiki/CVS/
Disallow: tiki/lib/
Disallow: tiki/doc/
Disallow: tiki/dump/
Disallow: tiki/img/
Disallow: tiki/templates/
Disallow: tiki/templates_c/
Disallow: tiki/var/
Disallow: tiki/lang/
Disallow: tiki/backups/
Disallow: tiki/bin/
Disallow: tiki/db/
Disallow: tiki/games/
Disallow: tiki/images/
Disallow: tiki/maps/
Disallow: tiki/modules/
Disallow: tiki/popups/
Disallow: tiki/temp/
Disallow: tiki/tests/
Disallow: tiki/tikimovies/
Disallow: tiki/tiki-install.php
Disallow: tiki/tiki-admin.php
Disallow: tiki/tiki-admin
Disallow: tiki/tiki-editpage.php
Disallow: tiki/tiki-print.php
Disallow: tiki/tiki-print_article.php
Disallow: tiki/tiki-print_blog_post.php
Disallow: tiki/tiki-print_multi_pages.php
Disallow: tiki/tiki-print_pages.php
Disallow: tiki/tiki-pagehistory.php
Disallow: tiki/tiki-backlinks.php
Disallow: tiki/tiki-likepages.php
Disallow: tiki/tiki-listpages.php
Disallow: tiki/tiki-lastchanges.php
Disallow: tiki/tiki-orphan_pages.php
Disallow: tiki/tiki-register.php
Disallow: tiki/tiki-login_scr.php
Disallow: tiki/tiki-view_cache.php
Disallow: tiki/tiki-user_information.php
Disallow: tiki/get_strings.php
Disallow: tiki/tiki-index.php?page=SandBox
Disallow: tiki/tiki-calendar.php
Disallow: tiki/tiki-browse_categories.php

Disallow: /tiki/tiki-download
Disallow: tiki/tiki-download