74 lines
1.3 KiB
Django/Jinja
74 lines
1.3 KiB
Django/Jinja
User-agent: baidu
|
|
crawl-delay: 1
|
|
|
|
{# SEO/SEM #}
|
|
User-agent: SemrushBot
|
|
Disallow: /
|
|
|
|
{# SEO/SEM #}
|
|
User-agent: AhrefsBot
|
|
Disallow: /
|
|
|
|
{# SEO/SEM #}
|
|
User-agent: DataForSeoBot
|
|
Disallow: /
|
|
|
|
{# https://www.ionos.de/terms-gtc/faq-crawler/ #}
|
|
{# disallowed on principle for not having that page in english #}
|
|
User-agent: IonCrawl
|
|
Disallow: /
|
|
|
|
{# SEO/SEM #}
|
|
User-agent: barkrowler
|
|
Disallow: /
|
|
|
|
{# SEO/SEM #}
|
|
User-agent: seoscanners.net
|
|
Disallow: /
|
|
|
|
{# SEO/SEM #}
|
|
User-agent: MegaIndex.ru
|
|
Disallow: /
|
|
User-agent: megaindex.com
|
|
Disallow: /
|
|
|
|
User-agent: *
|
|
|
|
Disallow: /*/pulse
|
|
Disallow: /*/tree/
|
|
Disallow: /gist/
|
|
Disallow: /*/forks
|
|
Disallow: /*/stars
|
|
Disallow: /*/download
|
|
Disallow: /*/revisions
|
|
Disallow: /*/issues/new
|
|
Disallow: /*/issues/search
|
|
Disallow: /*/commits/
|
|
Disallow: /*/commits/*?author
|
|
Disallow: /*/commits/*?path
|
|
Disallow: /*/branches
|
|
Disallow: /*/tags
|
|
Disallow: /*/contributors
|
|
Disallow: /*/comments
|
|
Disallow: /*/stargazers
|
|
Disallow: /*/archive/
|
|
Disallow: /*/blame/
|
|
Disallow: /*/watchers
|
|
Disallow: /*/network
|
|
Disallow: /*/graphs
|
|
Disallow: /*/raw/
|
|
Disallow: /*/compare/
|
|
Disallow: /*/cache/
|
|
Disallow: /.git/
|
|
Disallow: */.git/
|
|
Disallow: /*.git$
|
|
Disallow: /search/advanced
|
|
Disallow: /search
|
|
Disallow: */search
|
|
Disallow: /*q=
|
|
Disallow: /*.atom
|
|
|
|
Disallow: /user/login
|
|
|
|
Sitemap: https://{{ gitea_url }}/sitemap.xml
|