diff --git a/roles/authelia/tasks/authelia.yml b/roles/authelia/tasks/authelia.yml index 5af9891..a804d35 100644 --- a/roles/authelia/tasks/authelia.yml +++ b/roles/authelia/tasks/authelia.yml @@ -57,6 +57,7 @@ mode: 0755 tags: - authelia-nginx + - authelia-robots notify: reload nginx - name: template nginx vhost diff --git a/roles/gitea/tasks/gitea.yml b/roles/gitea/tasks/gitea.yml index b074fd6..72bbf1b 100644 --- a/roles/gitea/tasks/gitea.yml +++ b/roles/gitea/tasks/gitea.yml @@ -61,6 +61,7 @@ - sitemap.xml tags: - robots + - gitea-robots - name: copy gitea templates copy: diff --git a/roles/gitea/templates/robots.txt.j2 b/roles/gitea/templates/robots.txt.j2 index 3af3651..75c3614 100644 --- a/roles/gitea/templates/robots.txt.j2 +++ b/roles/gitea/templates/robots.txt.j2 @@ -1,6 +1,37 @@ User-agent: baidu crawl-delay: 1 +{# SEO/SEM #} +User-agent: SemrushBot +Disallow: / + +{# SEO/SEM #} +User-agent: AhrefsBot +Disallow: / + +{# SEO/SEM #} +User-agent: DataForSeoBot +Disallow: / + +{# https://www.ionos.de/terms-gtc/faq-crawler/ #} +{# disallowed on principle for not having that page in english #} +User-agent: IonCrawl +Disallow: / + +{# SEO/SEM #} +User-agent: barkrowler +Disallow: / + +{# SEO/SEM #} +User-agent: seoscanners.net +Disallow: / + +{# SEO/SEM #} +User-agent: MegaIndex.ru +Disallow: / +User-agent: megaindex.com +Disallow: / + User-agent: * Disallow: /*/pulse diff --git a/roles/jellyfin/tasks/jellyfin.yml b/roles/jellyfin/tasks/jellyfin.yml index 536480b..03e9f2f 100644 --- a/roles/jellyfin/tasks/jellyfin.yml +++ b/roles/jellyfin/tasks/jellyfin.yml @@ -209,6 +209,17 @@ daemon_reload: true name: jellyfin_auth +- name: template robots.txt + template: + src: "robots.txt.j2" + dest: "{{ systemuserlist.jellyfin.home }}/robots.txt" + owner: jellyfin + group: jellyfin + mode: 0755 + tags: + - jellyfin-nginx + - jellyfin-robots + notify: reload nginx - name: template jellyfin nginx config template: diff --git a/roles/jellyfin/templates/01-jellyfin.j2 b/roles/jellyfin/templates/01-jellyfin.j2 index 54eb0b1..23f4a25 100644 --- a/roles/jellyfin/templates/01-jellyfin.j2 +++ b/roles/jellyfin/templates/01-jellyfin.j2 @@ -69,6 +69,14 @@ server { return 302 https://$host/web/; } + # jellyfin-web already has a robots.txt file that disallows everything, but we still want to maintain our own. + # jellyfin (not -web) will issue a 302 redirect from {{ jellyfin_url }}/robots.txt to {{ jellyfin_url }}/web/robots.txt + # where the file is then served from. + # + # https://github.com/jellyfin/jellyfin-web/blob/master/src/robots.txt + location = /robots.txt { + alias {{ systemuserlist.jellyfin.home }}/robots.txt; + } location / { #include /etc/nginx/jellyfin/jellyfin_auth.conf; diff --git a/roles/jellyfin/templates/robots.txt.j2 b/roles/jellyfin/templates/robots.txt.j2 new file mode 100644 index 0000000..1f53798 --- /dev/null +++ b/roles/jellyfin/templates/robots.txt.j2 @@ -0,0 +1,2 @@ +User-agent: * +Disallow: /