diff --git a/README.md b/README.md index 89ffdd31..4b204143 100644 --- a/README.md +++ b/README.md @@ -805,6 +805,8 @@ Below is the complete list of available options that can be used to customize yo - **AWS_BACKUP_ACCESS_KEY_ID**: AWS access key id. No defaults. - **AWS_BACKUP_SECRET_ACCESS_KEY**: AWS secret access key. No defaults. - **AWS_BACKUP_BUCKET**: AWS bucket for backup uploads. No defaults. +- **GITLAB_ROBOTS_OVERRIDE**: Override `robots.txt`. Defaults to `false`. +- **GITLAB_ROBOTS_PATH**: Location of `robots.txt`. See [www.robotstxt.org](http://www.robotstxt.org) for examples. Defaults to `robots.txt` which [prevents robots scanning gitlab](http://www.robotstxt.org/faq/prevent.html). # Maintenance diff --git a/assets/config/gitlabhq/robots.txt b/assets/config/gitlabhq/robots.txt new file mode 100644 index 00000000..c6742d8a --- /dev/null +++ b/assets/config/gitlabhq/robots.txt @@ -0,0 +1,2 @@ +User-Agent: * +Disallow: / diff --git a/entrypoint.sh b/entrypoint.sh index 6b089960..aefb4cf5 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -132,6 +132,9 @@ GOOGLE_ANALYTICS_ID=${GOOGLE_ANALYTICS_ID:-} PIWIK_URL=${PIWIK_URL:-} PIWIK_SITE_ID=${PIWIK_SITE_ID:-} +GITLAB_ROBOTS_OVERRIDE=${GITLAB_ROBOTS_OVERRIDE:-false} +GITLAB_ROBOTS_PATH=${GITLAB_ROBOTS_PATH:-$SYSCONF_TEMPLATES_DIR/gitlabhq/robots.txt} + # is a mysql or postgresql database linked? # requires that the mysql or postgresql containers have exposed # port 3306 and 5432 respectively. @@ -290,6 +293,10 @@ sudo -HEu ${GITLAB_USER} cp ${SYSCONF_TEMPLATES_DIR}/gitlabhq/rack_attack.rb [[ ${SMTP_ENABLED} == true ]] && \ sudo -HEu ${GITLAB_USER} cp ${SYSCONF_TEMPLATES_DIR}/gitlabhq/smtp_settings.rb config/initializers/smtp_settings.rb +# allow to override robots.txt to block bots +[[ ${GITLAB_ROBOTS_OVERRIDE} == true ]] && \ +sudo -HEu ${GITLAB_USER} cp ${GITLAB_ROBOTS_PATH} public/robots.txt + # override default configuration templates with user templates case ${GITLAB_HTTPS} in true)