From 62e80d118f7b335b5e4cdd7dd752041f52c9bea0 Mon Sep 17 00:00:00 2001 From: Colin Brown Date: Tue, 28 Apr 2015 16:10:44 +0000 Subject: [PATCH 1/3] aws backup setting should now get passed through init script --- assets/config/gitlabhq/gitlab.yml | 18 +++++++++--------- assets/init | 20 ++++++++++++++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/assets/config/gitlabhq/gitlab.yml b/assets/config/gitlabhq/gitlab.yml index 292a29b8..b93ac6a1 100644 --- a/assets/config/gitlabhq/gitlab.yml +++ b/assets/config/gitlabhq/gitlab.yml @@ -225,15 +225,15 @@ production: &base backup: path: "{{GITLAB_BACKUP_DIR}}" # Relative paths are relative to Rails.root (default: tmp/backups/) keep_time: {{GITLAB_BACKUP_EXPIRY}} # default: 0 (forever) (in seconds) - # upload: - # # Fog storage connection settings, see http://fog.io/storage/ . - # connection: - # provider: AWS - # region: eu-west-1 - # aws_access_key_id: AKIAKIAKI - # aws_secret_access_key: 'secret123' - # # The remote 'directory' to store your backups. For S3, this would be the bucket name. - # remote_directory: 'my.s3.bucket' + upload: + # Fog storage connection settings, see http://fog.io/storage/ . + connection: + provider: AWS + region: {{AWS_BACKUP_REGION}} + aws_access_key_id: {{AWS_BACKUP_ACCESS_KEY_ID}} + aws_secret_access_key: '{{AWS_BACKUP_SECRET_ACCESS_KEY}}' + # The remote 'directory' to store your backups. For S3, this would be the bucket name. + remote_directory: '{{AWS_BACKUP_BUCKET}}' ## GitLab Shell settings gitlab_shell: diff --git a/assets/init b/assets/init index 212019ab..7ca0ccb1 100755 --- a/assets/init +++ b/assets/init @@ -45,6 +45,12 @@ GITLAB_BACKUPS=${GITLAB_BACKUPS:-disable} GITLAB_BACKUP_TIME=${GITLAB_BACKUP_TIME:-04:00} GITLAB_BACKUP_EXPIRY=${GITLAB_BACKUP_EXPIRY:-} +AWS_BACKUPS=${AWS_BACKUPS:-false} +AWS_BACKUP_REGION=${AWS_BACKUP_REGION} +AWS_BACKUP_ACCESS_KEY_ID=${AWS_BACKUP_ACCESS_KEY_ID} +AWS_BACKUP_SECRET_ACCESS_KEY=${AWS_BACKUP_SECRET_ACCESS_KEY} +AWS_BACKUP_BUCKET=${AWS_BACKUP_BUCKET} + NGINX_WORKERS=${NGINX_WORKERS:-1} NGINX_ACCEL_BUFFERING=${NGINX_ACCEL_BUFFERING:-no} NGINX_PROXY_BUFFERING=${NGINX_PROXY_BUFFERING:-off} @@ -508,6 +514,20 @@ sudo -u git -H sed 's/{{LDAP_BLOCK_AUTO_CREATED_USERS}}/'"${LDAP_BLOCK_AUTO_CREA sudo -u git -H sed 's/{{LDAP_BASE}}/'"${LDAP_BASE}"'/' -i config/gitlab.yml sudo -u git -H sed 's/{{LDAP_USER_FILTER}}/'"${LDAP_USER_FILTER}"'/' -i config/gitlab.yml +# apply aws s3 backup configuration +case "${AWS_BACKUPS}" in + true) + sudo -u git -H sed 's/{{AWS_BACKUP_REGION}}/'"${AWS_BACKUP_REGION}"'/' -i config/gitlab.yml + sudo -u git -H sed 's/{{AWS_BACKUP_ACCESS_KEY_ID}}/'"${AWS_BACKUP_ACCESS_KEY_ID}"'/' -i config/gitlab.yml + sudo -u git -H sed 's,{{AWS_BACKUP_SECRET_ACCESS_KEY}},'"${AWS_BACKUP_SECRET_ACCESS_KEY}"',' -i config/gitlab.yml + sudo -u git -H sed 's/{{AWS_BACKUP_BUCKET}}/'"${AWS_BACKUP_BUCKET}"'/' -i config/gitlab.yml + ;; + *) + # remove backup configuration lines + sudo -u git -H sed /upload:/,/remote_directory:/d -i config/gitlab.yml + ;; +esac + # apply gravatar configuration sudo -u git -H sed 's/{{GITLAB_GRAVATAR_ENABLED}}/'"${GITLAB_GRAVATAR_ENABLED}"'/' -i config/gitlab.yml if [ -n "${GITLAB_GRAVATAR_HTTP_URL}" ]; then From 6ef620a103502cbc64367e404d2afd6df90856c7 Mon Sep 17 00:00:00 2001 From: Colin Brown Date: Tue, 28 Apr 2015 19:10:49 -0500 Subject: [PATCH 2/3] updating readme to include aws backup instructions --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 7dae9117..44ea46c3 100644 --- a/README.md +++ b/README.md @@ -788,6 +788,11 @@ Below is the complete list of available options that can be used to customize yo - **GOOGLE_ANALYTICS_ID**: Google Analytics ID. No defaults. - **PIWIK_URL**: Sets the Piwik URL. No defaults. - **PIWIK_SITE_ID**: Sets the Piwik site ID. No defaults. +- **AWS_BACKUPS**: Enables automatic uploads to an Amazon S3 instance. Defaults to `false`. +- **AWS_BACKUP_REGION**: AWS region. No defaults. +- **AWS_BACKUP_ACCESS_KEY_ID**: AWS access key id. No defaults. +- **AWS_BACKUP_SECRET_ACCESS_KEY**: AWS secret access key. No defaults. +- **AWS_BACKUP_BUCKET**: AWS bucket for backup uploads. No defaults. # Maintenance @@ -844,6 +849,14 @@ Daily backups are created at `GITLAB_BACKUP_TIME` which defaults to `04:00` ever By default, when automated backups are enabled, backups are held for a period of 7 days. While when automated backups are disabled, the backups are held for an infinite period of time. This can behavior can be configured via the `GITLAB_BACKUP_EXPIRY` option. +## Automatic Amazon Web Services (AWS) S3 Uploads + +The image can be configured to automatically upload the backups to an AWS S3 bucket. To enable automatic AWS backups first add `-e 'AWS_BACKUPS=true'` to the docker run command. In addition `AWS_BACKUP_REGION` and `AWS_BACKUP_BUCKET` must be properly configured to point to the desired AWS location. Finally an IAM user must be configured with appropriate access permission and their AWS keys exposed through `AWS_BACKUP_ACCESS_KEY_ID` and `AWS_BACKUP_SECRET_ACCESS_KEY`. + +More details about the appropriate IAM user properties can found on [doc.gitlab.com](http://doc.gitlab.com/ce/raketasks/backup_restore.html#upload-backups-to-remote-cloud-storage) + +AWS uploads are performed alongside normal backups, both through the appropriate `app:rake` command and when an automatic backup is performed. + ## Shell Access For debugging and maintenance purposes you may want access the containers shell. If you are using docker version `1.3.0` or higher you can access a running containers shell using `docker exec` command. From 657ab305dd911a75addf9a43cd5635dbf836ed20 Mon Sep 17 00:00:00 2001 From: Colin Brown Date: Thu, 30 Apr 2015 09:36:45 -0500 Subject: [PATCH 3/3] fail to launch if AWS_BACKUPS=true, but other options are missing --- assets/init | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/assets/init b/assets/init index 7ca0ccb1..4d447236 100755 --- a/assets/init +++ b/assets/init @@ -517,6 +517,10 @@ sudo -u git -H sed 's/{{LDAP_USER_FILTER}}/'"${LDAP_USER_FILTER}"'/' -i config/g # apply aws s3 backup configuration case "${AWS_BACKUPS}" in true) + if [ -z "${AWS_BACKUP_REGION}" -o -z "${AWS_BACKUP_ACCESS_KEY_ID}" -o -z "${AWS_BACKUP_SECRET_ACCESS_KEY}" -o -z "${AWS_BACKUP_BUCKET}" ]; then + printf "\nMissing AWS options. Aborting...\n" + exit 1 + fi sudo -u git -H sed 's/{{AWS_BACKUP_REGION}}/'"${AWS_BACKUP_REGION}"'/' -i config/gitlab.yml sudo -u git -H sed 's/{{AWS_BACKUP_ACCESS_KEY_ID}}/'"${AWS_BACKUP_ACCESS_KEY_ID}"'/' -i config/gitlab.yml sudo -u git -H sed 's,{{AWS_BACKUP_SECRET_ACCESS_KEY}},'"${AWS_BACKUP_SECRET_ACCESS_KEY}"',' -i config/gitlab.yml