From 28bcedf7ebbdd41292365c668c7faaa9592c57e9 Mon Sep 17 00:00:00 2001 From: Sameer Naik Date: Thu, 24 Sep 2015 12:02:51 +0530 Subject: [PATCH] gitlab: upgrade to CE v.8.0.0 --- Changelog.md | 2 + Dockerfile | 3 +- README.md | 10 +++++ assets/config/gitlabhq/database.yml | 3 ++ assets/config/gitlabhq/gitlab.yml | 59 ++++++++++++++++++++++--- assets/config/gitlabhq/rack_attack.rb | 15 ++++--- assets/config/gitlabhq/secrets.yml | 12 +++++ assets/config/gitlabhq/smtp_settings.rb | 8 +++- assets/config/nginx/gitlab | 37 ++++++++++++++++ assets/config/nginx/gitlab-ssl | 37 ++++++++++++++++ assets/setup/install.sh | 37 +++++++++++++++- docker-compose.yml.dist | 1 + entrypoint.sh | 57 +++++++++++++++++------- kubernetes/gitlab-controller.yml | 3 ++ 14 files changed, 252 insertions(+), 32 deletions(-) create mode 100644 assets/config/gitlabhq/secrets.yml diff --git a/Changelog.md b/Changelog.md index 3326dedb..0a667125 100644 --- a/Changelog.md +++ b/Changelog.md @@ -6,6 +6,8 @@ This file only reflects the changes that are made in the the docker image. Pleas - set default value of `DB_TYPE` to `postgres` - added sample Kubernetes rc and service description files - expose `GITLAB_BACKUP_ARCHIVE_PERMISSIONS` parameter +- gitlab: upgrade to CE v.8.0.0 +- added `GITLAB_SECRETS_DB_KEY_BASE` parameter **7.14.3** - gitlab: upgrade to CE v.7.14.3 diff --git a/Dockerfile b/Dockerfile index a6903fbb..d2033de7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM sameersbn/ubuntu:14.04.20150825 MAINTAINER sameer@damagehead.com -ENV GITLAB_VERSION=7.14.3 \ +ENV GITLAB_VERSION=8.0.0 \ GITLAB_SHELL_VERSION=2.6.5 \ GITLAB_USER="git" \ GITLAB_HOME="/home/git" \ @@ -11,6 +11,7 @@ ENV GITLAB_VERSION=7.14.3 \ ENV GITLAB_INSTALL_DIR="${GITLAB_HOME}/gitlab" \ GITLAB_SHELL_INSTALL_DIR="${GITLAB_HOME}/gitlab-shell" \ + GITLAB_GIT_HTTP_SERVER_INSTALL_DIR="${GITLAB_HOME}/gitlab-git-http-server" \ GITLAB_DATA_DIR="${GITLAB_HOME}/data" RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv E1DD270288B4E6030699E45FA1715D88E1DF1F24 \ diff --git a/README.md b/README.md index 9165c7de..7024925b 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,13 @@ The quickest way to get started is using [docker-compose](https://docs.docker.co ```bash wget https://raw.githubusercontent.com/sameersbn/docker-gitlab/master/docker-compose.yml.dist -O docker-compose.yml +``` + +Generate a random string and assign to `GITLAB_SECRETS_DB_KEY_BASE` environment variable. Once set you should not change this value and ensure you keep a backup of this value. + +> **Tip**: You can generate a random string using `pwgen -Bsv1 64` and assign it as the value of `GITLAB_SECRETS_DB_KEY_BASE`. + +```bash docker-compose up ``` @@ -165,6 +172,7 @@ docker run --name gitlab -d \ --link gitlab-postgresql:postgresql --link gitlab-redis:redisio \ --publish 10022:22 --publish 10080:80 \ --env 'GITLAB_PORT=10080' --env 'GITLAB_SSH_PORT=10022' \ + --env 'GITLAB_SECRETS_DB_KEY_BASE=long-and-random-alpha-numeric-string' \ --volume /srv/docker/gitlab/gitlab:/home/git/data \ sameersbn/gitlab:7.14.3 ``` @@ -708,6 +716,7 @@ Below is the complete list of available options that can be used to customize yo - **DEBUG_ENTRYPOINT**: Set this to `true` to enable entrypoint debugging. - **GITLAB_HOST**: The hostname of the GitLab server. Defaults to `localhost` - **GITLAB_PORT**: The port of the GitLab server. This value indicates the public port on which the GitLab application will be accessible on the network and appropriately configures GitLab to generate the correct urls. It does not affect the port on which the internal nginx server will be listening on. Defaults to `443` if `GITLAB_HTTPS=true`, else defaults to `80`. +- **GITLAB_SECRETS_DB_KEY_BASE**: Used to encrypt build variables. Ensure that you don't lose it. You can generate one using `pwgen -Bsv1 64`. If you are migrating from GitLab CI, you need to set this value to the value of `GITLAB_CI_SECRETS_DB_KEY_BASE`. No defaults. - **GITLAB_TIMEZONE**: Configure the timezone for the gitlab application. This configuration does not effect cron jobs. Defaults to `UTC`. See the list of [acceptable values](http://api.rubyonrails.org/classes/ActiveSupport/TimeZone.html). - **GITLAB_ROOT_PASSWORD**: The password for the root user. Defaults to `5iveL!fe`. - **GITLAB_EMAIL**: The email address for the GitLab server. Defaults to `example@example.com`. @@ -725,6 +734,7 @@ Below is the complete list of available options that can be used to customize yo - **GITLAB_TIMEOUT**: Sets the timeout for git commands. Defaults to `10` seconds. - **GITLAB_REPOS_DIR**: The git repositories folder in the container. Defaults to `/home/git/data/repositories` - **GITLAB_BACKUP_DIR**: The backup folder in the container. Defaults to `/home/git/data/backups` +- **GITLAB_BUILDS_DIR**: The build traces directory. Defaults to `/home/git/data/builds` - **GITLAB_BACKUPS**: Setup cron job to automatic backups. Possible values `disable`, `daily`, `weekly` or `monthly`. Disabled by default - **GITLAB_BACKUP_EXPIRY**: Configure how long (in seconds) to keep backups before they are deleted. By default when automated backups are disabled backups are kept forever (0 seconds), else the backups expire in 7 days (604800 seconds). - **GITLAB_BACKUP_ARCHIVE_PERMISSIONS**: Sets the permissions of the backup archives. Defaults to `0600`. [See](http://doc.gitlab.com/ce/raketasks/backup_restore.html#backup-archive-permissions) diff --git a/assets/config/gitlabhq/database.yml b/assets/config/gitlabhq/database.yml index 912674a4..78fd2baa 100644 --- a/assets/config/gitlabhq/database.yml +++ b/assets/config/gitlabhq/database.yml @@ -1,3 +1,6 @@ +# +# PRODUCTION +# production: adapter: "{{DB_ADAPTER}}" encoding: "{{DB_ENCODING}}" diff --git a/assets/config/gitlabhq/gitlab.yml b/assets/config/gitlabhq/gitlab.yml index db2a5bad..1d34216f 100644 --- a/assets/config/gitlabhq/gitlab.yml +++ b/assets/config/gitlabhq/gitlab.yml @@ -4,7 +4,7 @@ # ########################### NOTE ##################################### # This file should not receive new settings. All configuration options # -# that do not require application restart are being moved to # +# that do not require an application restart are being moved to # # ApplicationSetting model! # # If you change this file in a Merge Request, please also create # # a MR on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests # @@ -94,16 +94,39 @@ production: &base # The default is 'tmp/repositories' relative to the root of the Rails app. # repository_downloads_path: tmp/repositories + ## Reply by email + # Allow users to comment on issues and merge requests by replying to notification emails. + # For documentation on how to set this up, see http://doc.gitlab.com/ce/incoming_email/README.html + incoming_email: + enabled: false + address: "incoming+%{key}@gitlab.example.com" + ## Gravatar ## For Libravatar see: http://doc.gitlab.com/ce/customization/libravatar.html gravatar: enabled: {{GITLAB_GRAVATAR_ENABLED}} # Use user avatar image from Gravatar.com (default: true) - # gravatar urls: possible placeholders: %{hash} %{size} %{email} + # plain_url: "http://..." # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon plain_url: "{{GITLAB_GRAVATAR_HTTP_URL}}" # default: http://www.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon ssl_url: "{{GITLAB_GRAVATAR_HTTPS_URL}}" # default: https://secure.gravatar.com/avatar/%{hash}?s=%{size}&d=identicon # - # 2. Auth settings + # 2. GitLab CI settings + # ========================== + + gitlab_ci: + # Default project notifications settings: + # + # Send emails only on broken builds (default: true) + all_broken_builds: true + # + # Add pusher to recipients list (default: false) + add_pusher: true + + # The location where build traces are stored (default: builds/). Relative paths are relative to Rails.root + builds_path: "{{GITLAB_BUILDS_DIR}}" + + # + # 3. Auth settings # ========================== ## LDAP settings @@ -173,6 +196,26 @@ production: &base # user_filter: "{{LDAP_USER_FILTER}}" + # LDAP attributes that GitLab will use to create an account for the LDAP user. + # The specified attribute can either be the attribute name as a string (e.g. 'mail'), + # or an array of attribute names to try in order (e.g. ['mail', 'email']). + # Note that the user's LDAP login will always be the attribute specified as `uid` above. + attributes: + # The username will be used in paths for the user's own projects + # (like `gitlab.example.com/username/project`) and when mentioning + # them in issues, merge request and comments (like `@username`). + # If the attribute specified for `username` contains an email address, + # the GitLab username will be the part of the email address before the '@'. + username: ['uid', 'userid', 'sAMAccountName'] + email: ['mail', 'email', 'userPrincipalName'] + + # If no full name could be found at the attribute specified for `name`, + # the full name is determined using the attributes specified for + # `first_name` and `last_name`. + name: 'cn' + first_name: 'givenName' + last_name: 'sn' + # GitLab EE only: add more LDAP servers # Choose an ID made of a-z and 0-9 . This ID will be stored in the database # so that GitLab can remember which LDAP server a user belongs to. @@ -230,12 +273,17 @@ production: &base idp_sso_target_url: '{{OAUTH_SAML_IDP_SSO_TARGET_URL}}', issuer: '{{OAUTH_SAML_ISSUER}}', name_identifier_format: '{{OAUTH_SAML_NAME_IDENTIFIER_FORMAT}}' } } + # - { name: 'crowd', + # args: { + # crowd_server_url: 'CROWD SERVER URL', + # application_name: 'YOUR_APP_NAME', + # application_password: 'YOUR_APP_PASSWORD' } } # - # 3. Advanced settings + # 4. Advanced settings # ========================== # GitLab Satellites @@ -249,6 +297,7 @@ production: &base path: "{{GITLAB_BACKUP_DIR}}" # Relative paths are relative to Rails.root (default: tmp/backups/) archive_permissions: {{GITLAB_BACKUP_ARCHIVE_PERMISSIONS}} # Permissions for the resulting backup.tar file (default: 0600) keep_time: {{GITLAB_BACKUP_EXPIRY}} # default: 0 (forever) (in seconds) + # pg_schema: public # default: nil, it means that all schemas will be backed up upload: # Fog storage connection settings, see http://fog.io/storage/ . connection: @@ -294,7 +343,7 @@ production: &base timeout: {{GITLAB_TIMEOUT}} # - # 4. Extra customization + # 5. Extra customization # ========================== extra: diff --git a/assets/config/gitlabhq/rack_attack.rb b/assets/config/gitlabhq/rack_attack.rb index 332865d2..2155ea14 100644 --- a/assets/config/gitlabhq/rack_attack.rb +++ b/assets/config/gitlabhq/rack_attack.rb @@ -1,15 +1,16 @@ # 1. Rename this file to rack_attack.rb # 2. Review the paths_to_be_protected and add any other path you need protecting # +# If you change this file in a Merge Request, please also create a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests paths_to_be_protected = [ - "#{Rails.application.config.relative_url_root}/users/password", - "#{Rails.application.config.relative_url_root}/users/sign_in", - "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session.json", - "#{Rails.application.config.relative_url_root}/api/#{API::API.version}/session", - "#{Rails.application.config.relative_url_root}/users", - "#{Rails.application.config.relative_url_root}/users/confirmation", - "#{Rails.application.config.relative_url_root}/unsubscribes/" + "#{Gitlab::Application.config.relative_url_root}/users/password", + "#{Gitlab::Application.config.relative_url_root}/users/sign_in", + "#{Gitlab::Application.config.relative_url_root}/api/#{API::API.version}/session.json", + "#{Gitlab::Application.config.relative_url_root}/api/#{API::API.version}/session", + "#{Gitlab::Application.config.relative_url_root}/users", + "#{Gitlab::Application.config.relative_url_root}/users/confirmation", + "#{Gitlab::Application.config.relative_url_root}/unsubscribes/" ] diff --git a/assets/config/gitlabhq/secrets.yml b/assets/config/gitlabhq/secrets.yml new file mode 100644 index 00000000..41986bf8 --- /dev/null +++ b/assets/config/gitlabhq/secrets.yml @@ -0,0 +1,12 @@ +production: + # db_key_base is used to encrypt for Variables. Ensure that you don't lose it. + # If you change or lose this key you will be unable to access variables stored in database. + # Make sure the secret is at least 30 characters and all random, + # no regular words or you'll be exposed to dictionary attacks. + db_key_base: {{GITLAB_SECRETS_DB_KEY_BASE}} + +development: + db_key_base: development + +test: + db_key_base: test diff --git a/assets/config/gitlabhq/smtp_settings.rb b/assets/config/gitlabhq/smtp_settings.rb index d7e3e5f9..81c79a81 100644 --- a/assets/config/gitlabhq/smtp_settings.rb +++ b/assets/config/gitlabhq/smtp_settings.rb @@ -1,8 +1,12 @@ -# To enable smtp email delivery for your GitLab instance do next: +# To enable smtp email delivery for your GitLab instance do the following: # 1. Rename this file to smtp_settings.rb # 2. Edit settings inside this file # 3. Restart GitLab instance # +# For full list of options and their values see http://api.rubyonrails.org/classes/ActionMailer/Base.html +# +# If you change this file in a Merge Request, please also create a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests + if Rails.env.production? Gitlab::Application.config.action_mailer.delivery_method = :smtp @@ -13,8 +17,8 @@ if Rails.env.production? password: "{{SMTP_PASS}}", domain: "{{SMTP_DOMAIN}}", authentication: "{{SMTP_AUTHENTICATION}}", - openssl_verify_mode: "{{SMTP_OPENSSL_VERIFY_MODE}}", enable_starttls_auto: {{SMTP_STARTTLS}}, + openssl_verify_mode: "{{SMTP_OPENSSL_VERIFY_MODE}}", ca_path: "{{SMTP_CA_PATH}}", ca_file: "{{SMTP_CA_FILE}}", tls: {{SMTP_TLS}} diff --git a/assets/config/nginx/gitlab b/assets/config/nginx/gitlab index 19989885..2ede17b5 100644 --- a/assets/config/nginx/gitlab +++ b/assets/config/nginx/gitlab @@ -38,8 +38,16 @@ upstream gitlab { server unix:{{GITLAB_INSTALL_DIR}}/tmp/sockets/gitlab.socket fail_timeout=0; } +upstream gitlab-git-http-server { + server unix:{{GITLAB_INSTALL_DIR}}/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +} + ## Normal HTTP host server { + ## Either remove "default_server" from the listen line below, + ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab + ## to be served if you visit any address that your server responds to, eg. + ## the ip address of the server (http://x.x.x.x/)n 0.0.0.0:80 default_server; listen 0.0.0.0:80 default_server; listen [::]:80 default_server; server_name {{YOUR_SERVER_FQDN}}; ## Replace this with something like gitlab.example.com @@ -108,6 +116,35 @@ server { proxy_pass http://gitlab; } + location ~ [-\/\w\.]+\.git\/ { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + # gzip off; + add_header X-Accel-Buffering {{NGINX_ACCEL_BUFFERING}}; + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + # Do not buffer Git HTTP responses + proxy_buffering {{NGINX_PROXY_BUFFERING}}; + + # The following settings only work with NGINX 1.7.11 or newer + # + # # Pass chunked request bodies to gitlab-git-http-server as-is + # proxy_request_buffering off; + # proxy_http_version 1.1; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto {{NGINX_X_FORWARDED_PROTO}}; + + proxy_pass http://gitlab-git-http-server; + } + ## Enable gzip compression as per rails guide: ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression ## WARNING: If you are using relative urls remove the block below diff --git a/assets/config/nginx/gitlab-ssl b/assets/config/nginx/gitlab-ssl index afd1a3c0..81b1cfe4 100644 --- a/assets/config/nginx/gitlab-ssl +++ b/assets/config/nginx/gitlab-ssl @@ -42,8 +42,16 @@ upstream gitlab { server unix:{{GITLAB_INSTALL_DIR}}/tmp/sockets/gitlab.socket fail_timeout=0; } +upstream gitlab-git-http-server { + server unix:{{GITLAB_INSTALL_DIR}}/tmp/sockets/gitlab-git-http-server.socket fail_timeout=0; +} + ## Redirects all HTTP traffic to the HTTPS host server { + ## Either remove "default_server" from the listen line below, + ## or delete the /etc/nginx/sites-enabled/default file. This will cause gitlab + ## to be served if you visit any address that your server responds to, eg. + ## the ip address of the server (http://x.x.x.x/) listen 0.0.0.0:80; listen [::]:80 default_server; server_name _; ## Replace this with something like gitlab.example.com @@ -158,6 +166,35 @@ server { proxy_pass http://gitlab; } + location ~ [-\/\w\.]+\.git\/ { + ## If you use HTTPS make sure you disable gzip compression + ## to be safe against BREACH attack. + gzip off; + add_header X-Accel-Buffering {{NGINX_ACCEL_BUFFERING}}; + + ## https://github.com/gitlabhq/gitlabhq/issues/694 + ## Some requests take more than 30 seconds. + proxy_read_timeout 300; + proxy_connect_timeout 300; + proxy_redirect off; + + # Do not buffer Git HTTP responses + proxy_buffering {{NGINX_PROXY_BUFFERING}}; + + # The following settings only work with NGINX 1.7.11 or newer + # + # # Pass chunked request bodies to gitlab-git-http-server as-is + # proxy_request_buffering off; + # proxy_http_version 1.1; + + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-Ssl on; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto {{NGINX_X_FORWARDED_PROTO}}; + proxy_pass http://gitlab-git-http-server; + } + ## Enable gzip compression as per rails guide: ## http://guides.rubyonrails.org/asset_pipeline.html#gzip-compression ## WARNING: If you are using relative urls remove the block below diff --git a/assets/setup/install.sh b/assets/setup/install.sh index 198ad42e..846faa55 100755 --- a/assets/setup/install.sh +++ b/assets/setup/install.sh @@ -3,12 +3,16 @@ set -e GEM_CACHE_DIR="${SETUP_DIR}/cache" +# add golang1.5 ppa +apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv B0B8B106A0CA2F79FBB616DBA65E2E5D742A38EE +echo "deb http://ppa.launchpad.net/evarlast/golang1.5/ubuntu trusty main" >> /etc/apt/sources.list + # rebuild apt cache apt-get update # install build dependencies for gem installation apt-get install -y gcc g++ make patch pkg-config cmake paxctl \ - libc6-dev ruby2.1-dev \ + libc6-dev ruby2.1-dev golang-go \ libmysqlclient-dev libpq-dev zlib1g-dev libyaml-dev libssl-dev \ libgdbm-dev libreadline-dev libncurses5-dev libffi-dev \ libxml2-dev libxslt-dev libcurl4-openssl-dev libicu-dev @@ -47,6 +51,11 @@ cd ${GITLAB_SHELL_INSTALL_DIR} sudo -u git -H cp -a config.yml.example config.yml sudo -u git -H ./bin/install +echo "Cloning gitlab-git-http-server..." +sudo -u git -H git clone -q https://gitlab.com/gitlab-org/gitlab-git-http-server.git --depth 1 ${GITLAB_GIT_HTTP_SERVER_INSTALL_DIR} +cd ${GITLAB_GIT_HTTP_SERVER_INSTALL_DIR} +sudo -u git -H make + # shallow clone gitlab-ce echo "Cloning gitlab-ce v.${GITLAB_VERSION}..." sudo -HEu ${GITLAB_USER} git clone -q -b v${GITLAB_VERSION} --depth 1 \ @@ -100,6 +109,9 @@ sudo -HEu ${GITLAB_USER} bundle install -j$(nproc) --deployment --without develo # make sure everything in ${GITLAB_HOME} is owned by the git user chown -R ${GITLAB_USER}:${GITLAB_USER} ${GITLAB_HOME}/ +# install schedules cronjob +sudo -HEu ${GITLAB_USER} bundle exec whenever -w + # install gitlab bootscript cp lib/support/init.d/gitlab /etc/init.d/gitlab chmod +x /etc/init.d/gitlab @@ -205,6 +217,8 @@ command=bundle exec sidekiq -c {{SIDEKIQ_CONCURRENCY}} -q system_hook -q project_web_hook -q gitlab_shell + -q incoming_email + -q runner -q common -q default -e ${RAILS_ENV} @@ -218,6 +232,25 @@ stdout_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log stderr_logfile=${GITLAB_LOG_DIR}/supervisor/%(program_name)s.log EOF +# configure supervisord to start gitlab-git-http-server +cat > /etc/supervisor/conf.d/gitlab-git-http-server.conf < /etc/supervisor/conf.d/sshd.conf < ${GITLAB_BUILDS_DIR} +rm -rf builds +ln -sf ${GITLAB_BUILDS_DIR} builds # remove old cache directory (remove this line after a few releases) rm -rf ${GITLAB_DATA_DIR}/cache @@ -828,21 +854,22 @@ appInit () { case ${GITLAB_BACKUPS} in daily|weekly|monthly) read hour min <<< ${GITLAB_BACKUP_TIME//[:]/ } + crontab -u ${GITLAB_USER} -l > /tmp/cron.${GITLAB_USER} case ${GITLAB_BACKUPS} in daily) - sudo -HEu ${GITLAB_USER} cat > /tmp/cron.${GITLAB_USER} <> /tmp/cron.${GITLAB_USER} < /tmp/cron.${GITLAB_USER} <> /tmp/cron.${GITLAB_USER} < /tmp/cron.${GITLAB_USER} <> /tmp/cron.${GITLAB_USER} < - Execute a rake task." echo " app:help - Displays the help" echo " [command] - Execute the specified linux command eg. bash." diff --git a/kubernetes/gitlab-controller.yml b/kubernetes/gitlab-controller.yml index 803823fe..0c1a7ac9 100644 --- a/kubernetes/gitlab-controller.yml +++ b/kubernetes/gitlab-controller.yml @@ -22,6 +22,9 @@ spec: - name: GITLAB_TIMEZONE value: Kolkata + - name: GITLAB_SECRETS_DB_KEY_BASE + value: long-and-random-alpha-numeric-string + - name: GITLAB_HOST value: git.example.com - name: GITLAB_PORT