From 260f548c5b4bd0c4e602740128aeca6058331f4d Mon Sep 17 00:00:00 2001 From: Kazunori Kimura Date: Thu, 20 Apr 2023 18:59:25 +0900 Subject: [PATCH 1/5] Ship with multiple versions of postgresql-client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It is intended to work even in environments without an internet connection. - Use latest version of pg_isready on checking database connection - Switch postgresql-client version checking server on startup - If the server versions do not match exactly, use the latest installed client and warn on docker log From postgresql documentation: https://www.postgresql.org/docs/15/app-pgdump.html#PG-DUMP-NOTES > Also, it is not guaranteed that pg_dump's output can be loaded > into a server of an older major version — not even if the dump > was taken from a server of that version. To switch client version, generate a file ~/.postgresqlrc with content like: {server side major version} {server hostname}:{server port} {default database} See detail on: https://manpages.ubuntu.com/manpages/bionic/man5/postgresqlrc.5.html Following messages appear on docker log: - Detected server version: 140007 - Generating /home/git/.postgresqlrc --- Dockerfile | 3 ++- assets/runtime/functions | 56 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e966e52b..f0f245c9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,8 @@ RUN set -ex && \ && apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \ sudo supervisor logrotate locales curl \ - nginx openssh-server postgresql-client-12 postgresql-contrib-12 redis-tools \ + nginx openssh-server postgresql-contrib redis-tools \ + postgresql-client-12 postgresql-client-13 postgresql-client-14 postgresql-client-15 \ python3 python3-docutils nodejs yarn gettext-base graphicsmagick \ libpq5 zlib1g libyaml-0-2 libssl1.1 \ libgdbm6 libreadline8 libncurses5 libffi7 \ diff --git a/assets/runtime/functions b/assets/runtime/functions index 787ce5c8..34a0a0e9 100644 --- a/assets/runtime/functions +++ b/assets/runtime/functions @@ -149,7 +149,7 @@ gitlab_finalize_database_parameters() { gitlab_check_database_connection() { - prog=$(find /usr/lib/postgresql/ -name pg_isready) + prog=$(find /usr/lib/postgresql/ -name pg_isready -type f | sort | tail -n1) prog="${prog} -h ${DB_HOST} -p ${DB_PORT} -U ${DB_USER} -d ${DB_NAME} -t 1" timeout=60 @@ -167,11 +167,65 @@ gitlab_check_database_connection() { echo } +gitlab_generate_postgresqlrc() { + echo "Configuring /home/${GITLAB_USER}/.postgresqlrc to avoid version mismatch on dumping" + # server_version_num property is a number built from version string: + # https://www.postgresql.org/docs/15/libpq-status.html#LIBPQ-PQSERVERVERSION + # > The result is formed by multiplying the server's major version number by 10000 and adding the minor version number. + # > For example, version 10.1 will be returned as 100001, and version 11.0 will be returned as 110000. Zero is returned if the connection is bad. + # > + # > Prior to major version 10, PostgreSQL used three-part version numbers in which the first two parts together represented the major version. + # > For those versions, PQserverVersion uses two digits for each part; + # > for example version 9.1.5 will be returned as 90105, and version 9.2.0 will be returned as 90200. + # + # This difference also appends to apt package name. + # For example, in ubuntu:focal, postgresql-client-{8.2, 8.3, 8.4, 9.0, 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 10, 11, 12, 13, 14, 15} are available. + # + DB_SERVER_VERSION=$(PGPASSWORD=${DB_PASS} psql -h "${DB_HOST}" -p "${DB_PORT}" -U "${DB_USER}" -d "${DB_NAME}" -Atw -c "SHOW server_version_num") + if [[ "${DB_SERVER_VERSION}" -eq 0 ]]; then + echo + echo "Could not retrieve database server version correctly. Aborting..." + return 1 + fi + + echo "- Detected server version: ${DB_SERVER_VERSION}" + + # Anyway, we can get major version (8, 9, 10 and so on) by dividing by 10000. + # DB_SERVER_VERSION_MAJOR=${DB_SERVER_VERSION%%.*} + DB_SERVER_VERSION_MAJOR=$((DB_SERVER_VERSION/10000)) + DB_CLIENT_VERSION_PACKAGE_NAME= + + if [[ "${DB_SERVER_VERSION_MAJOR}" -ge 10 ]]; then + # v10 or later: use "rought major version" as version number in package name + DB_CLIENT_VERSION_PACKAGE_NAME=${DB_SERVER_VERSION_MAJOR} + else + # prior to v10: convert + # FIXME: rough implementation + # It exploits the fact that there is no version such as 9.10, and it lacks versatility. + # divide by 100, then replace first 0 to comma + DB_CLIENT_VERSION_PACKAGE_NAME=$((DB_SERVER_VERSION/100)) + DB_CLIENT_VERSION_PACKAGE_NAME=${DB_CLIENT_VERSION_PACKAGE_NAME/0/.} + fi + + # if exact-match client not found, select latest version from installed clients + if [[ "$(apt-cache pkgnames postgresql-client | grep -e "-${DB_CLIENT_VERSION_PACKAGE_NAME}" | wc -l)" -ne 1 ]]; then + LATEST_CLIENT="$(apt-cache pkgnames postgresql-client | grep -v -e "-common" | sort --version-sort | tail -n1)" + DB_CLIENT_VERSION_PACKAGE_NAME=${LATEST_CLIENT/postgresql-client-/} + echo "gitlab_generate_postgresqlrc(): WARNING - Suitable client not installed. postgresql-client-${DB_CLIENT_VERSION_PACKAGE_NAME} will be used but you may face issue (database in backup will be empty, for example)" + fi + + # generate ~/.postgresqlrc to switch client version + GITLAB_USER_POSTGRESQLRC="/home/${GITLAB_USER}/.postgresqlrc" + echo "- Generating ${GITLAB_USER_POSTGRESQLRC}" + echo "${DB_CLIENT_VERSION_PACKAGE_NAME} ${DB_HOST}:${DB_PORT} ${DB_NAME}" | exec_as_git tee "${GITLAB_USER_POSTGRESQLRC}" +} + gitlab_configure_database() { echo -n "Configuring gitlab::database" gitlab_finalize_database_parameters gitlab_check_database_connection + gitlab_generate_postgresqlrc update_template ${GITLAB_DATABASE_CONFIG} \ DB_ENCODING \ From b3b700a7019a1a2a9f17a88f81d85cd2ec8a0e67 Mon Sep 17 00:00:00 2001 From: Kazunori Kimura Date: Thu, 20 Apr 2023 18:59:44 +0900 Subject: [PATCH 2/5] Uninstall unused postgresql-client on startup Unused clients are determinate by checking ~/.postgresqlrc Uninstall logs like the following will appear in the docker log: - Uninstalling unused version(s) of client: postgresql-client-12 --- assets/runtime/functions | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/assets/runtime/functions b/assets/runtime/functions index 34a0a0e9..4b020075 100644 --- a/assets/runtime/functions +++ b/assets/runtime/functions @@ -220,12 +220,36 @@ gitlab_generate_postgresqlrc() { echo "${DB_CLIENT_VERSION_PACKAGE_NAME} ${DB_HOST}:${DB_PORT} ${DB_NAME}" | exec_as_git tee "${GITLAB_USER_POSTGRESQLRC}" } +gitlab_uninstall_unused_database_client() { + if [[ -f "/home/${GITLAB_USER}/.postgresqlrc" ]]; then + # refer /home/${GITLAB_USER}/.postgresqlrc and pick up versions in use + # .postgresqlrc contains following information per line + # database_major_version host:port database_name + # - ignore lines starts with # by specifying pattern /^[^#]/ + # - first field is the version number in use. + # - cocnat whole lines into single string. convert newline to \| + # this is escaped regex "OR" + # now we got the following regex that can be used as an option to grep: + # \|-12\|-13 + DB_CLIENT_VERSIONS_IN_USE="$(awk '/^[^#]/ {printf("\|-%s",$1)}' "/home/${GITLAB_USER}/.postgresqlrc")" + + # we also need to keep postgresql-client-common package to switch based on ~/.postgresqlrc + REGEX_DB_CLIENT_VERSIONS_IN_USE="-common${DB_CLIENT_VERSIONS_IN_USE}" + + # remove unused client using regex above + UNUSED_DB_CLIENTS=$(apt-cache pkgnames postgresql-client | grep -v -e "${REGEX_DB_CLIENT_VERSIONS_IN_USE}" | tr '\n' ' ') + echo "- Uninstalling unused client(s): ${UNUSED_DB_CLIENTS}" + DEBIAN_FRONTEND=noninteractive apt-get -qq remove -- ${UNUSED_DB_CLIENTS} >/dev/null + fi +} + gitlab_configure_database() { echo -n "Configuring gitlab::database" gitlab_finalize_database_parameters gitlab_check_database_connection gitlab_generate_postgresqlrc + gitlab_uninstall_unused_database_client update_template ${GITLAB_DATABASE_CONFIG} \ DB_ENCODING \ From b736b992d696c147703faa347c801c7360ebeb3b Mon Sep 17 00:00:00 2001 From: Kazunori Kimura Date: Sun, 21 May 2023 21:23:53 +0900 Subject: [PATCH 3/5] README.md: Add description about postgresql-client switching behavior Co-authored by: Steven Achilles --- README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/README.md b/README.md index cbdeb966..3639dcc7 100644 --- a/README.md +++ b/README.md @@ -240,6 +240,29 @@ GitLab uses a database backend to store its data. You can configure this image t #### PostgreSQL +**Important note:** This image is shipped with different versions of the `postgresql-client`. + +During the startup of the container, the major version of the database system is checked based on the specified connection destination. Only the version of the `postgresql-client`, that matches the major version of the Postgres database is used. All other versions of the `postgresql-client` are deleted at runtime. + +This behavior can be checked using the command `docker logs` and an output like the following should be available: + +````sh +… +Configuring gitlab::database +- Installing postgresql client to avoid version mismatch on dumping +-- Detected server version: 140007 +- Generating /home/git/.postgresqlrc +14 postgresql:5432 gitlabhq_production +- Uninstalling unused client(s): postgresql-client-12 postgresql-client-13 postgresql-client-15 +… +```` + +Please note furthermore, that only combatible versions of the `postgresql-client` to GitLab are shipped with this image. Currently these belogn to + +- `postgresql-client-12`, +- `postgresql-client-13`, +- `postgresql-client-14`, +- and `postgresql-client-15`. *NOTE:* version 13.7.0 and later requires PostgreSQL version 12.x ##### External PostgreSQL Server From 69d490b75741dc08b84d88290149d31eb01d265a Mon Sep 17 00:00:00 2001 From: Kazunori Kimura Date: Tue, 23 May 2023 04:04:57 +0900 Subject: [PATCH 4/5] README.md: Add note for the case any version of client does not match --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3639dcc7..58b9996f 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ GitLab uses a database backend to store its data. You can configure this image t **Important note:** This image is shipped with different versions of the `postgresql-client`. -During the startup of the container, the major version of the database system is checked based on the specified connection destination. Only the version of the `postgresql-client`, that matches the major version of the Postgres database is used. All other versions of the `postgresql-client` are deleted at runtime. +During the startup of the container, the major version of the database system is checked based on the specified connection destination. Only the version of the `postgresql-client`, that matches the major version of the Postgres database is used. If the major version of any version of the client does not match, the latest client is used (but may causes issues). All other versions of the `postgresql-client` are deleted at runtime. This behavior can be checked using the command `docker logs` and an output like the following should be available: From 110a8d75abff870534bce4e21f7b40a18f058083 Mon Sep 17 00:00:00 2001 From: Steven Achilles Date: Mon, 22 May 2023 21:14:14 +0200 Subject: [PATCH 5/5] Update README.md --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 58b9996f..9cbda719 100644 --- a/README.md +++ b/README.md @@ -242,7 +242,7 @@ GitLab uses a database backend to store its data. You can configure this image t **Important note:** This image is shipped with different versions of the `postgresql-client`. -During the startup of the container, the major version of the database system is checked based on the specified connection destination. Only the version of the `postgresql-client`, that matches the major version of the Postgres database is used. If the major version of any version of the client does not match, the latest client is used (but may causes issues). All other versions of the `postgresql-client` are deleted at runtime. +During the startup of the container, the major version of the database system is checked based on the specified connection destination. Only the version of the `postgresql-client`, that matches the major version of the Postgres database is used. If the major version of any version of the included clients does not match, the latest client is used (but may causes issues). All other versions of the `postgresql-client` are deleted at runtime. This behavior can be checked using the command `docker logs` and an output like the following should be available: @@ -263,6 +263,7 @@ Please note furthermore, that only combatible versions of the `postgresql-client - `postgresql-client-13`, - `postgresql-client-14`, - and `postgresql-client-15`. + *NOTE:* version 13.7.0 and later requires PostgreSQL version 12.x ##### External PostgreSQL Server