Merge pull request #4385 from esl/use-erlang-images

Use prebuilt Docker images with Erlang to build packages

Author: JanuszJakubiec

.circleci/template.yml

@@ -2,11 +2,12 @@ version: 2.1
 
 references:
   # Latest OTP release version number (on the right): https://www.erlang.org/downloads
-  # Available cimg-erlang images: https://hub.docker.com/r/mongooseim/cimg-erlang/tags
+  # Available cimg-erlang images: https://hub.docker.com/r/erlangsolutions/erlang/tags?name=cimg
   # You could need to trigger a pipeline to create a Docker image:
   # https://github.com/esl/cimg-erlang#trigger-build-using-trigger-pipeline-on-circleci
-  - &OTP26 mongooseim/cimg-erlang:26.2.5.2
-  - &OTP27 mongooseim/cimg-erlang:27.0.1
+  - &LATEST_OTP_VERSION 27.1.2
+  - &OTP26 erlangsolutions/erlang:cimg-26.2.5.4
+  - &OTP27 erlangsolutions/erlang:cimg-27.1.2
   - &ENTRYPOINT ["/bin/sh", "-c", "eval ${INSTALL_DEPS_CMD:-echo} && echo __INJECT_FILES__ | eval ${BASE32DEC:-base32 --decode} | bash"]
   # Caches created via the save_cache step are stored for up to 15 days
   - &CERT_KEY certs-cache-{{ checksum "certs_cache_key" }}-v3
@@ -689,14 +690,15 @@ jobs:
         type: executor
       platform:
         type: enum
-        enum: [rockylinux_9, rockylinux_8, almalinux_9, almalinux_8, debian_bullseye, debian_buster, ubuntu_xenial, ubuntu_focal, ubuntu_bionic, ubuntu_jammy]
+        enum: [debian-bookworm, debian-bullseye, debian-buster, ubuntu-oracular, ubuntu-noble, ubuntu-jammy, ubuntu-focal, rockylinux-9, rockylinux-8, almalinux-9, almalinux-8]
         description: Platform type
-      otp_package:
+      otp_version:
         type: string
-        description: Version of the Erlang package to install
+        description: Version of Erlang/OTP to use
+        default: *LATEST_OTP_VERSION
     environment:
       pkg_PLATFORM: <<parameters.platform>>
-      ESL_ERLANG_PKG_VER: <<parameters.otp_package>>
+      pkg_OTP_VERSION: <<parameters.otp_version>>
       PRESET: pkg
     steps:
       - checkout
@@ -735,55 +737,72 @@ workflows:
           filters: &release_tags
             tags:
               only: /^\d+\.\d+\.\d+/
-      # ============= PACKAGES =============
+#       # ============= PACKAGES =============
       - package:
-          name: rockylinux_8
+          name: debian-bookworm
           executor: otp_27
-          platform: rockylinux_8
+          platform: debian-bookworm
           context: mongooseim-org
-          otp_package: "26.2.4"
           filters: *all_tags
       - package:
-          name: almalinux_8
+          name: debian-bullseye
           executor: otp_27
-          platform: almalinux_8
+          platform: debian-bullseye
           context: mongooseim-org
-          otp_package: "26.2.4"
           filters: *all_tags
       - package:
-          name: debian_bullseye
+          name: debian-buster
           executor: otp_27
-          platform: debian_bullseye
+          platform: debian-buster
           context: mongooseim-org
-          otp_package: "26.2.1"
           filters: *all_tags
       - package:
-          name: debian_buster
+          name: ubuntu-oracular
           executor: otp_27
-          platform: debian_buster
+          platform: ubuntu-oracular
           context: mongooseim-org
-          otp_package: "26.2.3"
           filters: *all_tags
       - package:
-          name: ubuntu_jammy
+          name: ubuntu-noble
           executor: otp_27
-          platform: ubuntu_jammy
+          platform: ubuntu-noble
           context: mongooseim-org
-          otp_package: "26.2.3"
           filters: *all_tags
       - package:
-          name: ubuntu_focal
+          name: ubuntu-jammy
           executor: otp_27
-          platform: ubuntu_focal
+          platform: ubuntu-jammy
           context: mongooseim-org
-          otp_package: "26.2.1"
           filters: *all_tags
       - package:
-          name: ubuntu_bionic
+          name: ubuntu-focal
           executor: otp_27
-          platform: ubuntu_bionic
+          platform: ubuntu-focal
+          context: mongooseim-org
+          filters: *all_tags
+      - package:
+          name: rockylinux-9
+          executor: otp_27
+          platform: rockylinux-9
+          context: mongooseim-org
+          filters: *all_tags
+      - package:
+          name: rockylinux-8
+          executor: otp_27
+          platform: rockylinux-8
+          context: mongooseim-org
+          filters: *all_tags
+      - package:
+          name: almalinux-9
+          executor: otp_27
+          platform: almalinux-9
+          context: mongooseim-org
+          filters: *all_tags
+      - package:
+          name: almalinux-8
+          executor: otp_27
+          platform: almalinux-8
           context: mongooseim-org
-          otp_package: "26.2.3"
           filters: *all_tags
       # ======== BASE DOCKER BUILDS ========
       - build_in_docker:

tools/pkg/Dockerfile_deb

@@ -1,7 +1,8 @@
 # vi: ft=dockerfile
-ARG dockerfile_platform
+ARG builder_image
+ARG target_image
 
-FROM $dockerfile_platform AS builder
+FROM $builder_image AS builder
 
 # Install build deps
 ENV DEBIAN_FRONTEND=noninteractive
@@ -13,37 +14,35 @@ ARG erlang_version
 # Fix locales
 RUN locale-gen en_US.UTF-8
 
-# Copy source code and put building files
+# Copy source code and put building files in proper directories
 # The .dockerignore file in root dir ensures only needed files
 # including not commited changes are used to build the package
 WORKDIR /root/
 COPY . ./mongooseim
 
 RUN cp -r ./mongooseim/tools/pkg/scripts/deb .
 
-ARG dockerfile_platform
-RUN ./deb/install_erlang.sh
-
 ARG version
 ARG revision
 
 RUN ./deb/build_package.sh $version $revision $erlang_version
 
 # Create image for sharing and validation of built package
-FROM $dockerfile_platform
+FROM $target_image AS target
 
 # Copy built package from previous image and install it with required dependencies
-RUN apt-get update && apt-get -y install openssl procps netcat && apt-get -y clean
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get -y install openssl procps && apt-get -y clean
 WORKDIR /root/
 COPY --from=builder /root/*.deb .
 
 # Install mongooseim with required dependencies
-RUN apt-get update; dpkg -i *.deb; apt-get install -y -f
+RUN apt-get update && dpkg -i *.deb && apt-get install -y -f
 
 # Simple check if MiM works
-COPY --from=builder /root/mongooseim/tools/wait-for-it.sh .
-COPY --from=builder /root/mongooseim/tools/pkg/scripts/smoke_test.sh .
-COPY --from=builder /root/mongooseim/tools/pkg/scripts/smoke_templates.escript .
+COPY tools/wait-for-it.sh .
+COPY tools/pkg/scripts/smoke_test.sh .
+COPY tools/pkg/scripts/smoke_templates.escript .
 
 RUN ./smoke_test.sh
 

tools/pkg/Dockerfile_rpm

@@ -1,27 +1,15 @@
 # vi: ft=dockerfile
-ARG dockerfile_platform
+ARG builder_image
+ARG target_image
 
-# Source container is useful for preparing code for the actual build.
-FROM $dockerfile_platform AS source
-COPY . /SOURCE
-# Separate the main code from the testing code.
-# So that any changes in the testing code do not trigger cache invalidation in the builder.
-RUN mkdir /TESTS \
-    && mv /SOURCE/tools/pkg/scripts/smoke_test.sh /TESTS/ \
-    && mv /SOURCE/tools/pkg/scripts/smoke_templates.escript /TESTS/
-
-FROM $dockerfile_platform AS builder
+FROM $builder_image AS builder
 
 # Install the build dependencies
-RUN yum install -y epel-release
-RUN yum install -y rpm-build rpmdevtools git make zlib-devel unixODBC-devel gcc gcc-c++ openssl openssl-devel chrpath
+RUN dnf install -y rpm-build rpmdevtools git make zlib-devel unixODBC-devel gcc gcc-c++ \
+    openssl openssl-devel chrpath glibc-locale-source systemd-rpm-macros
 
 # Fix locale setup
 # See https://github.com/CentOS/sig-cloud-instance-images/issues/71#issuecomment-266957519
-ARG dockerfile_platform
-RUN if [ "$dockerfile_platform" == "rockylinux:8" ] || [ "$dockerfile_platform" == "almalinux:8" ]; then \
-    dnf install -y glibc-locale-source; \
-    fi
 RUN localedef -i en_US -f UTF-8 en_US.UTF-8
 
 ARG erlang_version
@@ -32,8 +20,7 @@ ARG erlang_version
 # including not commited changes are used to build the package
 RUN rpmdev-setuptree
 WORKDIR /root/rpmbuild
-COPY --from=source /SOURCE ./BUILD/mongooseim
-RUN ./BUILD/mongooseim/tools/pkg/scripts/rpm/install_erlang.sh
+COPY . ./BUILD/mongooseim
 
 RUN cp ./BUILD/mongooseim/tools/pkg/scripts/rpm/mongooseim.spec ./SPECS/.
 RUN cp ./BUILD/mongooseim/tools/pkg/scripts/rpm/mongooseim.service \
@@ -45,16 +32,17 @@ ARG revision
 RUN ./BUILD/mongooseim/tools/pkg/scripts/rpm/build_package.sh $version $revision $erlang_version
 
 # Create image for sharing and validation of built package
-FROM $dockerfile_platform
+FROM $target_image AS target
 
 # Copy built package from previous image and install it with required dependencies
 WORKDIR /root/
 COPY --from=builder /root/mongooseim*.rpm .
-RUN yum -y update; yum install -y mongooseim*.rpm
+RUN dnf -y update && dnf install -y mongooseim*.rpm
 
 # Simple check if MiM works
-COPY --from=builder /root/rpmbuild/BUILD/mongooseim/tools/wait-for-it.sh .
-COPY --from=source /TESTS/* ./
+COPY tools/wait-for-it.sh .
+COPY tools/pkg/scripts/smoke_test.sh .
+COPY tools/pkg/scripts/smoke_templates.escript .
 
 RUN ./smoke_test.sh
 

tools/pkg/build.sh

@@ -72,9 +72,11 @@ for (( i = 0; i < $len - 1; i++ )); do
     i=$((i+1))
 done
 
-dockerfile_platform=${platform/_/:}
+builder_image="erlangsolutions/erlang:${platform}-${erlang_version}"
+target_image="${platform/-/:}"
 docker build -t mongooseim-${platform}:${version}-${revision} \
-    --build-arg dockerfile_platform=${dockerfile_platform} \
+    --build-arg builder_image=${builder_image} \
+    --build-arg target_image=${target_image} \
     --build-arg version=${version} \
     --build-arg revision=${revision} \
     --build-arg erlang_version=${erlang_version} \

tools/pkg/scripts/deb/debian/postinst

@@ -40,7 +40,7 @@ setup()
 
 case "$1" in
     configure)
-	adduser --quiet --system --shell /bin/sh --group --home $PKG_ROOT $NAME 2>/dev/null
+	useradd --system --shell /bin/sh --home-dir $PKG_ROOT --create-home $NAME
 	setup "$@"
     ;;
 

tools/pkg/scripts/deb/install_erlang.sh

@@ -23,7 +23,7 @@ BuildRequires:     gcc
 BuildRequires:     gcc-c++
 BuildRequires:     openssl
 BuildRequires:     openssl-devel
-BuildRequires:     esl-erlang
+BuildRequires:     systemd-rpm-macros
 
 %description
 MongooseIM is Erlang Solutions' robust and efficient XMPP server aimed at large
@@ -49,6 +49,10 @@ chrpath -d %{buildroot}/usr/lib/mongooseim/lib/crypto-*/priv/lib/otp_test_engine
 install -p -D -m 0644 %{SOURCE0} %{buildroot}%{_unitdir}/mongooseim.service
 
 %pre
+if ! [ -d %{_localstatedir}/lock ]; then
+    # /var/lock links to non-existing /run/lock on Rocky Linux 9
+    mkdir -m 755 `readlink -f %{_localstatedir}/lock`
+fi
 getent group mongooseim >/dev/null || groupadd mongooseim
 getent passwd mongooseim >/dev/null || adduser -g mongooseim mongooseim
 exit 0

tools/pkg/scripts/rpm/install_erlang.sh

@@ -278,7 +278,7 @@ build_pkg () {
   cd tools/pkg
 
   local platform=$1
-  local esl_erlang_pkg_vsn=$2
+  local erlang_version=$2
   local project_root=$(git rev-parse --show-toplevel)
 
   if [[ $platform == rockylinux* ]] || [[ $platform == almalinux* ]]; then
@@ -302,7 +302,7 @@ build_pkg () {
     --platform $platform \
     --version $version \
     --revision $revision \
-    --erlang_version $esl_erlang_pkg_vsn \
+    --erlang_version $erlang_version \
     --dockerfile_path "$project_root/tools/pkg/$dockerfile_name" \
     --context_path $project_root \
     --built_packages_directory "$project_root/tools/pkg/packages"
@@ -331,7 +331,7 @@ elif [ "$PRESET" == "edoc_only" ]; then
   tools/print-dots.sh stop
   exit ${RESULT}
 elif [ "$PRESET" == "pkg" ]; then
-  build_pkg $pkg_PLATFORM $ESL_ERLANG_PKG_VER
+  build_pkg $pkg_PLATFORM $pkg_OTP_VERSION
 elif [ "$PRESET" == "small_tests" ]; then
   time maybe_run_small_tests
   SMALL_RESULT=$?