From 262d1fcd4d67051c607c66046bf1db7fa2842979 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Mon, 20 Nov 2023 18:37:35 +0100 Subject: [PATCH 1/4] ci: Release on multiple distributions & fix Docker multiple-step build Signed-off-by: Thomas Citharel --- .gitlab-ci.yml | 93 ++++++++++++++++++++++++++++++------ docker/multiarch/Dockerfile | 6 ++- docker/production/Dockerfile | 2 +- docker/tests/Dockerfile | 4 +- 4 files changed, 85 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7ce24fa3..045ca95c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,8 +6,7 @@ stages: - build-js - sentry - test - - docker - - package + - build - upload - deploy @@ -125,7 +124,7 @@ deps: exunit: stage: test services: - - name: postgis/postgis:14-3.2 + - name: postgis/postgis:16-3.4 alias: postgres variables: MIX_ENV: test @@ -164,7 +163,7 @@ vitest: e2e: stage: test services: - - name: postgis/postgis:14-3.2 + - name: postgis/postgis:16-3.4 alias: postgres variables: MIX_ENV: "e2e" @@ -206,7 +205,7 @@ pages: - public .docker: &docker - stage: docker + stage: build image: docker:24 variables: DOCKER_TLS_CERTDIR: "/certs" @@ -214,6 +213,7 @@ pages: DOCKER_TLS_VERIFY: 1 DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client" DOCKER_DRIVER: overlay2 + DOCKER_CLI_EXPERIMENTAL: enabled services: - docker:24-dind cache: {} @@ -229,6 +229,8 @@ pages: # Install qemu/binfmt - docker pull tonistiigi/binfmt:latest - docker run --rm --privileged tonistiigi/binfmt:latest --install all + # Install jq + - apk --no-cache add jq # Login to DockerHub - mkdir -p ~/.docker - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$CI_REGISTRY_AUTH\",\"email\":\"$CI_REGISTRY_EMAIL\"}}}" > ~/.docker/config.json @@ -242,9 +244,9 @@ build-docker-main: when: never - if: '$CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_TRIGGERED == "true"' script: - - docker buildx build --push --platform linux/amd64 -t framasoft/mobilizon:main -f docker/production/Dockerfile . + - docker buildx build --platform linux/amd64 -t framasoft/mobilizon:main -f docker/production/Dockerfile . -build-and-push-to-latest-docker-tag: +build-docker-tag: <<: *docker rules: &release-tag-rules - if: '$CI_PROJECT_NAMESPACE != "framasoft"' @@ -257,9 +259,9 @@ build-and-push-to-latest-docker-tag: docker buildx build --push --platform linux/${ARCH} + --provenance=false --build-arg="${ERL_FLAGS}" - -t framasoft/mobilizon:$CI_COMMIT_TAG - -t framasoft/mobilizon:latest + -t framasoft/mobilizon:${CI_COMMIT_TAG}-${ARCH} -f docker/production/Dockerfile . parallel: matrix: @@ -280,15 +282,49 @@ build-and-push-docker-tag: script: - > docker buildx build - --push --platform linux/amd64 - -t framasoft/mobilizon:$CI_COMMIT_TAG + --provenance=false + -t framasoft/mobilizon:$CI_COMMIT_TAG-amd64 -f docker/production/Dockerfile . +# Create manifest and push +docker-manifest-push: + <<: *docker + needs: ["build-docker-tag"] + rules: &release-tag-rules + - if: '$CI_PROJECT_NAMESPACE != "framasoft"' + when: never + - if: $CI_COMMIT_TAG != null + when: on_success + script: + - > + docker manifest create framasoft/mobilizon:${CI_COMMIT_TAG} + --amend framasoft/mobilizon:${CI_COMMIT_TAG}-amd64 + --amend framasoft/mobilizon:${CI_COMMIT_TAG}-arm64 + - docker manifest push --purge framasoft/mobilizon:${CI_COMMIT_TAG} + +### +# Simply creating an alias to the tag doesn't work: +# « xxx is a manifest list » +# https://joonas.fi/2021/02/docker-multi-arch-image-tooling-buildx/ +### +docker-latest: + <<: *docker + needs: ["docker-manifest-push"] + rules: &release-tag-rules + - if: '$CI_PROJECT_NAMESPACE != "framasoft"' + when: never + - if: $CI_COMMIT_TAG != null && $CI_COMMIT_TAG !~ /alpha|beta|rc/ + when: on_success + script: + - echo docker manifest create framasoft/mobilizon:latest $(docker manifest inspect framasoft/mobilizon:$CI_COMMIT_TAG | jq '.manifests[] | .digest' | xargs -I {} echo framasoft/mobilizon@{}) + - docker manifest create framasoft/mobilizon:latest $(docker manifest inspect framasoft/mobilizon:$CI_COMMIT_TAG | jq -r '.manifests[] | .digest' | xargs -I {} echo framasoft/mobilizon@{}) + - docker manifest push --purge framasoft/mobilizon:latest + # Packaging app for amd64 package-app: - image: mobilizon/buildpack:1.15.5-erlang-26.0.2-debian-buster - stage: package + image: mobilizon/buildpack:1.15.7-erlang-26.1.2-${SYSTEM} + stage: build variables: &release-variables MIX_ENV: "prod" DEBIAN_FRONTEND: noninteractive @@ -312,9 +348,23 @@ package-app: expire_in: 2 days paths: - ${APP_ASSET} + parallel: + matrix: + - SYSTEM: + [ + "debian-bookworm", + "debian-bullseye", + "ubuntu-jammy", + "ubuntu-focal", + "ubuntu-bionic", + "alpine-3.17.5", + "alpine-3.18.4", + "fedora-38", + "fedora-39", + ] package-app-dev: - stage: package + stage: build variables: *release-variables script: *release-script except: @@ -326,7 +376,7 @@ package-app-dev: # Packaging app for multi-arch multi-arch-release: - stage: package + stage: build image: docker:24 variables: DOCKER_TLS_CERTDIR: "/certs" @@ -366,6 +416,19 @@ multi-arch-release: parallel: matrix: - ARCH: ["arm64"] + ## Currently not used as the hexpm base images do not have support for other architectures than amd64 + # SYSTEM: + # [ + # "debian-bookworm", + # "debian-bullseye", + # "ubuntu-jammy", + # "ubuntu-focal", + # "ubuntu-bionic", + # "alpine-3.17.5", + # "alpine-3.18.4", + # "fedora-38", + # "fedora-39", + # ] rules: - if: '$CI_COMMIT_TAG != null || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_TRIGGERED == "true"' timeout: 3h diff --git a/docker/multiarch/Dockerfile b/docker/multiarch/Dockerfile index 5f751c58..391507d7 100644 --- a/docker/multiarch/Dockerfile +++ b/docker/multiarch/Dockerfile @@ -1,4 +1,6 @@ -FROM elixir:1.15 as build +ARG IMAGE="elixir:1.15" + +FROM ${IMAGE} as build SHELL ["/bin/bash", "-c"] ENV MIX_ENV prod # ENV LANG en_US.UTF-8 @@ -12,7 +14,7 @@ ENV ERL_FLAGS=$ERL_FLAGS # Set the right versions ENV ELIXIR_VERSION latest ENV ERLANG_VERSION latest -ENV NODE_VERSION 18 +ENV NODE_VERSION 20 # Install system dependencies RUN apt-get update -yq && apt-get install -yq build-essential cmake postgresql-client git curl gnupg unzip exiftool webp imagemagick gifsicle diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile index 66db683e..385056bb 100644 --- a/docker/production/Dockerfile +++ b/docker/production/Dockerfile @@ -1,5 +1,5 @@ # First build the application assets -FROM node:18-alpine as assets +FROM node:20-alpine as assets RUN apk add --no-cache python3 build-base libwebp-tools bash imagemagick ncurses WORKDIR /build diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile index 6288c41b..835295eb 100644 --- a/docker/tests/Dockerfile +++ b/docker/tests/Dockerfile @@ -1,7 +1,7 @@ FROM elixir:latest -LABEL maintainer="Thomas Citharel " +LABEL maintainer="Thomas Citharel " -ENV REFRESHED_AT=2023-08-17 +ENV REFRESHED_AT=2023-11-20 RUN apt-get update -yq && apt-get install -yq build-essential inotify-tools postgresql-client git curl gnupg xvfb libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 cmake exiftool python3-pip python3-setuptools RUN curl -sL https://deb.nodesource.com/setup_20.x | bash && apt-get install nodejs -yq RUN npm install -g wait-on From cd53062c01c2ba07a7583c16fbcd60e5c3987a89 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 21 Nov 2023 18:10:54 +0100 Subject: [PATCH 2/4] fix: add a final fallback if we have default_language: nil in instance config Signed-off-by: Thomas Citharel --- lib/web/views/error_view.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/web/views/error_view.ex b/lib/web/views/error_view.ex index 6907b6dd..fa893e47 100644 --- a/lib/web/views/error_view.ex +++ b/lib/web/views/error_view.ex @@ -53,6 +53,8 @@ defmodule Mobilizon.Web.ErrorView do locale = Mobilizon.Config.instance_config() |> Keyword.get(:default_language, "en") + # Just in case + |> Kernel.||("en") Gettext.put_locale(locale) From b2bacbf6eb982adf939823cbf8dd76ec2e76b22d Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Tue, 21 Nov 2023 18:21:50 +0100 Subject: [PATCH 3/4] build: downgrade Sentry since it doesn't want to compile Signed-off-by: Thomas Citharel --- config/config.exs | 6 ------ mix.exs | 2 +- mix.lock | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/config/config.exs b/config/config.exs index 8cb6705e..f590043e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -394,12 +394,6 @@ config :mobilizon, Mobilizon.Service.AntiSpam, service: Mobilizon.Service.AntiSp config :mobilizon, Mobilizon.Service.SiteMap, path: System.get_env("MOBILIZON_SITEMAP_PATH", "/var/lib/mobilizon/sitemap") -config :sentry, - dsn: "", - environment_name: Mix.env(), - enable_source_code_context: true, - root_source_code_paths: [File.cwd!()] - # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{config_env()}.exs" diff --git a/mix.exs b/mix.exs index f0847fd1..ba97d561 100644 --- a/mix.exs +++ b/mix.exs @@ -201,7 +201,7 @@ defmodule Mobilizon.Mixfile do {:remote_ip, "~> 1.1"}, {:ex_cldr_languages, "~> 0.3.0"}, {:slugger, "~> 0.3"}, - {:sentry, "~> 9.0"}, + {:sentry, "~> 8.1"}, {:html_entities, "~> 0.5"}, {:sweet_xml, "~> 0.7"}, {:web_push_encryption, github: "danhper/elixir-web-push-encryption"}, diff --git a/mix.lock b/mix.lock index e4380f52..14967a38 100644 --- a/mix.lock +++ b/mix.lock @@ -125,7 +125,7 @@ "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "remote_ip": {:hex, :remote_ip, "1.1.0", "cb308841595d15df3f9073b7c39243a1dd6ca56e5020295cb012c76fbec50f2d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "616ffdf66aaad6a72fc546dabf42eed87e2a99e97b09cbd92b10cc180d02ed74"}, "replug": {:hex, :replug, "0.1.0", "61d35f8c873c0078a23c49579a48f36e45789414b1ec0daee3fd5f4e34221f23", [:mix], [{:plug, "~> 1.8", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "f71f7a57e944e854fe4946060c6964098e53958074c69fb844b96e0bd58cfa60"}, - "sentry": {:hex, :sentry, "9.1.0", "8689b85774003ddcebfd9d48a93bc3f3bf72223983514521aa30645c6f204f86", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "d70c88ab0c6a511594856ae2244d1bd70b8b7a4a42201a3569880f1dd2a3adec"}, + "sentry": {:hex, :sentry, "8.1.0", "8d235b62fce5f8e067ea1644e30939405b71a5e1599d9529ff82899d11d03f2b", [:mix], [{:hackney, "~> 1.8", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:plug, "~> 1.6", [hex: :plug, repo: "hexpm", optional: true]}, {:plug_cowboy, "~> 2.3", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm", "f9fc7641ef61e885510f5e5963c2948b9de1de597c63f781e9d3d6c9c8681ab4"}, "shortuuid": {:hex, :shortuuid, "3.0.0", "028684d9eeed0ad4b800e8481afd854e1a61c526f35952455b2ee4248601e7b8", [:mix], [], "hexpm", "dfd8f80f514cbb91622cb83f4ac0d6e2f06d98cc6d4aeba94444a212289d0d39"}, "sitemapper": {:hex, :sitemapper, "0.7.0", "4aee7930327a9a01b1c9b81d1d42f60c1a295e9f420108eb2d130c317415abd7", [:mix], [{:ex_aws_s3, "~> 2.0", [hex: :ex_aws_s3, repo: "hexpm", optional: true]}, {:xml_builder, "~> 2.1", [hex: :xml_builder, repo: "hexpm", optional: false]}], "hexpm", "60f7a684e5e9fe7f10ac5b69f48b0be2bcbba995afafcb3c143fc0c8ef1f223f"}, "sleeplocks": {:hex, :sleeplocks, "1.1.2", "d45aa1c5513da48c888715e3381211c859af34bee9b8290490e10c90bb6ff0ca", [:rebar3], [], "hexpm", "9fe5d048c5b781d6305c1a3a0f40bb3dfc06f49bf40571f3d2d0c57eaa7f59a5"}, From f34099d384212de14f642f10b7944d7cd551952c Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Thu, 23 Nov 2023 13:56:58 +0100 Subject: [PATCH 4/4] build(docker): optimize image size By executing the curl call directly as the nobody user Closes #1012 Signed-off-by: Thomas Citharel --- .dockerignore | 9 ++++++++- .gitlab-ci.yml | 2 +- docker/production/Dockerfile | 10 +++++----- docker/tests/Dockerfile | 6 +++--- 4 files changed, 17 insertions(+), 10 deletions(-) diff --git a/.dockerignore b/.dockerignore index d887b97b..7f29df34 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,10 @@ _build CONTRIBUTING.md +coverage +demo deps +doc +docs docker-compose.yml Dockerfile .elixir_ls @@ -16,4 +20,7 @@ README.md SECURITY.md ssh_match_hostname package-lock.json -node_modules \ No newline at end of file +node_modules +playwright-report +test +tests diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 045ca95c..81bc7c46 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -375,7 +375,7 @@ package-app-dev: - ${APP_ASSET} # Packaging app for multi-arch -multi-arch-release: +package-multi-arch-release: stage: build image: docker:24 variables: diff --git a/docker/production/Dockerfile b/docker/production/Dockerfile index 385056bb..6b40facd 100644 --- a/docker/production/Dockerfile +++ b/docker/production/Dockerfile @@ -51,7 +51,7 @@ LABEL org.opencontainers.image.title="mobilizon" \ org.opencontainers.image.created=$BUILD_DATE RUN apk add --no-cache curl openssl ca-certificates ncurses-libs file postgresql-client libgcc libstdc++ imagemagick python3 py3-pip py3-pillow py3-cffi py3-brotli gcc g++ musl-dev python3-dev pango libxslt-dev ttf-cantarell openssl1.1-compat -RUN pip install weasyprint pyexcel-ods3 +RUN pip --no-cache-dir install weasyprint pyexcel-ods3 # Create every data directory RUN mkdir -p /var/lib/mobilizon/uploads && chown nobody:nobody /var/lib/mobilizon/uploads @@ -60,14 +60,14 @@ RUN mkdir -p /var/lib/mobilizon/tzdata && chown nobody:nobody /var/lib/mobilizon RUN mkdir -p /var/lib/mobilizon/sitemap && chown nobody:nobody /var/lib/mobilizon/sitemap RUN mkdir -p /var/lib/mobilizon/uploads/exports/{csv,pdf,ods} && chown -R nobody:nobody /var/lib/mobilizon/uploads/exports -# Get timezone geodata -RUN curl -L 'https://packages.joinmobilizon.org/tz_world/timezones-geodata.dets' -o /var/lib/mobilizon/timezones/timezones-geodata.dets -RUN chown -R nobody:nobody /var/lib/mobilizon/timezones - # Create configuration directory RUN mkdir -p /etc/mobilizon && chown nobody:nobody /etc/mobilizon USER nobody + +# Get timezone geodata +RUN curl -L 'https://packages.joinmobilizon.org/tz_world/timezones-geodata.dets' -o /var/lib/mobilizon/timezones/timezones-geodata.dets + EXPOSE 4000 ENV MOBILIZON_DOCKER=true diff --git a/docker/tests/Dockerfile b/docker/tests/Dockerfile index 835295eb..7cccbada 100644 --- a/docker/tests/Dockerfile +++ b/docker/tests/Dockerfile @@ -2,10 +2,10 @@ FROM elixir:latest LABEL maintainer="Thomas Citharel " ENV REFRESHED_AT=2023-11-20 -RUN apt-get update -yq && apt-get install -yq build-essential inotify-tools postgresql-client git curl gnupg xvfb libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 cmake exiftool python3-pip python3-setuptools -RUN curl -sL https://deb.nodesource.com/setup_20.x | bash && apt-get install nodejs -yq +RUN apt-get update -yq && apt-get install -yq ca-certificates build-essential inotify-tools postgresql-client git curl gnupg xvfb libgtk-3-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 cmake exiftool python3-pip python3-setuptools +RUN mkdir -p /etc/apt/keyrings && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list && apt-get update && apt-get install nodejs -yq RUN npm install -g wait-on RUN apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN mix local.hex --force && mix local.rebar --force -RUN pip3 install -Iv weasyprint pyexcel_ods3 +RUN pip3 --no-cache-dir install -Iv weasyprint pyexcel_ods3 RUN curl https://dbip.mirror.framasoft.org/files/dbip-city-lite-latest.mmdb --output GeoLite2-City.mmdb -s && mkdir -p /usr/share/GeoIP && mv GeoLite2-City.mmdb /usr/share/GeoIP/