Copy freebsd-build-cross to the repo

2021-02-19 19:51:10 -08:00
5 changed files with 141 additions and 77 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -1,14 +1,15 @@
 variables:
  DOCKER_TLS_CERTDIR: "/certs"
-  DOCKER_DRIVER: overlay2

 services:
-  - docker:dind
+  - docker:20.10.0-dind

 .build: &build
-  image: docker:latest
+  image: docker:20.10.0
  stage: build
  script:
+    - NAME=${NAME:-${CI_JOB_NAME}}
+    - DOCKERFILE=${DOCKERFILE:-${CI_JOB_NAME}.Dockerfile}
    - docker login -u gitlab-ci-token -p "${CI_JOB_TOKEN}" "${CI_REGISTRY}"
    - docker build -f "${DOCKERFILE}" -t "${CI_REGISTRY_IMAGE}/${NAME}:${CI_COMMIT_SHA:0:8}" .
    - docker tag "${CI_REGISTRY_IMAGE}/${NAME}:${CI_COMMIT_SHA:0:8}" "${CI_REGISTRY_IMAGE}/${NAME}:${CI_COMMIT_REF_SLUG}"
@ -16,26 +17,6 @@ services:
    - docker push "${CI_REGISTRY_IMAGE}/${NAME}:${CI_COMMIT_REF_SLUG}"
    - if [[ "${CI_COMMIT_REF_SLUG}" == "main" ]]; then docker tag "${CI_REGISTRY_IMAGE}/${NAME}:${CI_COMMIT_REF_SLUG}" "${CI_REGISTRY_IMAGE}/${NAME}:latest" && docker push "${CI_REGISTRY_IMAGE}/${NAME}:latest"; fi

-.docker-multiarch-image: &docker-multiarch-image
-  image: jdrouet/docker-with-buildx:stable
-  stage: build
-  tags: [docker-builder]
-  before_script: # per recommendations from https://github.com/docker/buildx/issues/495#issuecomment-754200673
-    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    - docker buildx create --name cibuilder --driver docker-container --use
-    - docker buildx inspect --bootstrap
-    - docker login -u gitlab-ci-token -p "${CI_JOB_TOKEN}" "${CI_REGISTRY}"
-    - export platform_sanatized=$(echo ${PLATFORM:6} | sed 's#/#-#g')
-  script:
-    - docker buildx build --platform "${PLATFORM}" -f "${DOCKERFILE}" -t "${CI_REGISTRY_IMAGE}/${NAME}:${CI_COMMIT_SHA:0:8}" -t "${CI_REGISTRY_IMAGE}/${NAME}-${platform_sanatized}:${CI_COMMIT_SHA:0:8}" -t "${CI_REGISTRY_IMAGE}/${NAME}:${CI_COMMIT_REF_SLUG}" -t "${CI_REGISTRY_IMAGE}/${NAME}-${platform_sanatized}:${CI_COMMIT_REF_SLUG}" --push .
-    - if [[ "${CI_COMMIT_REF_SLUG}" == "main" ]]; then docker buildx build --platform "${PLATFORM}" -f "${DOCKERFILE}" -t "${CI_REGISTRY_IMAGE}/${NAME}:latest" -t "${CI_REGISTRY_IMAGE}/${NAME}-${platform_sanatized}:latest" --push . ; fi
-  parallel:
-    matrix:
-      - PLATFORM:
-        - linux/amd64
-        - linux/arm64/v8
-        - linux/arm/v7
-
 signal-server:
  image: docker:20.10.0
  stage: build
@ -52,19 +33,7 @@ signal-server:
  allow_failure: true

 freebsd-cross-build:
-  image: docker:20.10.0
-  stage: build
-  script:
-    - apk add git
-    - git clone https://github.com/wezm/freebsd-cross-build.git
-    - cd freebsd-cross-build
-    - echo "RUN apt-get update && apt-get install -y git" >> Dockerfile
-    - VERSION=$(git rev-parse --short=8 HEAD)
-    - docker login -u gitlab-ci-token -p ${CI_JOB_TOKEN} ${CI_REGISTRY}
-    - docker build -t ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${VERSION} .
-    - docker push ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${VERSION}
-    - if [[ "${CI_COMMIT_REF_SLUG}" == "main" ]]; then docker tag ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${VERSION} ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:latest && docker push ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:latest; fi
-    - echo "docker pull ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${VERSION}"
+  <<: *build

 test-helper:
  <<: *build
@ -73,20 +42,22 @@ test-helper:
    NAME: "test-helper"
  allow_failure: true

-signal-cross-builder:
+builder-image:x86:
  <<: *build
  variables:
-    DOCKERFILE: "signald-cross-builder.Dockerfile"
-    NAME: "signald-cross-builder"
+    DOCKERFILE: "signald-builder.Dockerfile"
+    NAME: "signald-builder-x86"

-builder-image-alpine:
-  <<: *docker-multiarch-image
+builder-image:aarch64:
+  <<: *build
+  tags: [arm-docker-builder]
+  variables:
+    DOCKERFILE: "signald-builder.Dockerfile"
+    NAME: "signald-builder-arm"
+
+builder-image-alpine:aarch64:
+  <<: *build
+  tags: [arm-docker-builder]
  variables:
    DOCKERFILE: "signald-builder-alpine.Dockerfile"
    NAME: "signald-builder-alpine-arm"
-
-signald-builder:
-  <<: *docker-multiarch-image
-  variables:
-    DOCKERFILE: "signald-builder.Dockerfile"
-    NAME: "signald-builder"
--- a/15
+++ b/15
@ -0,0 +1,15 @@
+#!/bin/sh
+
+set -e
+
+cd "$1"
+BROKEN_LINKS=$(find . -xtype l)
+
+for link in $BROKEN_LINKS; do
+  src=$(readlink "$link")
+  if echo "$src" | grep -q '^\.\./\.\./lib/'; then
+    # Relink without relative part
+    new_src=$(echo "$src" | sed 's!^\.\./\.\./lib/!!')
+    ln -vsf "$new_src" "$link" 
+  fi
+done
--- a/freebsd-cross-build.Dockerfile
+++ b/freebsd-cross-build.Dockerfile
@ -0,0 +1,105 @@
+# This is heavily based on https://github.com/wezm/freebsd-cross-build but has been modified to work well with gitlab-ci
+
+FROM debian:10-slim AS build
+
+# TODO: Add ARG for FreeBSD version/image?
+# TODO: Add ARG for parallism?
+
+RUN mkdir /freebsd
+
+WORKDIR /freebsd
+
+RUN apt-get -y update && \
+    apt-get -y install build-essential m4 bison flex libtool automake autoconf autogen pkg-config curl
+
+RUN curl -OL 'http://ftp.swin.edu.au/freebsd/releases/amd64/12.1-RELEASE/base.txz'
+RUN curl -OL 'http://ftp.swin.edu.au/gnu/binutils/binutils-2.32.tar.xz'
+RUN curl -OL 'http://ftp.swin.edu.au/gnu/gmp/gmp-6.1.2.tar.xz'
+RUN curl -OL 'http://ftp.swin.edu.au/gnu/mpfr/mpfr-4.0.2.tar.xz'
+RUN curl -OL 'http://ftp.swin.edu.au/gnu/mpc/mpc-1.1.0.tar.gz'
+RUN curl -OL 'http://ftp.swin.edu.au/gnu/gcc/gcc-6.4.0/gcc-6.4.0.tar.xz'
+
+RUN mkdir -p /src/base /freebsd/x86_64-pc-freebsd12 && \
+    tar -C /src/base -Jxf base.txz
+
+# Populate the path that gcc will look in for headers and libs
+RUN mv /src/base/usr/include /freebsd/x86_64-pc-freebsd12 && \
+    mv /src/base/usr/lib /freebsd/x86_64-pc-freebsd12 && \
+    mv /src/base/lib/* /freebsd/x86_64-pc-freebsd12/lib
+
+# These libraries are actually linker scripts (grep -lr ldscript usr)
+# https://sourceware.org/binutils/docs/ld/Scripts.html
+# Adjust the paths to match their new locations
+RUN sed -i'' \
+        -e 's!/usr/lib/!/freebsd/x86_64-pc-freebsd12/lib/!g' \
+        -e 's!/lib/libc.so.7!/freebsd/x86_64-pc-freebsd12/lib/libc.so.7!g' \
+        /freebsd/x86_64-pc-freebsd12/lib/libc.so && \
+    sed -i'' \
+        's!/usr/lib/!/freebsd/x86_64-pc-freebsd12/lib/!g' \
+        /freebsd/x86_64-pc-freebsd12/lib/libc++.so
+
+# Fix symlinks broken by moving libraries
+COPY fix-links /src
+RUN /src/fix-links /freebsd/x86_64-pc-freebsd12/lib
+
+RUN tar -C /src -Jxf binutils-2.32.tar.xz && \
+    tar -C /src -Jxf gcc-6.4.0.tar.xz && \
+    tar -C /src -Jxf gmp-6.1.2.tar.xz && \
+    tar -C /src -zxf mpc-1.1.0.tar.gz && \
+    tar -C /src -Jxf mpfr-4.0.2.tar.xz
+
+# Build toolchain
+RUN cd /src/binutils-2.32 && \
+    ./configure --enable-libssp --enable-ld --target=x86_64-pc-freebsd12 --prefix=/freebsd && \
+    make -j7 && \
+    make install
+RUN cd /src/gmp-6.1.2 && \
+    ./configure --prefix=/freebsd --enable-shared --enable-static \
+      --enable-mpbsd --enable-fft --enable-cxx --host=x86_64-pc-freebsd12 && \
+    make -j7 && \
+    make install
+RUN cd /src/mpfr-4.0.2 && \
+    ./configure --prefix=/freebsd --with-gnu-ld  --enable-static \
+      --enable-shared --with-gmp=/freebsd --host=x86_64-pc-freebsd12 && \
+    make -j7 && \
+    make install
+RUN cd /src/mpc-1.1.0/ && \
+    ./configure --prefix=/freebsd --with-gnu-ld \
+      --enable-static --enable-shared --with-gmp=/freebsd \
+      --with-mpfr=/freebsd --host=x86_64-pc-freebsd12  &&\
+    make -j7 && \
+    make install
+RUN mkdir -p /src/gcc-6.4.0/build && \
+    cd /src/gcc-6.4.0/build && \
+    ../configure --without-headers --with-gnu-as --with-gnu-ld --disable-nls \
+        --enable-languages=c,c++ --enable-libssp --enable-ld \
+        --disable-libitm --disable-libquadmath --target=x86_64-pc-freebsd12 \
+        --prefix=/freebsd --with-gmp=/freebsd \
+        --with-mpc=/freebsd --with-mpfr=/freebsd --disable-libgomp && \
+    LD_LIBRARY_PATH=/freebsd/lib make -j7 && \
+    make install
+
+# Now copy the toolchain to a fresh image
+FROM debian:10-slim
+
+COPY --from=build /freebsd /freebsd
+
+env LD_LIBRARY_PATH /freebsd/lib
+env PATH /freebsd/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+env CC x86_64-pc-freebsd12-gcc
+env CPP x86_64-pc-freebsd12-cpp
+env AS x86_64-pc-freebsd12-as
+env LD x86_64-pc-freebsd12-ld
+env AR x86_64-pc-freebsd12-ar
+env RANLIB x86_64-pc-freebsd12-ranlib
+env HOST x86_64-pc-freebsd12
+
+# A host compiler is needed to link Rust build scripts even when cross compiling:
+RUN apt-get update -y && apt-get install -y curl build-essential git
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --target x86_64-unknown-freebsd
+RUN mkdir -p /rust/project /rust/target
+WORKDIR /rust/project
+VOLUME /rust/project
+ENV PATH /root/.cargo/bin:/freebsd/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+# Required for the openssl-sys crate:
+ENV OPENSSL_DIR /freebsd/x86_64-pc-freebsd12/
--- a/signald-builder.Dockerfile
+++ b/signald-builder.Dockerfile
@ -1,6 +1,6 @@
 FROM debian:latest
+RUN dpkg --add-architecture armhf && dpkg --add-architecture arm64
 ADD apt-signing-key.asc /tmp/apt-signing-key.asc
-RUN for p in dpkg-split dpkg-deb tar rm; do ln -s /usr/bin/$p /usr/sbin/$p; done
 RUN apt-get update && apt-get install -y \
  gpg \
  dpkg-dev \
@ -16,6 +16,8 @@ RUN apt-get update && apt-get install -y \
  jq \
  aptly \
  build-essential \
+  crossbuild-essential-armhf \
+  crossbuild-essential-arm64 \
  && rm -rf /var/lib/apt/lists/*

 COPY deb-scripts/release-deb.sh /usr/bin/release-deb
--- a/signald-cross-builder.Dockerfile
+++ b/signald-cross-builder.Dockerfile
@ -1,29 +0,0 @@
-FROM debian:latest
-RUN dpkg --add-architecture armhf && dpkg --add-architecture arm64
-ADD apt-signing-key.asc /tmp/apt-signing-key.asc
-RUN apt-get update && apt-get install -y \
-  gpg \
-  dpkg-dev \
-  apt-utils \
-  wget \
-  dh-make \
-  debhelper \
-  javahelper \
-  gradle \
-  default-jdk-headless \
-  git-buildpackage \
-  gradle-debian-helper \
-  jq \
-  aptly \
-  build-essential \
-  crossbuild-essential-armhf \
-  crossbuild-essential-arm64 \
-  && rm -rf /var/lib/apt/lists/*
-
-COPY deb-scripts/release-deb.sh /usr/bin/release-deb
-COPY deb-scripts/get-component.sh /usr/bin/get-component
-COPY deb-scripts/repo-cron.sh /usr/bin/repo-cron
-
-RUN adduser signald
-USER signald
-RUN gpg --no-default-keyring --keyring trustedkeys.gpg --import /tmp/apt-signing-key.asc