tools/docker: rework python jobs

* use manylinux_2_28
* use muslinux_1_2

ref: https://github.com/pypa/manylinux?tab=readme-ov-file#docker-images
This commit is contained in:
Corentin Le Molgat
2025-01-17 11:08:54 +01:00
parent af05f3bbf4
commit fef7da4eb0
9 changed files with 450 additions and 156 deletions

View File

@@ -1,52 +0,0 @@
# Create a virtual environment with all tools installed
# ref: https://hub.docker.com/_/alpine
FROM alpine:edge AS env
# Install system build dependencies
ENV PATH=/usr/local/bin:$PATH
RUN apk add --no-cache git build-base linux-headers cmake xfce4-dev-tools
ENTRYPOINT ["/bin/sh", "-c"]
CMD ["/bin/sh"]
# SWIG
RUN apk add --no-cache swig
# Python
RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \
py3-numpy py3-pandas py3-matplotlib
RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \
&& python3 -m pip install absl-py mypy mypy-protobuf
################
## OR-TOOLS ##
################
FROM env AS devel
ENV GIT_URL https://github.com/google/or-tools
ARG GIT_BRANCH
ENV GIT_BRANCH ${GIT_BRANCH:-main}
ARG GIT_SHA1
ENV GIT_SHA1 ${GIT_SHA1:-unknown}
# Download sources
# use GIT_SHA1 to modify the command
# i.e. avoid docker reusing the cache when new commit is pushed
WORKDIR /root
RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \
&& cd /project \
&& git reset --hard "${GIT_SHA1}"
WORKDIR /project
# Build project
FROM devel AS build
RUN cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPS=ON -DBUILD_PYTHON=ON -DVENV_USE_SYSTEM_SITE_PACKAGES=ON \
-DBUILD_CXX_SAMPLES=OFF -DBUILD_CXX_EXAMPLES=OFF
RUN cmake --build build -v -j8
# Rename wheel package ortools-version+musl-....
RUN cp build/python/dist/ortools-*.whl .
RUN NAME=$(ls *.whl | sed -e "s/\(ortools-[0-9\.]\+\)/\1+musl/") && mv *.whl "${NAME}"
RUN rm build/python/dist/ortools-*.whl
RUN mv *.whl build/python/dist/
FROM build AS test
RUN cmake --build build --target test

View File

@@ -1,16 +1,17 @@
FROM quay.io/pypa/manylinux2014_x86_64:latest AS env
# note: CMake 3.30.5 and SWIG 4.2.1 are already installed
FROM quay.io/pypa/manylinux_2_28_x86_64:latest AS env
# note: Almalinux:8 based image with
# CMake 3.31.2 and SWIG 4.3.0 already installed
RUN yum -y update \
&& yum -y install \
RUN dnf -y update \
&& dnf -y install \
curl wget \
git patch \
which pkgconfig autoconf libtool \
make gcc-c++ \
redhat-lsb openssl-devel pcre2-devel \
zlib-devel unzip zip \
&& yum clean all \
&& rm -rf /var/cache/yum
&& dnf clean all \
&& rm -rf /var/cache/dnf
ENTRYPOINT ["/usr/bin/bash", "-c"]
CMD ["/usr/bin/bash"]

View File

@@ -0,0 +1,45 @@
FROM quay.io/pypa/musllinux_1_2_x86_64:latest AS env
# CMake 3.31.2 and SWIG 4.3.0 already installed
# Install system build dependencies
ENV PATH=/usr/local/bin:$PATH
RUN apk add --no-cache git build-base linux-headers xfce4-dev-tools
ENTRYPOINT ["/bin/sh", "-c"]
CMD ["/bin/sh"]
## Python
#RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \
# py3-numpy py3-pandas py3-matplotlib
#RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \
#&& python3 -m pip install absl-py mypy mypy-protobuf
################
## OR-TOOLS ##
################
FROM env AS devel
ENV GIT_URL https://github.com/google/or-tools
ARG GIT_BRANCH
ENV GIT_BRANCH ${GIT_BRANCH:-main}
ARG GIT_SHA1
ENV GIT_SHA1 ${GIT_SHA1:-unknown}
# Download sources
# use GIT_SHA1 to modify the command
# i.e. avoid docker reusing the cache when new commit is pushed
RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \
&& cd /project \
&& git reset --hard "${GIT_SHA1}"
WORKDIR /project
COPY build-musllinux.sh .
RUN chmod a+x "build-musllinux.sh"
FROM devel AS build
ENV PLATFORM x86_64
ARG PYTHON_VERSION
ENV PYTHON_VERSION ${PYTHON_VERSION:-3}
RUN ./build-musllinux.sh build
FROM build as test
RUN ./build-musllinux.sh test

View File

@@ -1,18 +1,19 @@
# To build it on x86_64 please read
# https://github.com/multiarch/qemu-user-static#getting-started
FROM quay.io/pypa/manylinux2014_aarch64:latest AS env
# note: CMake 3.30.5 and SWIG 4.2.1 are already installed
FROM quay.io/pypa/manylinux_2_28_aarch64:latest AS env
# note: Almalinux:8 based image with
# CMake 3.31.2 and SWIG 4.3.0 already installed
RUN yum -y update \
&& yum -y install \
RUN dnf -y update \
&& dnf -y install \
curl wget \
git patch \
which pkgconfig autoconf libtool \
make gcc-c++ \
redhat-lsb openssl-devel pcre2-devel \
zlib-devel unzip zip \
&& yum clean all \
&& rm -rf /var/cache/yum
&& dnf clean all \
&& rm -rf /var/cache/dnf
ENTRYPOINT ["/usr/bin/bash", "-c"]
CMD ["/usr/bin/bash"]

View File

@@ -0,0 +1,45 @@
FROM quay.io/pypa/musllinux_1_2_aarch64:latest AS env
# CMake 3.31.2 and SWIG 4.3.0 already installed
# Install system build dependencies
ENV PATH=/usr/local/bin:$PATH
RUN apk add --no-cache git build-base linux-headers xfce4-dev-tools
ENTRYPOINT ["/bin/sh", "-c"]
CMD ["/bin/sh"]
## Python
#RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \
# py3-numpy py3-pandas py3-matplotlib
#RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \
#&& python3 -m pip install absl-py mypy mypy-protobuf
################
## OR-TOOLS ##
################
FROM env AS devel
ENV GIT_URL https://github.com/google/or-tools
ARG GIT_BRANCH
ENV GIT_BRANCH ${GIT_BRANCH:-main}
ARG GIT_SHA1
ENV GIT_SHA1 ${GIT_SHA1:-unknown}
# Download sources
# use GIT_SHA1 to modify the command
# i.e. avoid docker reusing the cache when new commit is pushed
RUN git clone -b "${GIT_BRANCH}" --single-branch "${GIT_URL}" /project \
&& cd /project \
&& git reset --hard "${GIT_SHA1}"
WORKDIR /project
COPY build-musllinux.sh .
RUN chmod a+x "build-musllinux.sh"
FROM devel AS build
ENV PLATFORM aarch64
ARG PYTHON_VERSION
ENV PYTHON_VERSION ${PYTHON_VERSION:-3}
RUN ./build-musllinux.sh build
FROM build as test
RUN ./build-musllinux.sh test

View File

@@ -1,49 +0,0 @@
# Create a virtual environment with all tools installed
# ref: https://hub.docker.com/_/alpine
FROM arm64v8/alpine:edge AS env
# Install system build dependencies
ENV PATH=/usr/local/bin:$PATH
RUN apk add --no-cache git build-base linux-headers cmake xfce4-dev-tools
ENTRYPOINT ["/bin/sh", "-c"]
CMD ["/bin/sh"]
# SWIG
RUN apk add --no-cache swig
# Python
RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \
py3-numpy py3-pandas py3-matplotlib
RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \
&& python3 -m pip install absl-py mypy mypy-protobuf
################
## OR-TOOLS ##
################
FROM env AS devel
ENV GIT_URL https://github.com/google/or-tools
ARG GIT_BRANCH
ENV GIT_BRANCH ${GIT_BRANCH:-main}
ARG GIT_SHA1
ENV GIT_SHA1 ${GIT_SHA1:-unknown}
# Download sources
# use GIT_SHA1 to modify the command
# i.e. avoid docker reusing the cache when new commit is pushed
RUN git clone -b "${GIT_BRANCH}" --single-branch "${GIT_URL}" /project \
&& cd /project \
&& git reset --hard "${GIT_SHA1}"
WORKDIR /project
FROM devel AS build
RUN cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPS=ON -DBUILD_PYTHON=ON
RUN cmake --build build -v -j8
# Rename wheel package ortools-version+musl-....
RUN cp build/python/dist/ortools-*.whl .
RUN NAME=$(ls *.whl | sed -e "s/\(ortools-[0-9\.]\+\)/\1+musl/") && mv *.whl "${NAME}"
RUN rm build/python/dist/ortools-*.whl
RUN mv *.whl build/python/dist/
FROM build AS test
RUN cmake --build build --target test

View File

@@ -107,6 +107,7 @@ function build_wheel() {
function check_wheel() {
assert_defined BUILD_DIR
assert_defined VENV_DIR
# Check the wheel artifact
# Arguments:
# $1 the python root directory
@@ -115,9 +116,14 @@ function check_wheel() {
exit 1
fi
# shellcheck source=/dev/null
source "${VENV_DIR}/bin/activate"
pip install -U auditwheel
# Check mypy files
declare -a MYPY_FILES=(
"ortools/algorithms/python/knapsack_solver.pyi"
"ortools/algorithms/python/set_cover.pyi"
"ortools/constraint_solver/pywrapcp.pyi"
"ortools/graph/python/linear_sum_assignment.pyi"
"ortools/graph/python/max_flow.pyi"
@@ -126,7 +132,7 @@ function check_wheel() {
"ortools/linear_solver/python/model_builder_helper.pyi"
"ortools/linear_solver/pywraplp.pyi"
"ortools/pdlp/python/pdlp.pyi"
"ortools/sat/python/swig_helper.pyi"
"ortools/sat/python/cp_model_helper.pyi"
"ortools/scheduling/python/rcpsp.pyi"
"ortools/util/python/sorted_interval_list.pyi"
)
@@ -142,12 +148,15 @@ function check_wheel() {
for FILE in *.whl; do
# if no files found do nothing
[[ -e "$FILE" ]] || continue
auditwheel show "$FILE" || true
auditwheel -v repair --plat "manylinux2014_$PLATFORM" "$FILE" -w "$export_root"
#auditwheel -v repair --plat manylinux2014_x86_64 "$FILE" -w "$export_root"
#auditwheel -v repair --plat manylinux2014_aarch64 "$FILE" -w "$export_root"
python -m auditwheel show "$FILE" || true
python -m auditwheel -v repair --plat "manylinux_2_28_$PLATFORM" "$FILE" -w "$export_root"
#python -m auditwheel -v repair --plat manylinux_2_28_x86_64 "$FILE" -w "$export_root"
#python -m auditwheel -v repair --plat manylinux_2_28_aarch64 "$FILE" -w "$export_root"
done
popd
# Restore environment
deactivate
}
function test_wheel() {

View File

@@ -0,0 +1,271 @@
#!/usr/bin/env bash
# Copyright 2010-2025 Google LLC
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Build all the wheel artifacts for the platforms supported by musllinux_1_2 and
# export them to the specified location.
set -exo pipefail
function assert_defined(){
if [[ -z "${!1}" ]]; then
>&2 echo "Variable '${1}' must be defined"
exit 1
fi
}
function usage() {
local -r NAME=$(basename "$0")
echo -e "$NAME - Build using a cross toolchain.
SYNOPSIS
\t$NAME [-h|--help] [build|test|all]
DESCRIPTION
\tBuild wheel artifacts.
\tYou MUST define the following variables before running this script:
\t* PLATFORM: x86_64 aarch64
\t* PYTHON_VERSION: 3 38 39 310 311 312 313
note: PYTHON_VERSION=3 will generate for all pythons which could take time...
OPTIONS
\t-h --help: show this help text
\tbuild: build the project using each python (note: remove previous build dir)
\ttest: install each wheel in a venv then test it (note: don't build !)
\tall: build + test (default)
EXAMPLES
* Using export
export PLATFORM=x86_64
export PYTHON_VERSION=39
$0 build
* One-liner:
PLATFORM=x86_64 PYTHON_VERSION=39 $0 build"
}
function contains_element() {
# Look for the presence of an element in an array. Echoes '0' if found,
# '1' otherwise.
# Arguments:
# $1 the element to be searched
# $2 the array to search into
local e match="$1"
shift
for e; do
[[ "$e" == "$match" ]] && return 0
done
return 1
}
function build_wheel() {
assert_defined BUILD_DIR
assert_defined VENV_DIR
# Build the wheel artifact
# Arguments:
# $1 the python root directory
if [[ "$#" -ne 1 ]]; then
echo "$0 called with an illegal number of parameters"
exit 1
fi
# Create and activate virtualenv
# this is needed so protoc can call the correct python executable
local -r PYBIN="$1/bin"
"${PYBIN}/pip" install virtualenv
"${PYBIN}/virtualenv" -p "${PYBIN}/python" "${VENV_DIR}"
# shellcheck source=/dev/null
source "${VENV_DIR}/bin/activate"
pip install -U pip setuptools wheel absl-py # absl-py is needed by make test_python
pip install -U mypy mypy-protobuf # need to generate protobuf mypy files
echo "current dir: $(pwd)"
if [[ ! -e "CMakeLists.txt" ]] || [[ ! -d "cmake" ]]; then
>&2 echo "Can't find project's CMakeLists.txt or cmake"
exit 2
fi
cmake -S. -B"${BUILD_DIR}" \
-DCMAKE_BUILD_TYPE=Release \
-DBUILD_DEPS=ON -DBUILD_PYTHON=ON -DPython3_ROOT_DIR="$1" \
-DBUILD_TESTING=OFF -DBUILD_SAMPLES=OFF -DBUILD_EXAMPLES=OFF #--debug-find
cmake --build "${BUILD_DIR}" -v -j4
# Restore environment
deactivate
}
function check_wheel() {
assert_defined BUILD_DIR
# Check the wheel artifact
# Arguments:
# $1 the python root directory
if [[ "$#" -ne 1 ]]; then
echo "$0 called with an illegal number of parameters"
exit 1
fi
# Check mypy files
declare -a MYPY_FILES=(
"ortools/algorithms/python/knapsack_solver.pyi"
"ortools/algorithms/python/set_cover.pyi"
"ortools/constraint_solver/pywrapcp.pyi"
"ortools/graph/python/linear_sum_assignment.pyi"
"ortools/graph/python/max_flow.pyi"
"ortools/graph/python/min_cost_flow.pyi"
"ortools/init/python/init.pyi"
"ortools/linear_solver/python/model_builder_helper.pyi"
"ortools/linear_solver/pywraplp.pyi"
"ortools/pdlp/python/pdlp.pyi"
"ortools/sat/python/cp_model_helper.pyi"
"ortools/scheduling/python/rcpsp.pyi"
"ortools/util/python/sorted_interval_list.pyi"
)
for FILE in "${MYPY_FILES[@]}"; do
if [[ ! -f "${BUILD_DIR}/python/${FILE}" ]]; then
echo "error: ${FILE} missing in the python project"
exit 1
fi
done
# Check all generated wheel packages
pushd "${BUILD_DIR}/python/dist"
for FILE in *.whl; do
# if no files found do nothing
[[ -e "$FILE" ]] || continue
auditwheel show "$FILE" || true
auditwheel -v repair --plat "musllinux_1_2_$PLATFORM" "$FILE" -w "$export_root"
#auditwheel -v repair --plat musllinux_1_2_x86_64 "$FILE" -w "$export_root"
#auditwheel -v repair --plat musllinux_1_2_aarch64 "$FILE" -w "$export_root"
done
popd
}
function test_wheel() {
assert_defined BUILD_DIR
assert_defined TEST_DIR
# Test the wheel artifacts
# Arguments:
# $1 the python root directory
if [[ "$#" -ne 1 ]]; then
echo "$0 called with an illegal number of parameters"
exit 1
fi
# Create and activate virtualenv
local -r PYBIN="$1/bin"
"${PYBIN}/pip" install virtualenv
"${PYBIN}/virtualenv" -p "${PYBIN}/python" "${TEST_DIR}"
# shellcheck source=/dev/null
source "${TEST_DIR}/bin/activate"
pip install -U pip setuptools wheel
# Install the wheel artifact
#pwd
local -r WHEEL_FILE=$(find "${BUILD_DIR}"/python/dist/*.whl | head -1)
echo "WHEEL file: ${WHEEL_FILE}"
pip install --no-cache-dir "$WHEEL_FILE"
pip show ortools
# Python scripts to be used as tests for the installed wheel. This list of files
# has been taken from the 'test_python' make target.
declare -a TESTS=(
"ortools/algorithms/samples/simple_knapsack_program.py"
"ortools/graph/samples/simple_max_flow_program.py"
"ortools/graph/samples/simple_min_cost_flow_program.py"
"ortools/linear_solver/samples/simple_lp_program.py"
"ortools/linear_solver/samples/simple_mip_program.py"
"ortools/sat/samples/simple_sat_program.py"
"ortools/constraint_solver/samples/tsp.py"
"ortools/constraint_solver/samples/vrp.py"
"ortools/constraint_solver/samples/cvrptw_break.py"
)
# Run all the specified test scripts using the current environment.
local -r ROOT_DIR=$(pwd)
pushd "$(mktemp -d)" # ensure we are not importing something from $PWD
python --version
for TEST in "${TESTS[@]}"; do
python "${ROOT_DIR}/${TEST}"
done
popd
# Restore environment
deactivate
}
function build() {
# For each python platform provided by musllinux, build and test artifacts.
for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do
# shellcheck disable=SC2155
PYTAG=$(basename "$PYROOT")
echo "Python: $PYTAG"
# Check for platforms to be skipped
if contains_element "$PYTAG" "${SKIPS[@]}"; then
>&2 echo "skipping deprecated platform $PYTAG"
continue
fi
BUILD_DIR="build_${PYTAG}"
VENV_DIR="env_${PYTAG}"
build_wheel "$PYROOT"
check_wheel "$PYROOT"
done
}
function tests() {
# For each python platform provided by musllinux, build and test artifacts.
for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do
# shellcheck disable=SC2155
PYTAG=$(basename "$PYROOT")
echo "Python: $PYTAG"
# Check for platforms to be skipped
if contains_element "$PYTAG" "${SKIPS[@]}"; then
>&2 echo "skipping deprecated platform $PYTAG"
continue
fi
BUILD_DIR="build_${PYTAG}"
TEST_DIR="test_${PYTAG}"
test_wheel "$PYROOT"
done
}
# Main
function main() {
case ${1} in
-h | --help)
usage; exit ;;
esac
assert_defined PLATFORM
assert_defined PYTHON_VERSION
# Setup
declare -a SKIPS=( "cp36-cp36m" "cp37-cp37m" )
case ${1} in
build)
build ;;
test)
tests ;;
*)
build
tests ;;
esac
}
main "${1:-all}"