tools/docker: rework python jobs
* use manylinux_2_28 * use muslinux_1_2 ref: https://github.com/pypa/manylinux?tab=readme-ov-file#docker-images
This commit is contained in:
Corentin Le Molgat
@@ -33,25 +33,25 @@ help:
|
||||
@echo -e "\t${BOLD}test_archives${RESET}: Test each OR-Tools archives for all ${BOLD}<distro>${RESET} and ${BOLD}<lang>${RESET}."
|
||||
@echo
|
||||
@echo -e "${BOLD}PYTHON TARGETS${RESET}"
|
||||
@echo -e "\t${BOLD}python${RESET}: Build alpine and manylinux2014 python 'ortools' wheel packages (3.8+)."
|
||||
@echo -e "\t${BOLD}python${RESET}: Build musllinux and manylinux python 'ortools' wheel packages (3.8+)."
|
||||
@echo -e "\t${BOLD}python_<platform>${RESET}: Build all python 'ortools' wheel packages (3.8+) for a specific platform."
|
||||
@echo -e "\t${BOLD}python_<platform>_<step>${RESET}: Build all python 'ortools' wheel packages (3.8+) for a specific platform."
|
||||
@echo -e "\t${BOLD}python_<target>_<step>${RESET}: Build python 'ortools' wheel packages (3.8+) for a specific target."
|
||||
@echo -e "\t${BOLD}save_python_<target>${RESET}: Save python 'ortools' image."
|
||||
@echo -e "\t${BOLD}clean_python_<target>${RESET}: Clean manylinux2014 and alpine python 'ortools' wheel packages."
|
||||
@echo -e "\t${BOLD}clean_python_<target>${RESET}: Clean manylinux and musllinux python 'ortools' wheel packages."
|
||||
@echo -e "\t${BOLD}sh_python_<target>${RESET}: Run a container using the python 'ortools' image."
|
||||
@echo
|
||||
@echo -e "\t${BOLD}<platform>${RESET}:"
|
||||
@echo -e "\t\t${BOLD}amd64${RESET}"
|
||||
@echo -e "\t\t${BOLD}arm64v8${RESET}"
|
||||
@echo -e "\t\t${BOLD}arm64${RESET}"
|
||||
@echo
|
||||
@echo -e "\t${BOLD}<target>${RESET}:"
|
||||
@echo -e "\t\t${BOLD}<platform>_<distro>${RESET}"
|
||||
@echo -e "\t\t${BOLD}<platform>_manylinux_cp<version>${RESET}"
|
||||
@echo
|
||||
@echo -e "\t${BOLD}<distro>${RESET}:"
|
||||
@echo -e "\t\t${BOLD}alpine${RESET} (latest)"
|
||||
@echo -e "\t\t${BOLD}manylinux${RESET} (manylinux2014)"
|
||||
@echo -e "\t\t${BOLD}musllinux${RESET} (musllinux_1_2)"
|
||||
@echo -e "\t\t${BOLD}manylinux${RESET} (manylinux_2_28)"
|
||||
@echo
|
||||
@echo -e "\t${BOLD}<version>${RESET}:"
|
||||
@echo -e "\t\t${BOLD}38${RESET} Python3.8"
|
||||
@@ -68,7 +68,7 @@ help:
|
||||
@echo -e "\t\t${BOLD}test${RESET}"
|
||||
@echo -e "\t\t${BOLD}export${RESET}"
|
||||
@echo -e "\te.g. 'make python_amd64_manylinux_cp39_export'"
|
||||
@echo -e "\te.g. 'make python_arm64v8_alpine_export'"
|
||||
@echo -e "\te.g. 'make python_arm64_musllinux_export'"
|
||||
@echo
|
||||
@echo
|
||||
@echo -e "${BOLD}ARCHIVE TARGETS${RESET}"
|
||||
@@ -187,8 +187,8 @@ test_delivery: test_archives
|
||||
# $* stem
|
||||
# $< first prerequist
|
||||
# $@ target name
|
||||
PYTHON_PLATFORMS := amd64 arm64v8
|
||||
PYTHON_DISTROS := manylinux alpine
|
||||
PYTHON_PLATFORMS := amd64 arm64
|
||||
PYTHON_DISTROS := manylinux musllinux
|
||||
PYTHON_STAGES := env devel build test
|
||||
|
||||
export:
|
||||
@@ -230,9 +230,8 @@ python_$1_manylinux_cp$2_$3: python/$1/manylinux.Dockerfile export/python/manyli
|
||||
|
||||
.PHONY: save_python_$1_manylinux_cp$2_$3
|
||||
save_python_$1_manylinux_cp$2_$3: cache/python/docker_$1_manylinux_cp$2_$3.tar
|
||||
cache/python/docker_$1_manylinux_cp$2_$3.tar: python_$1_manylinux_cp$2_$3
|
||||
cache/python/docker_$1_manylinux_cp$2_$3.tar: python_$1_manylinux_cp$2_$3 | cache/python
|
||||
@rm -f $$@
|
||||
mkdir -p cache/python/
|
||||
docker save ${IMAGE}:$$< -o $$@
|
||||
|
||||
.PHONY: clean_python_$1_manylinux_cp$2_$3
|
||||
@@ -266,7 +265,7 @@ python_$1_manylinux_cp$2_export: python_$1_manylinux_cp$2_build
|
||||
endef
|
||||
|
||||
$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,amd64,${version})))
|
||||
$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,arm64v8,${version})))
|
||||
$(foreach version,${PYTHON_VERSIONS},$(eval $(call manylinux_outer,arm64,${version})))
|
||||
|
||||
# Merge
|
||||
define manylinux_merge =
|
||||
@@ -281,41 +280,45 @@ clean_python_$1_manylinux_$2: $(addprefix clean_python_$1_manylinux_cp, $(addsuf
|
||||
endef
|
||||
|
||||
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,amd64,${stage})))
|
||||
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,arm64v8,${stage})))
|
||||
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call manylinux_merge,arm64,${stage})))
|
||||
|
||||
## ALPINE ##
|
||||
export/python/alpine: | export/python
|
||||
## MUSLLINUX ##
|
||||
export/python/musllinux: | export/python
|
||||
-mkdir -p $@
|
||||
|
||||
define alpine_loop =
|
||||
#$$(info alpine_loop: PLATFORM:'$1' STAGE:'$2')
|
||||
export/python/musllinux/build-musllinux.sh: python/build-musllinux.sh | export/python/musllinux
|
||||
cp $< $@
|
||||
|
||||
.PHONY: python_$1_alpine_$2
|
||||
python_$1_alpine_$2: python/$1/alpine.Dockerfile | export/python/alpine
|
||||
define musllinux_inner =
|
||||
#$$(info musllinux_inner: PLATFORM:'$1' VERSION:'$2' STAGE:'$3')
|
||||
|
||||
.PHONY: python_$1_musllinux_cp$2_$3
|
||||
python_$1_musllinux_cp$2_$3: python/$1/musllinux.Dockerfile | export/python/musllinux/build-musllinux.sh
|
||||
@docker image rm -f ${IMAGE}:$$@ 2>/dev/null
|
||||
${DOCKER_BUILD_CMD} \
|
||||
${DOCKER_BUILDX_CMD} --platform linux/$1 \
|
||||
--tag ${IMAGE}:$$@ \
|
||||
--build-arg GIT_BRANCH=${OR_TOOLS_BRANCH} \
|
||||
--build-arg GIT_SHA1=${OR_TOOLS_SHA1} \
|
||||
--build-arg OR_TOOLS_PATCH=${OR_TOOLS_PATCH} \
|
||||
--target=$2 \
|
||||
--build-arg PYTHON_VERSION=$2 \
|
||||
--target=$3 \
|
||||
-f $$< \
|
||||
export/python/alpine
|
||||
export/python/musllinux
|
||||
|
||||
.PHONY: save_python_$1_alpine_$2
|
||||
save_python_$1_alpine_$2: cache/python/docker_$1_alpine_$2.tar
|
||||
cache/python/docker_$1_alpine_$2.tar: python_$1_alpine_$2 | cache/python
|
||||
.PHONY: save_python_$1_musllinux_cp$2_$3
|
||||
save_python_$1_musllinux_cp$2_$3: cache/python/docker_$1_musllinux_cp$2_$3.tar
|
||||
cache/python/docker_$1_musllinux_cp$2_$3.tar: python_$1_musllinux_cp$2_$3 | cache/python
|
||||
@rm -f $$@
|
||||
docker save ${IMAGE}:$$< -o $$@
|
||||
|
||||
.PHONY: clean_python_$1_alpine_$2
|
||||
clean_python_$1_alpine_$2: python/$1/alpine.Dockerfile | export/python/alpine
|
||||
docker image rm -f ${IMAGE}:python_$1_alpine_$2 2>/dev/null
|
||||
rm -f cache/python/docker_$1_alpine_$2.tar
|
||||
.PHONY: clean_python_$1_musllinux_cp$2_$3
|
||||
clean_python_$1_musllinux_cp$2_$3: python/$1/musllinux.Dockerfile | export/python/musllinux/build-musllinux.sh
|
||||
docker image rm -f ${IMAGE}:python_$1_musllinux_cp$2_$3 2>/dev/null
|
||||
rm -f cache/python/docker_$1_musllinux_cp$2_$3.tar
|
||||
|
||||
# Debug purpose
|
||||
.PHONY: sh_python_$1_alpine_$2
|
||||
sh_python_$1_alpine_$2: python_$1_alpine_$2
|
||||
.PHONY: sh_python_$1_musllinux_cp$2_$3
|
||||
sh_python_$1_musllinux_cp$2_$3: python_$1_musllinux_cp$2_$3
|
||||
${DOCKER_RUN_CMD} \
|
||||
-v `pwd`/export:/export \
|
||||
-it \
|
||||
@@ -323,18 +326,38 @@ sh_python_$1_alpine_$2: python_$1_alpine_$2
|
||||
${IMAGE}:$$<
|
||||
endef
|
||||
|
||||
$(foreach stage,${PYTHON_STAGES},$(eval $(call alpine_loop,amd64,${stage})))
|
||||
$(foreach stage,${PYTHON_STAGES},$(eval $(call alpine_loop,arm64v8,${stage})))
|
||||
define musllinux_outer =
|
||||
#$$(info musllinux_outer: PLATFORM: '$1' VERSION: '$2')
|
||||
|
||||
alpine_export_targets = $(addprefix python_, $(addsuffix _alpine_export, ${PYTHON_PLATFORMS}))
|
||||
.PHONY: ${alpine_export_targets}
|
||||
${alpine_export_targets}: python_%_alpine_export: python_%_alpine_build | export
|
||||
$$(foreach stage,${PYTHON_STAGES},$$(eval $$(call musllinux_inner,$1,$2,$${stage})))
|
||||
|
||||
.PHONY: python_$1_musllinux_cp$2_export
|
||||
python_$1_musllinux_cp$2_export: python_$1_musllinux_cp$2_build
|
||||
${DOCKER_RUN_CMD} \
|
||||
-v `pwd`/export:/export \
|
||||
-it \
|
||||
--name ortools_$< \
|
||||
${IMAGE}:$< \
|
||||
--name ortools_$$< \
|
||||
${IMAGE}:$$< \
|
||||
"cp build*/python/dist/*.whl /export/python"
|
||||
endef
|
||||
|
||||
$(foreach version,${PYTHON_VERSIONS},$(eval $(call musllinux_outer,amd64,${version})))
|
||||
$(foreach version,${PYTHON_VERSIONS},$(eval $(call musllinux_outer,arm64,${version})))
|
||||
|
||||
# Merge
|
||||
define musllinux_merge =
|
||||
#$$(info musllinux_merge: PLATFORM:'$1' STAGE:'$2')
|
||||
|
||||
.PHONY: python_$1_musllinux_$2
|
||||
python_$1_musllinux_$2: $(addprefix python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
|
||||
.PHONY: save_python_$1_musllinux_$2
|
||||
save_python_$1_musllinux_$2: $(addprefix save_python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
|
||||
.PHONY: clean_python_$1_musllinux_$2
|
||||
clean_python_$1_musllinux_$2: $(addprefix clean_python_$1_musllinux_cp, $(addsuffix _$2, ${PYTHON_VERSIONS}))
|
||||
endef
|
||||
|
||||
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call musllinux_merge,amd64,${stage})))
|
||||
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call musllinux_merge,arm64,${stage})))
|
||||
|
||||
## MERGE DISTRO ##
|
||||
define python_distro_merge =
|
||||
@@ -349,7 +372,7 @@ clean_python_$1_$2: $(addprefix clean_python_$1_, $(addsuffix _$2, ${PYTHON_DIST
|
||||
endef
|
||||
|
||||
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,amd64,${stage})))
|
||||
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,arm64v8,${stage})))
|
||||
$(foreach stage,${PYTHON_STAGES} export,$(eval $(call python_distro_merge,arm64,${stage})))
|
||||
|
||||
## MERGE PLATFORM ##
|
||||
define clean_python_platform =
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
# Create a virtual environment with all tools installed
|
||||
# ref: https://hub.docker.com/_/alpine
|
||||
FROM alpine:edge AS env
|
||||
|
||||
# Install system build dependencies
|
||||
ENV PATH=/usr/local/bin:$PATH
|
||||
RUN apk add --no-cache git build-base linux-headers cmake xfce4-dev-tools
|
||||
ENTRYPOINT ["/bin/sh", "-c"]
|
||||
CMD ["/bin/sh"]
|
||||
|
||||
# SWIG
|
||||
RUN apk add --no-cache swig
|
||||
|
||||
# Python
|
||||
RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \
|
||||
py3-numpy py3-pandas py3-matplotlib
|
||||
RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \
|
||||
&& python3 -m pip install absl-py mypy mypy-protobuf
|
||||
|
||||
################
|
||||
## OR-TOOLS ##
|
||||
################
|
||||
FROM env AS devel
|
||||
ENV GIT_URL https://github.com/google/or-tools
|
||||
|
||||
ARG GIT_BRANCH
|
||||
ENV GIT_BRANCH ${GIT_BRANCH:-main}
|
||||
ARG GIT_SHA1
|
||||
ENV GIT_SHA1 ${GIT_SHA1:-unknown}
|
||||
|
||||
# Download sources
|
||||
# use GIT_SHA1 to modify the command
|
||||
# i.e. avoid docker reusing the cache when new commit is pushed
|
||||
WORKDIR /root
|
||||
RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \
|
||||
&& cd /project \
|
||||
&& git reset --hard "${GIT_SHA1}"
|
||||
WORKDIR /project
|
||||
|
||||
# Build project
|
||||
FROM devel AS build
|
||||
RUN cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPS=ON -DBUILD_PYTHON=ON -DVENV_USE_SYSTEM_SITE_PACKAGES=ON \
|
||||
-DBUILD_CXX_SAMPLES=OFF -DBUILD_CXX_EXAMPLES=OFF
|
||||
RUN cmake --build build -v -j8
|
||||
# Rename wheel package ortools-version+musl-....
|
||||
RUN cp build/python/dist/ortools-*.whl .
|
||||
RUN NAME=$(ls *.whl | sed -e "s/\(ortools-[0-9\.]\+\)/\1+musl/") && mv *.whl "${NAME}"
|
||||
RUN rm build/python/dist/ortools-*.whl
|
||||
RUN mv *.whl build/python/dist/
|
||||
|
||||
FROM build AS test
|
||||
RUN cmake --build build --target test
|
||||
@@ -1,16 +1,17 @@
|
||||
FROM quay.io/pypa/manylinux2014_x86_64:latest AS env
|
||||
# note: CMake 3.30.5 and SWIG 4.2.1 are already installed
|
||||
FROM quay.io/pypa/manylinux_2_28_x86_64:latest AS env
|
||||
# note: Almalinux:8 based image with
|
||||
# CMake 3.31.2 and SWIG 4.3.0 already installed
|
||||
|
||||
RUN yum -y update \
|
||||
&& yum -y install \
|
||||
RUN dnf -y update \
|
||||
&& dnf -y install \
|
||||
curl wget \
|
||||
git patch \
|
||||
which pkgconfig autoconf libtool \
|
||||
make gcc-c++ \
|
||||
redhat-lsb openssl-devel pcre2-devel \
|
||||
zlib-devel unzip zip \
|
||||
&& yum clean all \
|
||||
&& rm -rf /var/cache/yum
|
||||
&& dnf clean all \
|
||||
&& rm -rf /var/cache/dnf
|
||||
ENTRYPOINT ["/usr/bin/bash", "-c"]
|
||||
CMD ["/usr/bin/bash"]
|
||||
|
||||
|
||||
45
tools/docker/python/amd64/musllinux.Dockerfile
Normal file
45
tools/docker/python/amd64/musllinux.Dockerfile
Normal file
@@ -0,0 +1,45 @@
|
||||
FROM quay.io/pypa/musllinux_1_2_x86_64:latest AS env
|
||||
# CMake 3.31.2 and SWIG 4.3.0 already installed
|
||||
|
||||
# Install system build dependencies
|
||||
ENV PATH=/usr/local/bin:$PATH
|
||||
RUN apk add --no-cache git build-base linux-headers xfce4-dev-tools
|
||||
ENTRYPOINT ["/bin/sh", "-c"]
|
||||
CMD ["/bin/sh"]
|
||||
|
||||
## Python
|
||||
#RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \
|
||||
# py3-numpy py3-pandas py3-matplotlib
|
||||
#RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \
|
||||
#&& python3 -m pip install absl-py mypy mypy-protobuf
|
||||
|
||||
################
|
||||
## OR-TOOLS ##
|
||||
################
|
||||
FROM env AS devel
|
||||
ENV GIT_URL https://github.com/google/or-tools
|
||||
|
||||
ARG GIT_BRANCH
|
||||
ENV GIT_BRANCH ${GIT_BRANCH:-main}
|
||||
ARG GIT_SHA1
|
||||
ENV GIT_SHA1 ${GIT_SHA1:-unknown}
|
||||
|
||||
# Download sources
|
||||
# use GIT_SHA1 to modify the command
|
||||
# i.e. avoid docker reusing the cache when new commit is pushed
|
||||
RUN git clone -b "${GIT_BRANCH}" --single-branch "$GIT_URL" /project \
|
||||
&& cd /project \
|
||||
&& git reset --hard "${GIT_SHA1}"
|
||||
WORKDIR /project
|
||||
|
||||
COPY build-musllinux.sh .
|
||||
RUN chmod a+x "build-musllinux.sh"
|
||||
|
||||
FROM devel AS build
|
||||
ENV PLATFORM x86_64
|
||||
ARG PYTHON_VERSION
|
||||
ENV PYTHON_VERSION ${PYTHON_VERSION:-3}
|
||||
RUN ./build-musllinux.sh build
|
||||
|
||||
FROM build as test
|
||||
RUN ./build-musllinux.sh test
|
||||
@@ -1,18 +1,19 @@
|
||||
# To build it on x86_64 please read
|
||||
# https://github.com/multiarch/qemu-user-static#getting-started
|
||||
FROM quay.io/pypa/manylinux2014_aarch64:latest AS env
|
||||
# note: CMake 3.30.5 and SWIG 4.2.1 are already installed
|
||||
FROM quay.io/pypa/manylinux_2_28_aarch64:latest AS env
|
||||
# note: Almalinux:8 based image with
|
||||
# CMake 3.31.2 and SWIG 4.3.0 already installed
|
||||
|
||||
RUN yum -y update \
|
||||
&& yum -y install \
|
||||
RUN dnf -y update \
|
||||
&& dnf -y install \
|
||||
curl wget \
|
||||
git patch \
|
||||
which pkgconfig autoconf libtool \
|
||||
make gcc-c++ \
|
||||
redhat-lsb openssl-devel pcre2-devel \
|
||||
zlib-devel unzip zip \
|
||||
&& yum clean all \
|
||||
&& rm -rf /var/cache/yum
|
||||
&& dnf clean all \
|
||||
&& rm -rf /var/cache/dnf
|
||||
ENTRYPOINT ["/usr/bin/bash", "-c"]
|
||||
CMD ["/usr/bin/bash"]
|
||||
|
||||
45
tools/docker/python/arm64/musllinux.Dockerfile
Normal file
45
tools/docker/python/arm64/musllinux.Dockerfile
Normal file
@@ -0,0 +1,45 @@
|
||||
FROM quay.io/pypa/musllinux_1_2_aarch64:latest AS env
|
||||
# CMake 3.31.2 and SWIG 4.3.0 already installed
|
||||
|
||||
# Install system build dependencies
|
||||
ENV PATH=/usr/local/bin:$PATH
|
||||
RUN apk add --no-cache git build-base linux-headers xfce4-dev-tools
|
||||
ENTRYPOINT ["/bin/sh", "-c"]
|
||||
CMD ["/bin/sh"]
|
||||
|
||||
## Python
|
||||
#RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \
|
||||
# py3-numpy py3-pandas py3-matplotlib
|
||||
#RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \
|
||||
#&& python3 -m pip install absl-py mypy mypy-protobuf
|
||||
|
||||
################
|
||||
## OR-TOOLS ##
|
||||
################
|
||||
FROM env AS devel
|
||||
ENV GIT_URL https://github.com/google/or-tools
|
||||
|
||||
ARG GIT_BRANCH
|
||||
ENV GIT_BRANCH ${GIT_BRANCH:-main}
|
||||
ARG GIT_SHA1
|
||||
ENV GIT_SHA1 ${GIT_SHA1:-unknown}
|
||||
|
||||
# Download sources
|
||||
# use GIT_SHA1 to modify the command
|
||||
# i.e. avoid docker reusing the cache when new commit is pushed
|
||||
RUN git clone -b "${GIT_BRANCH}" --single-branch "${GIT_URL}" /project \
|
||||
&& cd /project \
|
||||
&& git reset --hard "${GIT_SHA1}"
|
||||
WORKDIR /project
|
||||
|
||||
COPY build-musllinux.sh .
|
||||
RUN chmod a+x "build-musllinux.sh"
|
||||
|
||||
FROM devel AS build
|
||||
ENV PLATFORM aarch64
|
||||
ARG PYTHON_VERSION
|
||||
ENV PYTHON_VERSION ${PYTHON_VERSION:-3}
|
||||
RUN ./build-musllinux.sh build
|
||||
|
||||
FROM build as test
|
||||
RUN ./build-musllinux.sh test
|
||||
@@ -1,49 +0,0 @@
|
||||
# Create a virtual environment with all tools installed
|
||||
# ref: https://hub.docker.com/_/alpine
|
||||
FROM arm64v8/alpine:edge AS env
|
||||
|
||||
# Install system build dependencies
|
||||
ENV PATH=/usr/local/bin:$PATH
|
||||
RUN apk add --no-cache git build-base linux-headers cmake xfce4-dev-tools
|
||||
ENTRYPOINT ["/bin/sh", "-c"]
|
||||
CMD ["/bin/sh"]
|
||||
|
||||
# SWIG
|
||||
RUN apk add --no-cache swig
|
||||
|
||||
# Python
|
||||
RUN apk add --no-cache python3-dev py3-pip py3-wheel py3-virtualenv \
|
||||
py3-numpy py3-pandas py3-matplotlib
|
||||
RUN rm -f /usr/lib/python3.*/EXTERNALLY-MANAGED \
|
||||
&& python3 -m pip install absl-py mypy mypy-protobuf
|
||||
|
||||
################
|
||||
## OR-TOOLS ##
|
||||
################
|
||||
FROM env AS devel
|
||||
ENV GIT_URL https://github.com/google/or-tools
|
||||
|
||||
ARG GIT_BRANCH
|
||||
ENV GIT_BRANCH ${GIT_BRANCH:-main}
|
||||
ARG GIT_SHA1
|
||||
ENV GIT_SHA1 ${GIT_SHA1:-unknown}
|
||||
|
||||
# Download sources
|
||||
# use GIT_SHA1 to modify the command
|
||||
# i.e. avoid docker reusing the cache when new commit is pushed
|
||||
RUN git clone -b "${GIT_BRANCH}" --single-branch "${GIT_URL}" /project \
|
||||
&& cd /project \
|
||||
&& git reset --hard "${GIT_SHA1}"
|
||||
WORKDIR /project
|
||||
|
||||
FROM devel AS build
|
||||
RUN cmake -S. -Bbuild -DCMAKE_BUILD_TYPE=Release -DBUILD_DEPS=ON -DBUILD_PYTHON=ON
|
||||
RUN cmake --build build -v -j8
|
||||
# Rename wheel package ortools-version+musl-....
|
||||
RUN cp build/python/dist/ortools-*.whl .
|
||||
RUN NAME=$(ls *.whl | sed -e "s/\(ortools-[0-9\.]\+\)/\1+musl/") && mv *.whl "${NAME}"
|
||||
RUN rm build/python/dist/ortools-*.whl
|
||||
RUN mv *.whl build/python/dist/
|
||||
|
||||
FROM build AS test
|
||||
RUN cmake --build build --target test
|
||||
@@ -107,6 +107,7 @@ function build_wheel() {
|
||||
|
||||
function check_wheel() {
|
||||
assert_defined BUILD_DIR
|
||||
assert_defined VENV_DIR
|
||||
# Check the wheel artifact
|
||||
# Arguments:
|
||||
# $1 the python root directory
|
||||
@@ -115,9 +116,14 @@ function check_wheel() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${VENV_DIR}/bin/activate"
|
||||
pip install -U auditwheel
|
||||
|
||||
# Check mypy files
|
||||
declare -a MYPY_FILES=(
|
||||
"ortools/algorithms/python/knapsack_solver.pyi"
|
||||
"ortools/algorithms/python/set_cover.pyi"
|
||||
"ortools/constraint_solver/pywrapcp.pyi"
|
||||
"ortools/graph/python/linear_sum_assignment.pyi"
|
||||
"ortools/graph/python/max_flow.pyi"
|
||||
@@ -126,7 +132,7 @@ function check_wheel() {
|
||||
"ortools/linear_solver/python/model_builder_helper.pyi"
|
||||
"ortools/linear_solver/pywraplp.pyi"
|
||||
"ortools/pdlp/python/pdlp.pyi"
|
||||
"ortools/sat/python/swig_helper.pyi"
|
||||
"ortools/sat/python/cp_model_helper.pyi"
|
||||
"ortools/scheduling/python/rcpsp.pyi"
|
||||
"ortools/util/python/sorted_interval_list.pyi"
|
||||
)
|
||||
@@ -142,12 +148,15 @@ function check_wheel() {
|
||||
for FILE in *.whl; do
|
||||
# if no files found do nothing
|
||||
[[ -e "$FILE" ]] || continue
|
||||
auditwheel show "$FILE" || true
|
||||
auditwheel -v repair --plat "manylinux2014_$PLATFORM" "$FILE" -w "$export_root"
|
||||
#auditwheel -v repair --plat manylinux2014_x86_64 "$FILE" -w "$export_root"
|
||||
#auditwheel -v repair --plat manylinux2014_aarch64 "$FILE" -w "$export_root"
|
||||
python -m auditwheel show "$FILE" || true
|
||||
python -m auditwheel -v repair --plat "manylinux_2_28_$PLATFORM" "$FILE" -w "$export_root"
|
||||
#python -m auditwheel -v repair --plat manylinux_2_28_x86_64 "$FILE" -w "$export_root"
|
||||
#python -m auditwheel -v repair --plat manylinux_2_28_aarch64 "$FILE" -w "$export_root"
|
||||
done
|
||||
popd
|
||||
|
||||
# Restore environment
|
||||
deactivate
|
||||
}
|
||||
|
||||
function test_wheel() {
|
||||
|
||||
271
tools/docker/python/build-musllinux.sh
Executable file
271
tools/docker/python/build-musllinux.sh
Executable file
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2010-2025 Google LLC
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Build all the wheel artifacts for the platforms supported by musllinux_1_2 and
|
||||
# export them to the specified location.
|
||||
set -exo pipefail
|
||||
|
||||
function assert_defined(){
|
||||
if [[ -z "${!1}" ]]; then
|
||||
>&2 echo "Variable '${1}' must be defined"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function usage() {
|
||||
local -r NAME=$(basename "$0")
|
||||
echo -e "$NAME - Build using a cross toolchain.
|
||||
|
||||
SYNOPSIS
|
||||
\t$NAME [-h|--help] [build|test|all]
|
||||
|
||||
DESCRIPTION
|
||||
\tBuild wheel artifacts.
|
||||
|
||||
\tYou MUST define the following variables before running this script:
|
||||
\t* PLATFORM: x86_64 aarch64
|
||||
\t* PYTHON_VERSION: 3 38 39 310 311 312 313
|
||||
note: PYTHON_VERSION=3 will generate for all pythons which could take time...
|
||||
|
||||
OPTIONS
|
||||
\t-h --help: show this help text
|
||||
\tbuild: build the project using each python (note: remove previous build dir)
|
||||
\ttest: install each wheel in a venv then test it (note: don't build !)
|
||||
\tall: build + test (default)
|
||||
|
||||
EXAMPLES
|
||||
* Using export
|
||||
export PLATFORM=x86_64
|
||||
export PYTHON_VERSION=39
|
||||
$0 build
|
||||
|
||||
* One-liner:
|
||||
PLATFORM=x86_64 PYTHON_VERSION=39 $0 build"
|
||||
}
|
||||
|
||||
function contains_element() {
|
||||
# Look for the presence of an element in an array. Echoes '0' if found,
|
||||
# '1' otherwise.
|
||||
# Arguments:
|
||||
# $1 the element to be searched
|
||||
# $2 the array to search into
|
||||
local e match="$1"
|
||||
shift
|
||||
for e; do
|
||||
[[ "$e" == "$match" ]] && return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
function build_wheel() {
|
||||
assert_defined BUILD_DIR
|
||||
assert_defined VENV_DIR
|
||||
# Build the wheel artifact
|
||||
# Arguments:
|
||||
# $1 the python root directory
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "$0 called with an illegal number of parameters"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create and activate virtualenv
|
||||
# this is needed so protoc can call the correct python executable
|
||||
local -r PYBIN="$1/bin"
|
||||
"${PYBIN}/pip" install virtualenv
|
||||
"${PYBIN}/virtualenv" -p "${PYBIN}/python" "${VENV_DIR}"
|
||||
# shellcheck source=/dev/null
|
||||
source "${VENV_DIR}/bin/activate"
|
||||
pip install -U pip setuptools wheel absl-py # absl-py is needed by make test_python
|
||||
pip install -U mypy mypy-protobuf # need to generate protobuf mypy files
|
||||
|
||||
echo "current dir: $(pwd)"
|
||||
|
||||
if [[ ! -e "CMakeLists.txt" ]] || [[ ! -d "cmake" ]]; then
|
||||
>&2 echo "Can't find project's CMakeLists.txt or cmake"
|
||||
exit 2
|
||||
fi
|
||||
cmake -S. -B"${BUILD_DIR}" \
|
||||
-DCMAKE_BUILD_TYPE=Release \
|
||||
-DBUILD_DEPS=ON -DBUILD_PYTHON=ON -DPython3_ROOT_DIR="$1" \
|
||||
-DBUILD_TESTING=OFF -DBUILD_SAMPLES=OFF -DBUILD_EXAMPLES=OFF #--debug-find
|
||||
cmake --build "${BUILD_DIR}" -v -j4
|
||||
|
||||
# Restore environment
|
||||
deactivate
|
||||
}
|
||||
|
||||
function check_wheel() {
|
||||
assert_defined BUILD_DIR
|
||||
# Check the wheel artifact
|
||||
# Arguments:
|
||||
# $1 the python root directory
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "$0 called with an illegal number of parameters"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check mypy files
|
||||
declare -a MYPY_FILES=(
|
||||
"ortools/algorithms/python/knapsack_solver.pyi"
|
||||
"ortools/algorithms/python/set_cover.pyi"
|
||||
"ortools/constraint_solver/pywrapcp.pyi"
|
||||
"ortools/graph/python/linear_sum_assignment.pyi"
|
||||
"ortools/graph/python/max_flow.pyi"
|
||||
"ortools/graph/python/min_cost_flow.pyi"
|
||||
"ortools/init/python/init.pyi"
|
||||
"ortools/linear_solver/python/model_builder_helper.pyi"
|
||||
"ortools/linear_solver/pywraplp.pyi"
|
||||
"ortools/pdlp/python/pdlp.pyi"
|
||||
"ortools/sat/python/cp_model_helper.pyi"
|
||||
"ortools/scheduling/python/rcpsp.pyi"
|
||||
"ortools/util/python/sorted_interval_list.pyi"
|
||||
)
|
||||
for FILE in "${MYPY_FILES[@]}"; do
|
||||
if [[ ! -f "${BUILD_DIR}/python/${FILE}" ]]; then
|
||||
echo "error: ${FILE} missing in the python project"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
|
||||
# Check all generated wheel packages
|
||||
pushd "${BUILD_DIR}/python/dist"
|
||||
for FILE in *.whl; do
|
||||
# if no files found do nothing
|
||||
[[ -e "$FILE" ]] || continue
|
||||
auditwheel show "$FILE" || true
|
||||
auditwheel -v repair --plat "musllinux_1_2_$PLATFORM" "$FILE" -w "$export_root"
|
||||
#auditwheel -v repair --plat musllinux_1_2_x86_64 "$FILE" -w "$export_root"
|
||||
#auditwheel -v repair --plat musllinux_1_2_aarch64 "$FILE" -w "$export_root"
|
||||
done
|
||||
popd
|
||||
}
|
||||
|
||||
function test_wheel() {
|
||||
assert_defined BUILD_DIR
|
||||
assert_defined TEST_DIR
|
||||
# Test the wheel artifacts
|
||||
# Arguments:
|
||||
# $1 the python root directory
|
||||
if [[ "$#" -ne 1 ]]; then
|
||||
echo "$0 called with an illegal number of parameters"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create and activate virtualenv
|
||||
local -r PYBIN="$1/bin"
|
||||
"${PYBIN}/pip" install virtualenv
|
||||
"${PYBIN}/virtualenv" -p "${PYBIN}/python" "${TEST_DIR}"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${TEST_DIR}/bin/activate"
|
||||
pip install -U pip setuptools wheel
|
||||
|
||||
# Install the wheel artifact
|
||||
#pwd
|
||||
local -r WHEEL_FILE=$(find "${BUILD_DIR}"/python/dist/*.whl | head -1)
|
||||
echo "WHEEL file: ${WHEEL_FILE}"
|
||||
pip install --no-cache-dir "$WHEEL_FILE"
|
||||
pip show ortools
|
||||
|
||||
# Python scripts to be used as tests for the installed wheel. This list of files
|
||||
# has been taken from the 'test_python' make target.
|
||||
declare -a TESTS=(
|
||||
"ortools/algorithms/samples/simple_knapsack_program.py"
|
||||
"ortools/graph/samples/simple_max_flow_program.py"
|
||||
"ortools/graph/samples/simple_min_cost_flow_program.py"
|
||||
"ortools/linear_solver/samples/simple_lp_program.py"
|
||||
"ortools/linear_solver/samples/simple_mip_program.py"
|
||||
"ortools/sat/samples/simple_sat_program.py"
|
||||
"ortools/constraint_solver/samples/tsp.py"
|
||||
"ortools/constraint_solver/samples/vrp.py"
|
||||
"ortools/constraint_solver/samples/cvrptw_break.py"
|
||||
)
|
||||
|
||||
# Run all the specified test scripts using the current environment.
|
||||
local -r ROOT_DIR=$(pwd)
|
||||
pushd "$(mktemp -d)" # ensure we are not importing something from $PWD
|
||||
python --version
|
||||
for TEST in "${TESTS[@]}"; do
|
||||
python "${ROOT_DIR}/${TEST}"
|
||||
done
|
||||
popd
|
||||
|
||||
# Restore environment
|
||||
deactivate
|
||||
}
|
||||
|
||||
function build() {
|
||||
# For each python platform provided by musllinux, build and test artifacts.
|
||||
for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do
|
||||
# shellcheck disable=SC2155
|
||||
PYTAG=$(basename "$PYROOT")
|
||||
echo "Python: $PYTAG"
|
||||
|
||||
# Check for platforms to be skipped
|
||||
if contains_element "$PYTAG" "${SKIPS[@]}"; then
|
||||
>&2 echo "skipping deprecated platform $PYTAG"
|
||||
continue
|
||||
fi
|
||||
|
||||
BUILD_DIR="build_${PYTAG}"
|
||||
VENV_DIR="env_${PYTAG}"
|
||||
build_wheel "$PYROOT"
|
||||
check_wheel "$PYROOT"
|
||||
done
|
||||
}
|
||||
|
||||
function tests() {
|
||||
# For each python platform provided by musllinux, build and test artifacts.
|
||||
for PYROOT in /opt/python/cp"${PYTHON_VERSION}"*-cp"${PYTHON_VERSION}"*; do
|
||||
# shellcheck disable=SC2155
|
||||
PYTAG=$(basename "$PYROOT")
|
||||
echo "Python: $PYTAG"
|
||||
|
||||
# Check for platforms to be skipped
|
||||
if contains_element "$PYTAG" "${SKIPS[@]}"; then
|
||||
>&2 echo "skipping deprecated platform $PYTAG"
|
||||
continue
|
||||
fi
|
||||
|
||||
BUILD_DIR="build_${PYTAG}"
|
||||
TEST_DIR="test_${PYTAG}"
|
||||
test_wheel "$PYROOT"
|
||||
done
|
||||
}
|
||||
|
||||
# Main
|
||||
function main() {
|
||||
case ${1} in
|
||||
-h | --help)
|
||||
usage; exit ;;
|
||||
esac
|
||||
|
||||
assert_defined PLATFORM
|
||||
assert_defined PYTHON_VERSION
|
||||
|
||||
# Setup
|
||||
declare -a SKIPS=( "cp36-cp36m" "cp37-cp37m" )
|
||||
|
||||
case ${1} in
|
||||
build)
|
||||
build ;;
|
||||
test)
|
||||
tests ;;
|
||||
*)
|
||||
build
|
||||
tests ;;
|
||||
esac
|
||||
}
|
||||
|
||||
main "${1:-all}"
|
||||
Reference in New Issue
Block a user