From 2e221ffc1d77da25e1dd914978d162e3a7f9bb1a Mon Sep 17 00:00:00 2001 From: Federico Ficarelli Date: Sun, 3 Sep 2017 13:12:46 +0200 Subject: [PATCH] build default configuration moved to dockerfile --- tools/docker/Makefile | 2 +- tools/docker/build-manylinux1.sh | 116 +++++++++++++---------------- tools/docker/manylinux1.Dockerfile | 51 ++++++++----- 3 files changed, 87 insertions(+), 82 deletions(-) diff --git a/tools/docker/Makefile b/tools/docker/Makefile index 3e07051164..2c645bce05 100644 --- a/tools/docker/Makefile +++ b/tools/docker/Makefile @@ -157,4 +157,4 @@ manylinux1-image: docker build -f manylinux1.Dockerfile -t or-tools-manylinux1-image . manylinux1-pypi3: export manylinux1-image - docker run -v `pwd`/export:/export or-tools-manylinux1-image:latest /bin/bash -c "/root/build-manylinux1.sh /root /export" + docker run -v `pwd`/export:/export or-tools-manylinux1-image:latest /bin/bash -c "/root/build/build-manylinux1.sh /root/src /root/build /export" diff --git a/tools/docker/build-manylinux1.sh b/tools/docker/build-manylinux1.sh index 3a29b1a037..ae32de5643 100755 --- a/tools/docker/build-manylinux1.sh +++ b/tools/docker/build-manylinux1.sh @@ -3,16 +3,26 @@ # export them to the specified location. # # Arguments: +# $1 the path of the or-tools sources. +# $2 the build process root directory. If not specified, a default location is +# used. +# $3 the artifacts export directory. If not specified, a default location is +# used. # -# $1 the build process root directory. If not specified as argument, the -# env var BUILD_ROOT will be used. If not specified at all, a default -# location will be used. -# -# $2 the artifacts export directory. If not specified as argument, the env -# var EXPORT_ROOT will be used. If not specified at all, a default location -# will be used. +# Environment variables: +# SKIP_PLATFORMS manylinux1 platforms (e.g.: cp26-cp26m) to be skipped. +# EXPORT_ROOT if not specified at command line, this value is used as the +# destination path for the wheels export. +# BUILD_ROOT if not specified at command line, this value is used as the +# root path for the build process. set -e +DEFAULT_BUILD_ROOT="$HOME" +DEFAULT_EXPORT_ROOT="${HOME}/export" + +function usage() { + echo "Usage: build-manylinux1.sh SRC_ROOT " +} function contains_element () { # Look for the presence of an element in an array. Echoes '0' if found, @@ -26,7 +36,6 @@ function contains_element () { echo '1' && return } - function export_manylinux_wheel { # Build all the wheel artifacts assuming that the current 'python' command # is the target interpreter. Please note that in order to have or-tools @@ -49,7 +58,7 @@ function export_manylinux_wheel { make -B install_python_modules # regenerates Makefile.local make python make test_python - make pypi_archive + make pypi_archive_dir # Build and repair wheels cd temp-python*/ortools python setup.py bdist_wheel @@ -57,7 +66,6 @@ function export_manylinux_wheel { auditwheel repair ./*.whl -w "$export_root" } - function test_installed { # Run all the specified test scripts using the current environment. # Arguments: @@ -70,34 +78,35 @@ function test_installed { done } - ############################################################################### # Setup -if [ -n "$1" ]; then BUILD_ROOT="$1"; fi -if [ -n "$2" ]; then EXPORT_ROOT="$2"; fi - -if [ -z "$BUILD_ROOT" ] -then - echo "\$BUILD_ROOT is not set, using default location: $HOME" - BUILD_ROOT="$HOME" +if [ -z "$1" ]; then + (>&2 usage) + exit 1 fi -if [ -z "$EXPORT_ROOT" ] -then - echo "\$EXPORT_ROOT is not set, using default location: ${HOME}/export" - EXPORT_ROOT="${HOME}/export" +SRC_ROOT="$1"; +if [ -n "$2" ]; then BUILD_ROOT="$2"; fi +if [ -n "$3" ]; then EXPORT_ROOT="$3"; fi + +if [ ! -d "$SRC_ROOT" ]; then + (>&2 echo "Can't find or-tools sources at the specified location: $SRC_ROOT") + exit 1 fi -SRC_ROOT="${BUILD_ROOT}/or-tools" +if [ -z "$BUILD_ROOT" ]; then + (>&2 echo "\$BUILD_ROOT is not set, using default location: $DEFAULT_BUILD_ROOT") + BUILD_ROOT="$DEFAULT_BUILD_ROOT" +fi -# Platforms to be ignored. -# The build of Python 2.6.x bindings is known to be broken (and 2.6 itself is -# deprecated). -SKIP_PLATFORMS=( - cp26-cp26m - cp26-cp26mu -) +if [ -z "$EXPORT_ROOT" ]; then + (>&2 echo "\$EXPORT_ROOT is not set, using default location: $DEFAULT_EXPORT_ROOT") + EXPORT_ROOT="$DEFAULT_EXPORT_ROOT" +fi + +# Split SKIP_PLATFORMS string and put values into an array +read -r -a SKIP <<< "$SKIP_PLATFORMS" # Python scripts to be used as tests for the installed wheel. This list of files # has been taken from the 'test_python' make target. @@ -112,12 +121,13 @@ TESTS=( "${SRC_ROOT}/examples/tests/test_lp_api.py" ) -echo "BUILD_ROOT=${BUILD_ROOT}" -echo "SRC_ROOT=${SRC_ROOT}" -echo "EXPORT_ROOT=${EXPORT_ROOT}" -echo "SKIP_PLATFORMS=(${SKIP_PLATFORMS[*]})" -echo "TESTS=(${TESTS[*]})" - +( + >&2 echo "BUILD_ROOT=${BUILD_ROOT}" + >&2 echo "SRC_ROOT=${SRC_ROOT}" + >&2 echo "EXPORT_ROOT=${EXPORT_ROOT}" + >&2 echo "SKIP_PLATFORMS=( ${SKIP[*]} )" + >&2 echo "TESTS=( ${TESTS[*]} )" +) ############################################################################### # Main @@ -125,44 +135,22 @@ echo "TESTS=(${TESTS[*]})" mkdir -p "${BUILD_ROOT}" mkdir -p "${EXPORT_ROOT}" -# Retrieve or-tools if needed -if [ ! -d "$SRC_ROOT" ] -then - echo "SRC_ROOT doesn't exist, retrieving source from: https://github.com/google/or-tools" - git clone https://github.com/google/or-tools "$SRC_ROOT" -fi - # Make third_party if needed -if [ ! -f "${SRC_ROOT}/Makefile.local" ] -then - echo "\${SRC_ROOT}/Makefile.local doesn't exist, building third_party" +if [ ! -f "${SRC_ROOT}/Makefile.local" ]; then + (>&2 echo "\${SRC_ROOT}/Makefile.local doesn't exist, building third_party") cd "$SRC_ROOT" make third_party fi -# TODO remove all patching, write Makefile targets for manylinux - -# Patch makefile, remove offending command (the setup.py-e file doesn't exist) -cd "$SRC_ROOT" -sed -i -e "s=-rm \$(PYPI_ARCHIVE_TEMP_DIR)/ortools/setup.py-e==g" makefiles/Makefile.python.mk - -# Patch makefile, remove manual libortools.so grafting and RPATH setting. All of -# this stuff will be carried out by auditwheel, otherwise we would end up with a -# broken manylinux wheel file -cd "$SRC_ROOT" -sed -i -e "s=cp lib/libortools=#=g" makefiles/Makefile.python.mk -sed -i -e "s=tools/fix_python_libraries_on_linux.sh=#=g" makefiles/Makefile.python.mk - # For each python platform provided by manylinux, build, export and test # artifacts. for PYROOT in /opt/python/* do PYTAG=$(basename "$PYROOT") # Check for platforms to be skipped - SKIP=$(contains_element "$PYTAG" "${SKIP_PLATFORMS[@]}") - if [ "$SKIP" -eq '0' ] - then - echo "skipping deprecated platform $PYTAG" + _skip=$(contains_element "$PYTAG" "${SKIP[@]}") + if [ "$_skip" -eq '0' ]; then + (>&2 echo "skipping deprecated platform $PYTAG") continue fi ### @@ -172,6 +160,7 @@ do PYBIN="${PYROOT}/bin" "${PYBIN}/pip" install virtualenv "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${BUILD_ROOT}/${PYTAG}" + # shellcheck source=/dev/null source "${BUILD_ROOT}/${PYTAG}/bin/activate" pip install -U pip setuptools wheel six # six is needed by make test_python # Build artifact @@ -188,6 +177,7 @@ do WHEEL_FILE=$(echo "${EXPORT_ROOT}"/*-"${PYTAG}"-*.whl) # Create and activate a new virtualenv "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${BUILD_ROOT}/${PYTAG}-test" + # shellcheck source=/dev/null source "${BUILD_ROOT}/${PYTAG}-test/bin/activate" pip install -U pip setuptools wheel six # Install wheel and run tests diff --git a/tools/docker/manylinux1.Dockerfile b/tools/docker/manylinux1.Dockerfile index 99db4d29ec..8ef2164030 100644 --- a/tools/docker/manylinux1.Dockerfile +++ b/tools/docker/manylinux1.Dockerfile @@ -1,20 +1,27 @@ FROM quay.io/pypa/manylinux1_x86_64:latest +ENV SRC_ROOT /root/src +ENV BUILD_ROOT /root/build +ENV EXPORT_ROOT /export +ENV SRC_GIT_URL https://github.com/google/or-tools +ENV SRC_GIT_BRANCH master +# The build of Python 2.6.x bindings is known to be broken. +ENV SKIP_PLATFORMS "cp26-cp26m cp26-cp26mu" + RUN yum -y update && yum -y install \ - yum-utils \ - redhat-lsb \ - git \ autoconf \ - libtool \ - zlib-devel \ + curl \ gawk \ gcc-c++ \ - curl \ - subversion \ + git \ + libtool \ make \ + patch \ pcre-devel \ + redhat-lsb \ + subversion \ which \ - curl + zlib-devel # WARNING # We cannot use wget to download the needed packages due to a bug that leads @@ -26,28 +33,36 @@ RUN yum -y update && yum -y install \ # Note: 'wget --no-check-certificate' is not an option since we are building # distribution binaries. +RUN mkdir -p "$BUILD_ROOT" + # Update cmake, the system shipped version is too old for or-tools. -WORKDIR /root -RUN curl --location-trusted --remote-name https://cmake.org/files/v3.8/cmake-3.8.2.tar.gz +WORKDIR "$BUILD_ROOT" +RUN curl --location-trusted --remote-name https://cmake.org/files/v3.8/cmake-3.8.2.tar.gz -o cmake-3.8.2.tar.gz RUN tar xzf cmake-3.8.2.tar.gz -WORKDIR /root/cmake-3.8.2 +WORKDIR cmake-3.8.2 RUN ./bootstrap --prefix=/usr RUN make RUN make install # Update swig, the system shipped version doesn't support PY3. -WORKDIR /root -RUN curl --location-trusted --remote-name https://downloads.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz +WORKDIR "$BUILD_ROOT" +RUN curl --location-trusted --remote-name https://downloads.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz -o swig-3.0.12.tar.gz RUN tar xzf swig-3.0.12.tar.gz -WORKDIR /root/swig-3.0.12 +WORKDIR swig-3.0.12 RUN ./configure --prefix=/usr RUN make RUN make install -RUN git clone https://github.com/google/or-tools /root/or-tools -WORKDIR /root/or-tools +RUN git clone -b "$SRC_GIT_BRANCH" --single-branch "$SRC_GIT_URL" "$SRC_ROOT" +WORKDIR "$SRC_ROOT" RUN make third_party -ADD build-manylinux1.sh /root +COPY build-manylinux1.sh "$BUILD_ROOT" +RUN chmod ugo+x "${BUILD_ROOT}/build-manylinux1.sh" -WORKDIR /root +# TEMPORARY HACK +# Patch Makefile.python.mk to add manylinux1 targets, +# to be removed when changes are integrated into mainstream. +COPY Makefile.python.mk.patch "$BUILD_ROOT" +WORKDIR "$SRC_ROOT" +RUN patch --force -p1 < "${BUILD_ROOT}/Makefile.python.mk.patch" \ No newline at end of file