build default configuration moved to dockerfile

2017-09-03 13:12:46 +02:00
parent dc72c2c98e
commit 2e221ffc1d
3 changed files with 87 additions and 82 deletions
--- a/tools/docker/Makefile
+++ b/tools/docker/Makefile
@@ -157,4 +157,4 @@ manylinux1-image:
 	docker build -f manylinux1.Dockerfile -t or-tools-manylinux1-image .

 manylinux1-pypi3: export manylinux1-image
-	docker run -v `pwd`/export:/export or-tools-manylinux1-image:latest /bin/bash -c "/root/build-manylinux1.sh /root /export"
+	docker run -v `pwd`/export:/export or-tools-manylinux1-image:latest /bin/bash -c "/root/build/build-manylinux1.sh /root/src /root/build /export"
--- a/tools/docker/build-manylinux1.sh
+++ b/tools/docker/build-manylinux1.sh
@@ -3,16 +3,26 @@
 # export them to the specified location.
 #
 # Arguments:
+#   $1 the path of the or-tools sources.
+#   $2 the build process root directory. If not specified, a default location is
+#      used.
+#   $3 the artifacts export directory. If not specified, a default location is
+#      used.
 #
-#   $1 the build process root directory. If not specified as argument, the
-#      env var BUILD_ROOT will be used. If not specified at all, a default
-#      location will be used.
-#
-#   $2 the artifacts export directory. If not specified as argument, the env
-#      var EXPORT_ROOT will be used. If not specified at all, a default location
-#      will be used.
+# Environment variables:
+#   SKIP_PLATFORMS manylinux1 platforms (e.g.: cp26-cp26m) to be skipped.
+#   EXPORT_ROOT    if not specified at command line, this value is used as the
+#                  destination path for the wheels export.
+#   BUILD_ROOT     if not specified at command line, this value is used as the
+#                  root path for the build process.
 set -e

+DEFAULT_BUILD_ROOT="$HOME"
+DEFAULT_EXPORT_ROOT="${HOME}/export"
+
+function usage() {
+  echo "Usage: build-manylinux1.sh SRC_ROOT <BUILD_ROOT> <EXPORT_ROOT>"
+}

 function contains_element () {
    # Look for the presence of an element in an array. Echoes '0' if found,
@@ -26,7 +36,6 @@ function contains_element () {
    echo '1' && return
 }

-
 function export_manylinux_wheel {
    # Build all the wheel artifacts assuming that the current 'python' command
    # is the target interpreter. Please note that in order to have or-tools
@@ -49,7 +58,7 @@ function export_manylinux_wheel {
    make -B install_python_modules  # regenerates Makefile.local
    make python
    make test_python
-    make pypi_archive
+    make pypi_archive_dir
    # Build and repair wheels
    cd temp-python*/ortools
    python setup.py bdist_wheel
@@ -57,7 +66,6 @@ function export_manylinux_wheel {
    auditwheel repair ./*.whl -w "$export_root"
 }

-
 function test_installed {
    # Run all the specified test scripts using the current environment.
    # Arguments:
@@ -70,34 +78,35 @@ function test_installed {
    done
 }

-
 ###############################################################################
 # Setup

-if [ -n "$1" ]; then BUILD_ROOT="$1"; fi
-if [ -n "$2" ]; then EXPORT_ROOT="$2"; fi
-
-if [ -z "$BUILD_ROOT" ]
-then
-    echo "\$BUILD_ROOT is not set, using default location: $HOME"
-    BUILD_ROOT="$HOME"
+if [ -z "$1" ]; then
+  (>&2 usage)
+  exit 1
 fi

-if [ -z "$EXPORT_ROOT" ]
-then
-    echo "\$EXPORT_ROOT is not set, using default location: ${HOME}/export"
-    EXPORT_ROOT="${HOME}/export"
+SRC_ROOT="$1";
+if [ -n "$2" ]; then BUILD_ROOT="$2"; fi
+if [ -n "$3" ]; then EXPORT_ROOT="$3"; fi
+
+if [ ! -d "$SRC_ROOT" ]; then
+    (>&2 echo "Can't find or-tools sources at the specified location: $SRC_ROOT")
+    exit 1
 fi

-SRC_ROOT="${BUILD_ROOT}/or-tools"
+if [ -z "$BUILD_ROOT" ]; then
+    (>&2 echo "\$BUILD_ROOT is not set, using default location: $DEFAULT_BUILD_ROOT")
+    BUILD_ROOT="$DEFAULT_BUILD_ROOT"
+fi

-# Platforms to be ignored.
-# The build of Python 2.6.x bindings is known to be broken (and 2.6 itself is
-# deprecated).
-SKIP_PLATFORMS=(
-    cp26-cp26m
-    cp26-cp26mu
-)
+if [ -z "$EXPORT_ROOT" ]; then
+    (>&2 echo "\$EXPORT_ROOT is not set, using default location: $DEFAULT_EXPORT_ROOT")
+    EXPORT_ROOT="$DEFAULT_EXPORT_ROOT"
+fi
+
+# Split SKIP_PLATFORMS string and put values into an array
+read -r -a SKIP <<< "$SKIP_PLATFORMS"

 # Python scripts to be used as tests for the installed wheel. This list of files
 # has been taken from the 'test_python' make target.
@@ -112,12 +121,13 @@ TESTS=(
    "${SRC_ROOT}/examples/tests/test_lp_api.py"
 )

-echo "BUILD_ROOT=${BUILD_ROOT}"
-echo "SRC_ROOT=${SRC_ROOT}"
-echo "EXPORT_ROOT=${EXPORT_ROOT}"
-echo "SKIP_PLATFORMS=(${SKIP_PLATFORMS[*]})"
-echo "TESTS=(${TESTS[*]})"
-
+(
+    >&2 echo "BUILD_ROOT=${BUILD_ROOT}"
+    >&2 echo "SRC_ROOT=${SRC_ROOT}"
+    >&2 echo "EXPORT_ROOT=${EXPORT_ROOT}"
+    >&2 echo "SKIP_PLATFORMS=( ${SKIP[*]} )"
+    >&2 echo "TESTS=( ${TESTS[*]} )"
+)

 ###############################################################################
 # Main
@@ -125,44 +135,22 @@ echo "TESTS=(${TESTS[*]})"
 mkdir -p "${BUILD_ROOT}"
 mkdir -p "${EXPORT_ROOT}"

-# Retrieve or-tools if needed
-if [ ! -d "$SRC_ROOT" ]
-then
-  echo "SRC_ROOT doesn't exist, retrieving source from: https://github.com/google/or-tools"
-  git clone https://github.com/google/or-tools "$SRC_ROOT"
-fi
-
 # Make third_party if needed
-if [ ! -f "${SRC_ROOT}/Makefile.local" ]
-then
-  echo "\${SRC_ROOT}/Makefile.local doesn't exist, building third_party"
+if [ ! -f "${SRC_ROOT}/Makefile.local" ]; then
+  (>&2 echo "\${SRC_ROOT}/Makefile.local doesn't exist, building third_party")
  cd "$SRC_ROOT"
  make third_party
 fi

-# TODO remove all patching, write Makefile targets for manylinux
-
-# Patch makefile, remove offending command (the setup.py-e file doesn't exist)
-cd "$SRC_ROOT"
-sed -i -e "s=-rm \$(PYPI_ARCHIVE_TEMP_DIR)/ortools/setup.py-e==g" makefiles/Makefile.python.mk
-
-# Patch makefile, remove manual libortools.so grafting and RPATH setting. All of
-# this stuff will be carried out by auditwheel, otherwise we would end up with a
-# broken manylinux wheel file
-cd "$SRC_ROOT"
-sed -i -e "s=cp lib/libortools=#=g" makefiles/Makefile.python.mk
-sed -i -e "s=tools/fix_python_libraries_on_linux.sh=#=g" makefiles/Makefile.python.mk
-
 # For each python platform provided by manylinux, build, export and test
 # artifacts.
 for PYROOT in /opt/python/*
 do
    PYTAG=$(basename "$PYROOT")
    # Check for platforms to be skipped
-    SKIP=$(contains_element "$PYTAG" "${SKIP_PLATFORMS[@]}")
-    if [ "$SKIP" -eq '0' ]
-    then
-        echo "skipping deprecated platform $PYTAG"
+    _skip=$(contains_element "$PYTAG" "${SKIP[@]}")
+    if [ "$_skip" -eq '0' ]; then
+        (>&2 echo "skipping deprecated platform $PYTAG")
        continue
    fi
    ###
@@ -172,6 +160,7 @@ do
    PYBIN="${PYROOT}/bin"
    "${PYBIN}/pip" install virtualenv
    "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${BUILD_ROOT}/${PYTAG}"
+    # shellcheck source=/dev/null
    source "${BUILD_ROOT}/${PYTAG}/bin/activate"
    pip install -U pip setuptools wheel six  # six is needed by make test_python
    # Build artifact
@@ -188,6 +177,7 @@ do
    WHEEL_FILE=$(echo "${EXPORT_ROOT}"/*-"${PYTAG}"-*.whl)
    # Create and activate a new virtualenv
    "${PYBIN}/virtualenv" -p "${PYBIN}/python" "${BUILD_ROOT}/${PYTAG}-test"
+    # shellcheck source=/dev/null
    source "${BUILD_ROOT}/${PYTAG}-test/bin/activate"
    pip install -U pip setuptools wheel six
    # Install wheel and run tests
--- a/tools/docker/manylinux1.Dockerfile
+++ b/tools/docker/manylinux1.Dockerfile
@@ -1,20 +1,27 @@
 FROM quay.io/pypa/manylinux1_x86_64:latest

+ENV SRC_ROOT /root/src
+ENV BUILD_ROOT /root/build
+ENV EXPORT_ROOT /export
+ENV SRC_GIT_URL https://github.com/google/or-tools
+ENV SRC_GIT_BRANCH master
+# The build of Python 2.6.x bindings is known to be broken.
+ENV SKIP_PLATFORMS "cp26-cp26m cp26-cp26mu"
+
 RUN yum -y update && yum -y install \
-    yum-utils \
-    redhat-lsb \
-    git \
    autoconf \
-    libtool \
-    zlib-devel \
+    curl \
    gawk \
    gcc-c++ \
-    curl \
-    subversion \
+    git \
+    libtool \
    make \
+    patch \
    pcre-devel \
+    redhat-lsb \
+    subversion \
    which \
-    curl
+    zlib-devel

 # WARNING
 # We cannot use wget to download the needed packages due to a bug that leads
@@ -26,28 +33,36 @@ RUN yum -y update && yum -y install \
 # Note: 'wget --no-check-certificate' is not an option since we are building
 #       distribution binaries.

+RUN mkdir -p "$BUILD_ROOT"
+
 # Update cmake, the system shipped version is too old for or-tools.
-WORKDIR /root
-RUN curl --location-trusted --remote-name https://cmake.org/files/v3.8/cmake-3.8.2.tar.gz
+WORKDIR "$BUILD_ROOT"
+RUN curl --location-trusted --remote-name https://cmake.org/files/v3.8/cmake-3.8.2.tar.gz -o cmake-3.8.2.tar.gz
 RUN tar xzf cmake-3.8.2.tar.gz
-WORKDIR /root/cmake-3.8.2
+WORKDIR cmake-3.8.2
 RUN ./bootstrap --prefix=/usr
 RUN make
 RUN make install

 # Update swig, the system shipped version doesn't support PY3.
-WORKDIR /root
-RUN curl --location-trusted --remote-name https://downloads.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz
+WORKDIR "$BUILD_ROOT"
+RUN curl --location-trusted --remote-name https://downloads.sourceforge.net/project/swig/swig/swig-3.0.12/swig-3.0.12.tar.gz -o swig-3.0.12.tar.gz
 RUN tar xzf swig-3.0.12.tar.gz
-WORKDIR /root/swig-3.0.12
+WORKDIR swig-3.0.12
 RUN ./configure --prefix=/usr
 RUN make
 RUN make install

-RUN git clone https://github.com/google/or-tools /root/or-tools
-WORKDIR /root/or-tools
+RUN git clone -b "$SRC_GIT_BRANCH" --single-branch "$SRC_GIT_URL" "$SRC_ROOT"
+WORKDIR "$SRC_ROOT"
 RUN make third_party

-ADD build-manylinux1.sh /root
+COPY build-manylinux1.sh "$BUILD_ROOT"
+RUN chmod ugo+x "${BUILD_ROOT}/build-manylinux1.sh"

-WORKDIR /root
+# TEMPORARY HACK
+# Patch Makefile.python.mk to add manylinux1 targets,
+# to be removed when changes are integrated into mainstream.
+COPY Makefile.python.mk.patch "$BUILD_ROOT"
+WORKDIR "$SRC_ROOT"
+RUN patch --force -p1 < "${BUILD_ROOT}/Makefile.python.mk.patch"