From 6a172e58e04bedfd4daa4ad271493cbe91fe8261 Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Fri, 25 Nov 2022 17:38:16 +0100 Subject: [PATCH 1/6] tools/release: provide publish_delivery_manylinux* --- ...sh => publish_delivery_manylinux_amd64.sh} | 28 +++- .../publish_delivery_manylinux_arm64.sh | 138 ++++++++++++++++++ 2 files changed, 158 insertions(+), 8 deletions(-) rename tools/release/{publish_delivery_unix.sh => publish_delivery_manylinux_amd64.sh} (77%) create mode 100755 tools/release/publish_delivery_manylinux_arm64.sh diff --git a/tools/release/publish_delivery_unix.sh b/tools/release/publish_delivery_manylinux_amd64.sh similarity index 77% rename from tools/release/publish_delivery_unix.sh rename to tools/release/publish_delivery_manylinux_amd64.sh index 579ce9c6f3..bb694ed92a 100755 --- a/tools/release/publish_delivery_unix.sh +++ b/tools/release/publish_delivery_manylinux_amd64.sh @@ -58,20 +58,27 @@ function publish_delivery() { assert_defined ORTOOLS_SHA1 assert_defined ORTOOLS_TOKEN assert_defined ORTOOLS_DELIVERY + assert_defined DOCKERFILE + assert_defined ORTOOLS_IMG # Clean - docker image rm -f ortools:linux_delivery 2>/dev/null + echo -n "Remove previous docker images..." | tee -a "${ROOT_DIR}/publish.log" + docker image rm -f "${ORTOOLS_IMG}":"publish_${ORTOOLS_DELIVERY}" 2>/dev/null + echo "DONE" | tee -a "${ROOT_DIR}/publish.log" cd "${RELEASE_DIR}" || exit 2 - # Build delivery - docker build --tag ortools/linux_delivery:"publish_${ORTOOLS_DELIVERY}" \ + # Publish delivery + echo -n "Build ${ORTOOLS_IMG}:publish_${ORTOOLS_DELIVERY}..." | tee -a "${ROOT_DIR}/publish.log" + docker buildx build \ + --tag "${ORTOOLS_IMG}":"publish_${ORTOOLS_DELIVERY}" \ --build-arg ORTOOLS_GIT_BRANCH="${ORTOOLS_BRANCH}" \ --build-arg ORTOOLS_GIT_SHA1="${ORTOOLS_SHA1}" \ --build-arg ORTOOLS_TOKEN="${ORTOOLS_TOKEN}" \ --build-arg ORTOOLS_DELIVERY="${ORTOOLS_DELIVERY}" \ --target=publish \ - -f Dockerfile . + -f "${DOCKERFILE}" . + echo "DONE" | tee -a "${ROOT_DIR}/publish.log" } # Java publish @@ -95,18 +102,23 @@ function main() { assert_defined ORTOOLS_TOKEN echo "ORTOOLS_TOKEN: FOUND" | tee publish.log - make print-OR_TOOLS_VERSION | tee -a publish.log local -r ROOT_DIR="$(cd -P -- "$(dirname -- "$0")/../.." && pwd -P)" - echo "ROOT_DIR: '${ROOT_DIR}'" + echo "ROOT_DIR: '${ROOT_DIR}'" | tee -a publish.log local -r RELEASE_DIR="$(cd -P -- "$(dirname -- "$0")" && pwd -P)" - echo "RELEASE_DIR: '${RELEASE_DIR}'" + echo "RELEASE_DIR: '${RELEASE_DIR}'" | tee -a publish.log + + (cd "${ROOT_DIR}" && make print-OR_TOOLS_VERSION | tee -a build.log) local -r ORTOOLS_BRANCH=$(git rev-parse --abbrev-ref HEAD) local -r ORTOOLS_SHA1=$(git rev-parse --verify HEAD) + local -r DOCKERFILE="amd64.Dockerfile" + local -r ORTOOLS_IMG="ortools/manylinux_delivery_amd64" + local -r PLATFORM=$(uname -m) + + mkdir -p "${ROOT_DIR}/export" - mkdir -p export case ${1} in java|python) "publish_$1" diff --git a/tools/release/publish_delivery_manylinux_arm64.sh b/tools/release/publish_delivery_manylinux_arm64.sh new file mode 100755 index 0000000000..653b0b89a4 --- /dev/null +++ b/tools/release/publish_delivery_manylinux_arm64.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +# Copyright 2010-2022 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +function help() { + local -r NAME=$(basename "$0") + local -r BOLD="\e[1m" + local -r RESET="\e[0m" + local -r help=$(cat << EOF +${BOLD}NAME${RESET} +\t$NAME - Publish delivery using an ${BOLD}Ubuntu 18.04 LTS docker image${RESET}. +${BOLD}SYNOPSIS${RESET} +\t$NAME [-h|--help] [java|python] +${BOLD}DESCRIPTION${RESET} +\tPublish Google OR-Tools deliveries. +\tYou ${BOLD}MUST${RESET} define the following variables before running this script: +\t* ORTOOLS_TOKEN: secret use to decrypt key to sign Java package. + +${BOLD}OPTIONS${RESET} +\t-h --help: display this help text +\tjava: publish the Java runtime packages +\tpython: publish all Pyhon packages +\tall: publish everything (default) + +${BOLD}EXAMPLES${RESET} +Using export to define the ${BOLD}ORTOOLS_TOKEN${RESET} env and only publishing the Java packages: +export ORTOOLS_TOKEN=SECRET +$0 java + +note: the 'export ORTOOLS_TOKEN=...' should be placed in your bashrc to avoid any leak +of the secret in your bash history +EOF +) + echo -e "$help" +} + +function assert_defined(){ + if [[ -z "${!1}" ]]; then + >&2 echo "Variable '${1}' must be defined" + exit 1 + fi +} + +function publish_delivery() { + assert_defined ORTOOLS_BRANCH + assert_defined ORTOOLS_SHA1 + assert_defined ORTOOLS_TOKEN + assert_defined ORTOOLS_DELIVERY + assert_defined DOCKERFILE + assert_defined ORTOOLS_IMG + + # Clean + echo -n "Remove previous docker images..." | tee -a "${ROOT_DIR}/publish.log" + docker image rm -f "${ORTOOLS_IMG}":"publish_${ORTOOLS_DELIVERY}" 2>/dev/null + echo "DONE" | tee -a "${ROOT_DIR}/publish.log" + + cd "${RELEASE_DIR}" || exit 2 + + # Publish delivery + echo -n "Build ${ORTOOLS_IMG}:publish_${ORTOOLS_DELIVERY}..." | tee -a "${ROOT_DIR}/publish.log" + docker buildx build \ + --tag "${ORTOOLS_IMG}":"publish_${ORTOOLS_DELIVERY}" \ + --build-arg ORTOOLS_GIT_BRANCH="${ORTOOLS_BRANCH}" \ + --build-arg ORTOOLS_GIT_SHA1="${ORTOOLS_SHA1}" \ + --build-arg ORTOOLS_TOKEN="${ORTOOLS_TOKEN}" \ + --build-arg ORTOOLS_DELIVERY="${ORTOOLS_DELIVERY}" \ + --target=publish \ + -f "${DOCKERFILE}" . + echo "DONE" | tee -a "${ROOT_DIR}/publish.log" +} + +# Java publish +function publish_java() { + local -r ORTOOLS_DELIVERY=java + publish_delivery +} + +# Python publish +function publish_python() { + local -r ORTOOLS_DELIVERY=python + publish_delivery +} + +# Main +function main() { + case ${1} in + -h | --help) + help; exit ;; + esac + + assert_defined ORTOOLS_TOKEN + echo "ORTOOLS_TOKEN: FOUND" | tee publish.log + + local -r ROOT_DIR="$(cd -P -- "$(dirname -- "$0")/../.." && pwd -P)" + echo "ROOT_DIR: '${ROOT_DIR}'" | tee -a publish.log + + local -r RELEASE_DIR="$(cd -P -- "$(dirname -- "$0")" && pwd -P)" + echo "RELEASE_DIR: '${RELEASE_DIR}'" | tee -a publish.log + + (cd "${ROOT_DIR}" && make print-OR_TOOLS_VERSION | tee -a build.log) + + local -r ORTOOLS_BRANCH=$(git rev-parse --abbrev-ref HEAD) + local -r ORTOOLS_SHA1=$(git rev-parse --verify HEAD) + local -r DOCKERFILE="arm64.Dockerfile" + local -r ORTOOLS_IMG="ortools/manylinux_delivery_arm64" + local -r PLATFORM=$(uname -m) + + mkdir -p "${ROOT_DIR}/export" + + case ${1} in + java|python) + "publish_$1" + exit ;; + all) + publish_java + publish_python + exit ;; + *) + >&2 echo "Target '${1}' unknown" + exit 1 + esac + exit 0 +} + +main "${1:-all}" + From c43d540ee3bc47fb07fa22bde66db395f9e06e1d Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Fri, 25 Nov 2022 17:51:48 +0100 Subject: [PATCH 2/6] tools/release: rename test_delivery_unix -> test_delivery_linux todo(mizux): cleanup and make it having the same layout than build/publish --- tools/release/{test_delivery_unix.sh => test_delivery_linux.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/release/{test_delivery_unix.sh => test_delivery_linux.sh} (100%) diff --git a/tools/release/test_delivery_unix.sh b/tools/release/test_delivery_linux.sh similarity index 100% rename from tools/release/test_delivery_unix.sh rename to tools/release/test_delivery_linux.sh From cfec85e2eb01cc1488174ef199f5cd717fa2c651 Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Fri, 25 Nov 2022 17:52:26 +0100 Subject: [PATCH 3/6] tools/release: Add M1 support to test_delivery_macos --- tools/release/test_delivery_macos.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tools/release/test_delivery_macos.sh b/tools/release/test_delivery_macos.sh index 00b35296d7..78dab46836 100755 --- a/tools/release/test_delivery_macos.sh +++ b/tools/release/test_delivery_macos.sh @@ -14,13 +14,20 @@ set -euxo pipefail +local -r PLATFORM=$(uname -m) + # Check all prerequisite # cc command -v cmake | xargs echo "cmake: " | tee test.log command -v make | xargs echo "make: " | tee -a test.log command -v swig | xargs echo "swig: " | tee -a test.log # python -PY=(3.6 3.7 3.8 3.9 3.10 3.11) +if [[ ${PLATFORM} == "arm64" ]]; then + local -r PY=(3.8 3.9 3.10 3.11) +else + local -r PY=(3.7 3.8 3.9 3.10 3.11) +fi + for i in "${PY[@]}"; do command -v "python$i" | xargs echo "python$i: " | tee -a test.log done From 7cfa7bc61038123020c39c2f90bdaa0683a49648 Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Mon, 28 Nov 2022 11:29:17 +0100 Subject: [PATCH 4/6] tools/release: Fix macOS Intel python build --- tools/release/build_delivery_macos.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tools/release/build_delivery_macos.sh b/tools/release/build_delivery_macos.sh index 2cb97a7108..38e513828b 100755 --- a/tools/release/build_delivery_macos.sh +++ b/tools/release/build_delivery_macos.sh @@ -258,13 +258,15 @@ function build_python() { #echo "cmake test_python$i: DONE" | tee -a build.log cp "temp_python$i"/python/dist/*.whl export/ - if [[ ${PLATFORM} == "arm64" ]]; then - pushd export - for i in *universal2.whl; do - mv "$i" "${i%universal2.whl}arm64.whl" - done - popd - fi + pushd export + for i in *_universal2.whl; do + if [[ ${PLATFORM} == "arm64" ]]; then + mv "$i" "${i%_universal2.whl}_arm64.whl" + else + mv "$i" "${i%_universal2.whl}_x86_64.whl" + fi + done + popd done # Reset PATH From 6aa1f4aee6128784e8391abe31d7b84724070d13 Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Fri, 25 Nov 2022 17:52:55 +0100 Subject: [PATCH 5/6] tools/release: Add test_delivery_manylinux_* --- .../release/test_delivery_manylinux_amd64.sh | 56 +++++++++++++++++++ .../release/test_delivery_manylinux_arm64.sh | 56 +++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100755 tools/release/test_delivery_manylinux_amd64.sh create mode 100755 tools/release/test_delivery_manylinux_arm64.sh diff --git a/tools/release/test_delivery_manylinux_amd64.sh b/tools/release/test_delivery_manylinux_amd64.sh new file mode 100755 index 0000000000..df811c8ccb --- /dev/null +++ b/tools/release/test_delivery_manylinux_amd64.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2010-2022 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euxo pipefail + +# Check all prerequisite +# cc +command -v cmake | xargs echo "cmake: " | tee test.log +command -v make | xargs echo "make: " | tee -a test.log +command -v swig | xargs echo "swig: " | tee -a test.log +# python +command -v python3 | xargs echo "python3: " | tee -a test.log + +################## +## PYTHON 3.X ## +################## +echo Cleaning Python... | tee -a test.log +make clean_python +echo Cleaning Python...DONE | tee -a test.log + +echo Rebuild Python3 pypi archive... | tee -a test.log +make package_python UNIX_PYTHON_VER=3 +echo Rebuild Python3 pypi archive...DONE | tee -a test.log + +echo Creating Python3 venv... | tee -a test.log +TEMP_DIR=temp_python3 +VENV_DIR=${TEMP_DIR}/venv +python3 -m pip install --user virtualenv +python3 -m virtualenv ${VENV_DIR} +echo Creating Python3 venv...DONE | tee -a test.log + +echo Installing ortools Python3 venv... | tee -a test.log +${VENV_DIR}/bin/python -m pip install ${TEMP_DIR}/ortools/dist/*.whl +echo Installing ortools Python3 venv...DONE | tee -a test.log + +set +e +echo Testing ortools Python3... | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.linear_solver import pywraplp") 2>&1 | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.constraint_solver import pywrapcp") 2>&1 | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.sat import pywrapsat") 2>&1 | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.graph import pywrapgraph") 2>&1 | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.algorithms import pywrapknapsack_solver") 2>&1 | tee -a test.log +"${VENV_DIR}/bin/python" "cmake/samples/python/sample.py" 2>&1 | tee -a test.log +echo Testing ortools Python3...DONE | tee -a test.log +set -e diff --git a/tools/release/test_delivery_manylinux_arm64.sh b/tools/release/test_delivery_manylinux_arm64.sh new file mode 100755 index 0000000000..df811c8ccb --- /dev/null +++ b/tools/release/test_delivery_manylinux_arm64.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2010-2022 Google LLC +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euxo pipefail + +# Check all prerequisite +# cc +command -v cmake | xargs echo "cmake: " | tee test.log +command -v make | xargs echo "make: " | tee -a test.log +command -v swig | xargs echo "swig: " | tee -a test.log +# python +command -v python3 | xargs echo "python3: " | tee -a test.log + +################## +## PYTHON 3.X ## +################## +echo Cleaning Python... | tee -a test.log +make clean_python +echo Cleaning Python...DONE | tee -a test.log + +echo Rebuild Python3 pypi archive... | tee -a test.log +make package_python UNIX_PYTHON_VER=3 +echo Rebuild Python3 pypi archive...DONE | tee -a test.log + +echo Creating Python3 venv... | tee -a test.log +TEMP_DIR=temp_python3 +VENV_DIR=${TEMP_DIR}/venv +python3 -m pip install --user virtualenv +python3 -m virtualenv ${VENV_DIR} +echo Creating Python3 venv...DONE | tee -a test.log + +echo Installing ortools Python3 venv... | tee -a test.log +${VENV_DIR}/bin/python -m pip install ${TEMP_DIR}/ortools/dist/*.whl +echo Installing ortools Python3 venv...DONE | tee -a test.log + +set +e +echo Testing ortools Python3... | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.linear_solver import pywraplp") 2>&1 | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.constraint_solver import pywrapcp") 2>&1 | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.sat import pywrapsat") 2>&1 | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.graph import pywrapgraph") 2>&1 | tee -a test.log +(cd ${VENV_DIR}/bin && ./python -c "from ortools.algorithms import pywrapknapsack_solver") 2>&1 | tee -a test.log +"${VENV_DIR}/bin/python" "cmake/samples/python/sample.py" 2>&1 | tee -a test.log +echo Testing ortools Python3...DONE | tee -a test.log +set -e From c627d194f00c37685d788278bedd551960479d8f Mon Sep 17 00:00:00 2001 From: Corentin Le Molgat Date: Mon, 28 Nov 2022 18:11:56 +0100 Subject: [PATCH 6/6] tools/release: build_delivery_linux fixup --- tools/release/build_delivery_linux.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/release/build_delivery_linux.sh b/tools/release/build_delivery_linux.sh index 3fd15c3094..736e0534a1 100755 --- a/tools/release/build_delivery_linux.sh +++ b/tools/release/build_delivery_linux.sh @@ -76,6 +76,8 @@ function build_dotnet() { if [[ -x $(command -v openssl11) ]]; then OPENSSL_PRG=openssl11 fi + echo "check ${OPENSSL_PRG}..." + command -v ${OPENSSL_PRG} | xargs echo "openssl: " | tee -a build.log $OPENSSL_PRG aes-256-cbc -iter 42 -pass pass:"$ORTOOLS_TOKEN" \ -in "${RELEASE_DIR}/or-tools.snk.enc" \