From c2481c9bad3d81341209d2ecad2ffb51a41cdcff Mon Sep 17 00:00:00 2001
From: Corentin Le Molgat <corentinl@google.com>
Date: Mon, 28 Feb 2022 15:12:37 +0100
Subject: [PATCH] tools/release: Fix functions (#3150)

* migrate python jobs to cmake
* fix build_dotnet
* fix build_java
---
 tools/release/build_delivery_linux.sh |  58 ++++++-----
 tools/release/build_delivery_mac.sh   |  64 +++++++-----
 tools/release/build_delivery_win.cmd  | 144 ++++++++++++++------------
 3 files changed, 145 insertions(+), 121 deletions(-)

diff --git a/tools/release/build_delivery_linux.sh b/tools/release/build_delivery_linux.sh
index d6da3ac274..b87623beae 100755
--- a/tools/release/build_delivery_linux.sh
+++ b/tools/release/build_delivery_linux.sh
@@ -77,9 +77,10 @@ function build_dotnet() {
     echo "build .Net up to date!"
     return 0
   fi
-  build_cxx
 
+  command -v swig
   command -v swig | xargs echo "swig: " | tee -a build.log
+  command -v dotnet
   command -v dotnet | xargs echo "dotnet: " | tee -a build.log
 
   # Install .Net SNK
@@ -96,16 +97,19 @@ function build_dotnet() {
   echo "DONE" | tee -a build.log
 
   # Clean dotnet
+  echo -n "Clean .Net..." | tee -a build.log
   cd "${ROOT_DIR}" || exit 2
-  make clean_dotnet
+  rm -rf "${ROOT_DIR}/temp_dotnet"
+  echo "DONE" | tee -a build.log
 
   echo -n "Build .Net..." | tee -a build.log
-  make dotnet -l 4 UNIX_PYTHON_VER=3
+  cmake -S. -Btemp_dotnet -DBUILD_DOTNET=ON
+  cmake --build temp_dotnet -j8 -v
   echo "DONE" | tee -a build.log
-  #make test_dotnet -l 4 UNIX_PYTHON_VER=3
-  #echo "make test_dotnet: DONE" | tee -a build.log
+  #cmake --build temp_dotnet --target test
+  #echo "cmake test: DONE" | tee -a build.log
 
-  cp temp_dotnet/packages/*nupkg export/
+  cp temp_dotnet/dotnet/packages/*nupkg export/
   echo "${ORTOOLS_BRANCH} ${ORTOOLS_SHA1}" > "${ROOT_DIR}/export/dotnet_build"
 }
 
@@ -115,8 +119,8 @@ function build_java() {
     echo "build Java up to date!" | tee -a build.log
     return 0
   fi
-  build_cxx
 
+  command -v swig
   command -v swig | xargs echo "swig: " | tee -a build.log
   # maven require JAVA_HOME
   if [[ -z "${JAVA_HOME}" ]]; then
@@ -151,17 +155,20 @@ function build_java() {
   echo "DONE" | tee -a build.log
 
   # Clean java
+  echo -n "Clean Java..." | tee -a build.log
   cd "${ROOT_DIR}" || exit 2
-  make clean_java
+  rm -rf "${ROOT_DIR}/temp_java"
+  echo "DONE" | tee -a build.log
 
   echo -n "Build Java..." | tee -a build.log
-  make java -l 4 UNIX_PYTHON_VER=3
+  cmake -S. -Btemp_java -DBUILD_JAVA=ON -DSKIP_GPG=OFF
+  cmake --build temp_java -j8 -v
   echo "DONE" | tee -a build.log
-  #make test_java -l 4 UNIX_PYTHON_VER=3
-  #echo "make test_java: DONE" | tee -a build.log
+  #cmake --build temp_java --target test
+  #echo "cmake test: DONE" | tee -a build.log
 
-  cp temp_java/ortools-linux-x86-64/target/*.jar* export/
-  cp temp_java/ortools-java/target/*.jar* export/
+  cp temp_java/java/ortools-linux-x86-64/target/*.jar* export/
+  cp temp_java/java/ortools-java/target/*.jar* export/
   echo "${ORTOOLS_BRANCH} ${ORTOOLS_SHA1}" > "${ROOT_DIR}/export/java_build"
 }
 
@@ -241,29 +248,30 @@ function build_examples() {
 # Python 3
 # todo(mizux) Use `make --directory tools/docker python` instead
 function build_python() {
-  build_cxx
+  if echo "${ORTOOLS_BRANCH} ${ORTOOLS_SHA1}" | cmp --silent "${ROOT_DIR}/export/python_build" -; then
+    echo "build python up to date!" | tee -a build.log
+    return 0
+  fi
 
+  command -v swig
   command -v swig | xargs echo "swig: " | tee -a build.log
   command -v python3 | xargs echo "python3: " | tee -a build.log
   command -v protoc-gen-mypy | xargs echo "protoc-gen-mypy: " | tee -a build.log
+  protoc-gen-mypy --version | xargs echo "protoc-gen-mypy version: " | tee -a build.log
+  protoc-gen-mypy --version | grep "3\.2\.0"
 
   echo -n "Cleaning Python 3..." | tee -a build.log
-  make clean_python UNIX_PYTHON_VER=3
+  rm -rf temp_python
   echo "DONE" | tee -a build.log
 
   echo -n "Build Python 3..." | tee -a build.log
-  make python -l 4 UNIX_PYTHON_VER=3
-  echo "DONE" | tee -a build.log
-  #make test_python UNIX_PYTHON_VER=3
-  #echo "make test_python3: DONE" | tee -a build.log
-  echo -n "Build Python 3 wheel archive..." | tee -a build.log
-  make package_python UNIX_PYTHON_VER=3
-  echo "DONE" | tee -a build.log
-  echo -n "Test Python 3 wheel archive..." | tee -a build.log
-  make test_package_python UNIX_PYTHON_VER=3
+  cmake -S . -B temp_python -DBUILD_PYTHON=ON
+  cmake --build temp_python build -j8
   echo "DONE" | tee -a build.log
+  #cmake --build test_python --target test
+  #echo "cmake test_python: DONE" | tee -a build.log
 
-  cp temp_python3/ortools/dist/*.whl export/
+  cp temp_python/python/dist/*.whl export/
   echo "${ORTOOLS_BRANCH} ${ORTOOLS_SHA1}" > "${ROOT_DIR}/export/python_build"
 }
 
diff --git a/tools/release/build_delivery_mac.sh b/tools/release/build_delivery_mac.sh
index 1771d2977f..382963050d 100755
--- a/tools/release/build_delivery_mac.sh
+++ b/tools/release/build_delivery_mac.sh
@@ -81,7 +81,6 @@ function build_dotnet() {
     echo "build .Net up to date!" | tee -a build.log
     return 0
   fi
-  build_cxx
 
   command -v swig
   command -v swig | xargs echo "swig: " | tee -a build.log
@@ -97,16 +96,19 @@ function build_dotnet() {
   echo "DONE" | tee -a build.log
 
   # Clean dotnet
+  echo -n "Clean .Net..." | tee -a build.log
   cd "${ROOT_DIR}" || exit 2
-  make clean_dotnet
+  rm -rf "${ROOT_DIR}/temp_dotnet"
+  echo "DONE" | tee -a build.log
 
   echo -n "Build .Net..." | tee -a build.log
-  make dotnet -l 4 UNIX_PYTHON_VER=3.9
+  cmake -S. -Btemp_dotnet -DBUILD_DOTNET=ON
+  cmake --build temp_dotnet -j8 -v
   echo "DONE" | tee -a build.log
-  #make test_dotnet -l 4 UNIX_PYTHON_VER=3.9
-  #echo "make test_dotnet: DONE" | tee -a build.log
+  #cmake --build build --target test
+  #echo "cmake test: DONE" | tee -a build.log
 
-  cp temp_dotnet/packages/*nupkg export/
+  cp temp_dotnet/dotnet/packages/*nupkg export/
   echo "${ORTOOLS_BRANCH} ${ORTOOLS_SHA1}" > "${ROOT_DIR}/export/dotnet_build"
 }
 
@@ -116,7 +118,6 @@ function build_java() {
     echo "build Java up to date!" | tee -a build.log
     return 0
   fi
-  build_cxx
 
   command -v swig
   command -v swig | xargs echo "swig: " | tee -a build.log
@@ -159,17 +160,20 @@ function build_java() {
   echo "DONE" | tee -a build.log
 
   # Clean java
+  echo -n "Clean Java..." | tee -a build.log
   cd "${ROOT_DIR}" || exit 2
-  make clean_java
+  rm -rf "${ROOT_DIR}/temp_java"
+  echo "DONE" | tee -a build.log
 
   echo -n "Build Java..." | tee -a build.log
-  make java -l 4 UNIX_PYTHON_VER=3.9
+  cmake -S. -Btemp_java -DBUILD_JAVA=ON -DSKIP_GPG=OFF
+  cmake --build temp_java -j8 -v
   echo "DONE" | tee -a build.log
-  #make test_java -l 4 UNIX_PYTHON_VER=3.9
-  #echo "make test_java: DONE" | tee -a build.log
+  #cmake --build temp_java --target test
+  #echo "cmake test: DONE" | tee -a build.log
 
-  cp temp_java/ortools-darwin-x86-64/target/*.jar* export/
-  cp temp_java/ortools-java/target/*.jar* export/
+  cp temp_java/java/ortools-darwin-x86-64/target/*.jar* export/
+  cp temp_java/java/ortools-java/target/*.jar* export/
   echo "${ORTOOLS_BRANCH} ${ORTOOLS_SHA1}" > "${ROOT_DIR}/export/java_build"
 }
 
@@ -249,7 +253,10 @@ function build_examples() {
 # Python 3
 # todo(mizux) Use `make --directory tools/docker python` instead
 function build_python() {
-  build_cxx
+  if echo "${ORTOOLS_BRANCH} ${ORTOOLS_SHA1}" | cmp --silent "${ROOT_DIR}/export/python_build" -; then
+    echo "build python up to date!" | tee -a build.log
+    return 0
+  fi
 
   command -v swig
   command -v swig | xargs echo "swig: " | tee -a build.log
@@ -258,28 +265,31 @@ function build_python() {
     command -v "python$i"
     command -v "python$i" | xargs echo "python$i: " | tee -a build.log
     "python$i" -c "import distutils.util as u; print(u.get_platform())" | tee -a build.log
-    "python$i" -m pip install --user wheel absl-py mypy-protobuf
+    "python$i" -m pip install --upgrade --user wheel absl-py mypy-protobuf
   done
   command -v protoc-gen-mypy | xargs echo "protoc-gen-mypy: " | tee -a build.log
+  protoc-gen-mypy --version | xargs echo "protoc-gen-mypy version: " | tee -a build.log
+  protoc-gen-mypy --version | grep "3\.2\.0"
 
   for i in "${PY[@]}"; do
-    echo -n "Cleaning Python 3..." | tee -a build.log
-    make clean_python UNIX_PYTHON_VER="$i"
+    echo -n "Cleaning Python $i..." | tee -a build.log
+    rm -rf "temp_python$i"
     echo "DONE" | tee -a build.log
 
+    PY_PATH="/Library/Frameworks/Python.framework/Versions/$i"
+    if [ ! -d "$PY_PATH" ]; then
+      echo "Error: Python $i is not found (${PY_PATH})." | tee -a build.log
+      exit 1
+    fi
+
     echo -n "Build Python $i..." | tee -a build.log
-    make python -l 4 UNIX_PYTHON_VER="$i"
-    echo "DONE" | tee -a build.log
-    #make test_python UNIX_PYTHON_VER=$i
-    #echo "make test_python$i: DONE" | tee -a build.log
-    echo -n "Build Python $i wheel archive..." | tee -a build.log
-    make package_python UNIX_PYTHON_VER="$i"
-    echo "DONE" | tee -a build.log
-    echo -n "Test Python $i wheel archive..." | tee -a build.log
-    make test_package_python UNIX_PYTHON_VER="$i"
+    cmake -S. -B"temp_python$i" -DBUILD_PYTHON=ON -DPython3_ROOT_DIR="$PY_PATH"
+    cmake --build "temp_python$i" -j8 -v
     echo "DONE" | tee -a build.log
+    #cmake --build temp_python$i --target test
+    #echo "cmake test_python$i: DONE" | tee -a build.log
 
-    cp "temp_python$i"/ortools/dist/*.whl export/
+    cp "temp_python$i"/python/dist/*.whl export/
   done
   echo "${ORTOOLS_BRANCH} ${ORTOOLS_SHA1}" > "${ROOT_DIR}/export/python_build"
 }
diff --git a/tools/release/build_delivery_win.cmd b/tools/release/build_delivery_win.cmd
index fa9afdc605..488d9e7e51 100644
--- a/tools/release/build_delivery_win.cmd
+++ b/tools/release/build_delivery_win.cmd
@@ -35,28 +35,22 @@ echo SHA1: %SHA1% | tee.exe -a build.log
 
 md export
 
-if "%1"=="java" (
-call :BUILD_CXX
-call :BUILD_JAVA
-exit /B %ERRORLEVEL%
-)
-
 if "%1"=="dotnet" (
-call :BUILD_CXX
 call :BUILD_DOTNET
 exit /B %ERRORLEVEL%
 )
 
+if "%1"=="java" (
+call :BUILD_JAVA
+exit /B %ERRORLEVEL%
+)
+
 if "%1"=="python" (
-call :BUILD_CXX
 call :BUILD_PYTHON
 exit /B %ERRORLEVEL%
 )
 
 if "%1"=="archive" (
-call :BUILD_CXX
-call :BUILD_JAVA
-call :BUILD_DOTNET
 call :BUILD_ARCHIVE
 exit /B %ERRORLEVEL%
 )
@@ -67,9 +61,8 @@ exit /B %ERRORLEVEL%
 )
 
 if "%1"=="all" (
-call :BUILD_CXX
-call :BUILD_JAVA
 call :BUILD_DOTNET
+call :BUILD_JAVA
 call :BUILD_ARCHIVE
 call :BUILD_EXAMPLES
 call :BUILD_PYTHON
@@ -80,6 +73,11 @@ if "%1"=="reset" (
 echo clean everything... | tee.exe -a build.log
 make.exe clean || exit 1
 make.exe clean_third_party || exit 1
+rm.exe -rf temp_dotnet
+rm.exe -rf temp_java
+for %%v in (6 7 8 9 10) do (
+  rm.exe -rf temp_python3%%v
+)
 rm.exe -rf export
 del or-tools.snk
 for %%i in (*.zip) DO del %%i
@@ -153,6 +151,44 @@ echo %BRANCH% %SHA1%>build_cxx.log
 exit /B 0
 
 
+:BUILD_DOTNET
+title Build .Net
+set HASH=
+for /F "tokens=* delims=" %%x in (build_dotnet.log) do (set HASH=%%x)
+if "%HASH%"=="%BRANCH% %SHA1%" (
+echo .Net build seems up to date, skipping
+exit /B 0
+)
+
+REM Check .Net
+which.exe dotnet || exit 1
+which.exe dotnet | tee.exe -a build.log
+
+REM Install .Net snk
+echo Install .Net snk | tee.exe -a build.log
+openssl aes-256-cbc -iter 42 -pass pass:%ORTOOLS_TOKEN% -in tools\release\or-tools.snk.enc -out or-tools.snk -d
+set DOTNET_SNK=or-tools.snk
+
+echo Cleaning .Net... | tee.exe -a build.log
+rm.exe -rf temp_dotnet
+echo DONE | tee.exe -a build.log
+
+echo Build dotnet: ... | tee.exe -a build.log
+cmake -S. -Btemp_dotnet -DBUILD_DOTNET=ON
+cmake --build temp_dotnet --config Release -j8 -v
+echo DONE | tee.exe -a build.log
+REM make.exe test_dotnet WINDOWS_PATH_TO_PYTHON=c:\python39-64 || exit 1
+REM echo make test_dotnet: DONE | tee.exe -a build.log
+
+for %%i in (temp_dotnet\dotnet\packages\*nupkg*) do (
+  echo Copy %%i to export... | tee.exe -a build.log
+  copy %%i export\.
+  echo Copy %%i to export...DONE | tee.exe -a build.log
+)
+echo %BRANCH% %SHA1%>build_dotnet.log
+exit /B 0
+
+
 :BUILD_JAVA
 title Build Java
 set HASH=
@@ -183,18 +219,23 @@ mkdir -p %userprofile%/.m2
 openssl aes-256-cbc -iter 42 -pass pass:%ORTOOLS_TOKEN% -in tools\release\settings.xml.enc -out %userprofile%/.m2/settings.xml -d
 echo Install Java GPG: DONE | tee -a build.log
 
-echo make java: ... | tee.exe -a build.log
-make.exe java WINDOWS_PATH_TO_PYTHON=c:\python39-64 || exit 1
-echo make java: DONE | tee.exe -a build.log
-REM make.exe test_java WINDOWS_PATH_TO_PYTHON=c:\python39-64 || exit 1
-REM echo make test_java: DONE | tee.exe -a build.log
+echo Cleaning Java... | tee.exe -a build.log
+rm.exe -rf temp_java
+echo DONE | tee.exe -a build.log
 
-for %%i in (temp_java\ortools-win32-x86-64\target\*.jar*) do (
+echo Build java: ... | tee.exe -a build.log
+cmake -S. -Btemp_java -DBUILD_JAVA=ON -DSKIP_GPG=OFF
+cmake --build temp_java --config Release -j8 -v
+echo DONE | tee.exe -a build.log
+REM cmake --build temp_java --config Release --target RUN_TEST || exit 1
+REM echo cmake test_java: DONE | tee.exe -a build.log
+
+for %%i in (temp_java\java\ortools-win32-x86-64\target\*.jar*) do (
   echo Copy %%i to export... | tee.exe -a build.log
   copy %%i export\.
   echo Copy %%i to export...DONE | tee.exe -a build.log
 )
-for %%i in (temp_java\ortools-java\target\*.jar*) do (
+for %%i in (temp_java\java\ortools-java\target\*.jar*) do (
   echo Copy %%i to export... | tee.exe -a build.log
   copy %%i export\.
   echo Copy %%i to export...DONE | tee.exe -a build.log
@@ -203,39 +244,6 @@ echo %BRANCH% %SHA1%>build_java.log
 exit /B 0
 
 
-:BUILD_DOTNET
-title Build .Net
-set HASH=
-for /F "tokens=* delims=" %%x in (build_dotnet.log) do (set HASH=%%x)
-if "%HASH%"=="%BRANCH% %SHA1%" (
-echo .Net build seems up to date, skipping
-exit /B 0
-)
-
-REM Check .Net
-which.exe dotnet || exit 1
-which.exe dotnet | tee.exe -a build.log
-
-REM Install .Net snk
-echo Install .Net snk | tee.exe -a build.log
-openssl aes-256-cbc -iter 42 -pass pass:%ORTOOLS_TOKEN% -in tools\release\or-tools.snk.enc -out or-tools.snk -d
-set DOTNET_SNK=or-tools.snk
-
-echo make dotnet: ... | tee.exe -a build.log
-make.exe dotnet WINDOWS_PATH_TO_PYTHON=c:\python39-64 || exit 1
-echo make dotnet: DONE | tee.exe -a build.log
-REM make.exe test_dotnet WINDOWS_PATH_TO_PYTHON=c:\python39-64 || exit 1
-REM echo make test_dotnet: DONE | tee.exe -a build.log
-
-for %%i in (packages\*nupkg*) do (
-  echo Copy %%i to export... | tee.exe -a build.log
-  copy %%i export\.
-  echo Copy %%i to export...DONE | tee.exe -a build.log
-)
-echo %BRANCH% %SHA1%>build_dotnet.log
-exit /B 0
-
-
 REM Create Archive
 :BUILD_ARCHIVE
 title Build archives
@@ -295,7 +303,7 @@ echo %BRANCH% %SHA1%>build_examples.log
 exit /B 0
 
 
-REM PYTHON 3.6,3.7,3.8,3.9,3.10
+REM PYTHON 3.6, 3.7, 3.8, 3.9, 3.10
 :BUILD_PYTHON
 title Build Python
 set HASH=
@@ -310,26 +318,24 @@ for %%v in (6 7 8 9 10) do (
   REM Check Python
   which.exe C:\python3%%v-64\python.exe || exit 1
   echo C:\python3%%v-64\python.exe: FOUND | tee.exe -a build.log
-  C:\python3%%v-64\python.exe -m pip install --user absl-py mypy-protobuf
+  C:\python3%%v-64\python.exe -m pip install --upgrade --user absl-py mypy-protobuf
   set PATH+=;%userprofile%\appdata\roaming\python\python3%%v\Scripts"
   set PATH+=;C:\python3%%v-64\Scripts"
 
-  echo Cleaning Python... | tee.exe -a build.log
- make.exe clean_python WINDOWS_PATH_TO_PYTHON=c:\python3%%v-64
-  echo Cleaning Python...DONE | tee.exe -a build.log
+  echo Cleaning Python 3.%%v... | tee.exe -a build.log
+  rm.exe -rf temp_python3%%v
+  echo DONE | tee.exe -a build.log
 
-  REM make.exe python WINDOWS_PATH_TO_PYTHON=c:\python3%%v-64 || exit 1
-  REM echo make python3.%%v: DONE | tee.exe -a build.log
-  REM make.exe test_python WINDOWS_PATH_TO_PYTHON=c:\python3%%v-64 || exit 1
-  REM echo make test_python3.%%v: DONE | tee.exe -a build.log
-  echo Rebuild Python3.%%v pypi archive... | tee.exe -a build.log
-  make.exe package_python WINDOWS_PATH_TO_PYTHON=c:\python3%%v-64 || exit 1
-  echo Rebuild Python3.%%v pypi archive...DONE | tee.exe -a build.log
-  echo Test Python3.%%v pypi archive... | tee.exe -a build.log
-  make.exe test_package_python WINDOWS_PATH_TO_PYTHON=c:\python3%%v-64 || exit 1
-  echo Test Python3.%%v pypi archive...DONE | tee.exe -a build.log
+  echo Build Python 3.%%v... | tee.exe -a build.log
+  cmake -S. -Btemp_python3%%v -DBUILD_PYTHON=ON -DPython3_ROOT_DIR=C:\python3%%v-64
+  cmake --build temp_python3%%v --config Release -j8 -v
+  echo DONE | tee.exe -a build.log
 
-  for %%i in (temp_python3%%v\ortools\dist\*.whl) do (
+  ::echo Test Python 3.%%v pypi archive... | tee.exe -a build.log
+  ::cmake --build temp_python3%%v --config Release --taget RUN_TEST || exit 1
+  ::echo DONE | tee.exe -a build.log
+
+  for %%i in (temp_python3%%v\python\dist\*.whl) do (
     echo Copy %%i to export... | tee.exe -a build.log
     copy %%i export\.
     echo Copy %%i to export...DONE | tee.exe -a build.log