diff --git a/.github/workflows/docker_bazel.yml b/.github/workflows/docker_bazel.yml new file mode 100644 index 0000000000..f47b5c544d --- /dev/null +++ b/.github/workflows/docker_bazel.yml @@ -0,0 +1,23 @@ +name: Docker Bazel + +on: [push, pull_request] + +jobs: + Distros: + runs-on: ubuntu-latest + strategy: + matrix: + distro: [alpine, archlinux, centos, debian, fedora, opensuse, ubuntu] + fail-fast: false + env: + DISTRO: ${{ matrix.distro }} + steps: + - uses: actions/checkout@v2 + - name: Build env image + run: make --directory=bazel ${DISTRO}_env + - name: Build devel image + run: make --directory=bazel ${DISTRO}_devel + - name: Build project + run: make --directory=bazel ${DISTRO}_build + - name: Test project + run: make --directory=bazel ${DISTRO}_test diff --git a/bazel/Makefile b/bazel/Makefile new file mode 100644 index 0000000000..44f07d2d10 --- /dev/null +++ b/bazel/Makefile @@ -0,0 +1,135 @@ +PROJECT := ortools +BUILD_SYSTEM := bazel +BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +SHA1 := $(shell git rev-parse --verify HEAD) + +# General commands +.PHONY: help +BOLD=\e[1m +RESET=\e[0m + +help: + @echo -e "${BOLD}SYNOPSIS${RESET}" + @echo -e "\tmake [NOCACHE=1]" + @echo + @echo -e "${BOLD}DESCRIPTION${RESET}" + @echo -e "\ttest build inside docker container to have a reproductible build." + @echo + @echo -e "${BOLD}MAKE TARGETS${RESET}" + @echo -e "\t${BOLD}help${RESET}: display this help and exit." + @echo + @echo -e "\t${BOLD}${RESET}: build all docker images." + @echo -e "\t${BOLD}_${RESET}: build the docker image for a specific distro." + @echo -e "\t${BOLD}save_${RESET}: Save all docker images." + @echo -e "\t${BOLD}save__${RESET}: Save the docker image for a specific distro." + @echo -e "\t${BOLD}sh__${RESET}: run a container using the docker image (debug purpose)." + @echo + @echo -e "\t${BOLD}${RESET}:" + @echo -e "\t\t${BOLD}env${RESET}" + @echo -e "\t\t${BOLD}devel${RESET}" + @echo -e "\t\t${BOLD}build${RESET}" + @echo -e "\t\t${BOLD}test${RESET}" + @echo + @echo -e "\t${BOLD}${RESET}:" + @echo -e "\t\t${BOLD}alpine${RESET} (edge)" + @echo -e "\t\t${BOLD}archlinux${RESET} (latest)" + @echo -e "\t\t${BOLD}centos${RESET} (latest)" + @echo -e "\t\t${BOLD}debian${RESET} (latest)" + @echo -e "\t\t${BOLD}fedora${RESET} (latest)" + @echo -e "\t\t${BOLD}opensuse${RESET} (tumbleweed)" + @echo -e "\t\t${BOLD}ubuntu${RESET} (rolling)" + @echo -e "\te.g. 'make ubuntu_test'" + @echo + @echo -e "\t${BOLD}clean${RESET}: Remove cache and ALL docker images." + @echo -e "\t${BOLD}clean_${RESET}: Remove cache and docker images for the specified distro." + @echo + @echo -e "\t${BOLD}NOCACHE=1${RESET}: use 'docker build --no-cache' when building container (default use cache)." + @echo + @echo -e "branch: $(BRANCH)" + @echo -e "sha1: $(SHA1)" + +# Need to add cmd_distro to PHONY otherwise target are ignored since they do not +# contain recipe (using FORCE do not work here) +.PHONY: all +all: build + +# Delete all implicit rules to speed up makefile +MAKEFLAGS += --no-builtin-rules +.SUFFIXES: +# Remove some rules from gmake that .SUFFIXES does not remove. +SUFFIXES = +# Keep all intermediate files +# ToDo: try to remove it later +.SECONDARY: + +# Docker image name prefix. +IMAGE := ${PROJECT}/${BUILD_SYSTEM} + +ifdef NOCACHE +DOCKER_BUILD_CMD := docker build --no-cache +else +DOCKER_BUILD_CMD := docker build +endif + +DOCKER_RUN_CMD := docker run --rm --init --net=host + +# Currently supported distro +DISTROS = alpine archlinux centos debian fedora opensuse ubuntu + +# $* stem +# $< first prerequist +# $@ target name + +STAGES = env devel build test +define make-stage-target +#$$(info STAGE: $1) +.PHONY: $1 +#$$(info Create targets: $1 $(addsuffix _$1, $(DISTROS)).) +targets_$1 = $(addsuffix _$1, $(DISTROS)) +.PHONY: $(targets_$1) +$1: $$(targets_$1) +$$(targets_$1): %_$1: docker/%/Dockerfile + #@docker image rm -f ${IMAGE}:$$*_$1 2>/dev/null + ${DOCKER_BUILD_CMD} --target=$1 --tag ${IMAGE}:$$*_$1 -f $$< .. + +#$$(info Create targets: save_$1 $(addprefix save_, $(addsuffix _$1, $(DISTROS))) (debug).) +save_targets_$1 = $(addprefix save_, $(addsuffix _$1, $(DISTROS))) +.PHONY: save_$1 $(save_targets_$1) +save_$1: $$(save_targets_$1) +$$(save_targets_$1): save_%_$1: cache/%/docker_$1.tar +cache/%/docker_$1.tar: %_$1 + @rm -f $$@ + mkdir -p cache/$$* + docker save ${IMAGE}:$$*_$1 -o $$@ + +#$$(info Create targets: $(addprefix sh_, $(addsuffix _$1, $(DISTROS))) (debug).) +sh_targets_$1 = $(addprefix sh_, $(addsuffix _$1, $(DISTROS))) +.PHONY: $(sh_targets_$1) +$$(sh_targets_$1): sh_%_$1: %_$1 + ${DOCKER_RUN_CMD} -it --name ${PROJECT}_${BUILD_SYSTEM}_$$*_$1 ${IMAGE}:$$*_$1 + +#$$(info Create targets: $(addprefix clean_, $(addsuffix _$1, $(DISTROS))).) +clean_targets_$1 = $(addprefix clean_, $(addsuffix _$1, $(DISTROS))) +.PHONY: clean_$1 $(clean_targets_$1) +clean_$1: $$(clean_targets_$1) +$$(clean_targets_$1): clean_%_$1: + docker image rm -f ${IMAGE}:$$*_$1 2>/dev/null + rm -f cache/$$*/docker_$1.tar +endef + +$(foreach stage,$(STAGES),$(eval $(call make-stage-target,$(stage)))) + +## CLEAN ## +clean_targets = $(addprefix clean_, $(DISTROS)) +.PHONY: clean $(clean_targets) +clean: $(clean_targets) + docker container prune -f + docker image prune -f + -rmdir cache +$(clean_targets): clean_%: $(addprefix clean_%_, $(STAGES)) + -rmdir cache/$* + +.PHONY: distclean +distclean: clean + -docker container rm -f $$(docker container ls -aq) + -docker image rm -f $$(docker image ls -aq) diff --git a/bazel/README.md b/bazel/README.md index cdba8cd852..5888e917cb 100644 --- a/bazel/README.md +++ b/bazel/README.md @@ -1,6 +1,39 @@ -# Docker file for testing +# OR-Tools Bazel Build Instructions -# Bazel Test -```sh -docker build -t bazel -f bazel.Dockerfile .. -``` +| OS | | +|:--------|------------------------------------| +| Linux | [![Status][linux_svg]][linux_link] | +| MacOS | [![Status][osx_svg]][osx_link] | +| Windows | [![Status][win_svg]][win_link] | + +Dockers: [![Status][docker_svg]][docker_link] + +[docker_svg]: https://github.com/google/or-tools/workflows/Docker%20Bazel/badge.svg +[docker_link]: https://github.com/google/or-tools/actions?query=workflow%3A"Docker+Bazel" + +## Introduction + + +OR-Tools comes with a Bazel based build ([WORKSPACE](../WORKSPACE)) that can be +used on a wide range of platforms. If you don't have Bazel installed already, +you can download it for free from . + +**warning: Currently OR-Tools Bazel doesn't support Python, Java nor .Net, please use +the Makefile or CMake based build instead.** + +## [Dependencies](#deps) +OR-Tools depends on severals mandatory libraries. + +* Google Abseil-cpp, +* Google Gflags, +* Google Glog, +* Google Protobuf, +* Google Gtest, +* GLPK (GNU Linear Programming Kit) + +## [Integrating OR-Tools in your Bazel Project](#integration) +TODO diff --git a/bazel/doc/ci.md b/bazel/doc/ci.md new file mode 100644 index 0000000000..92024ac435 --- /dev/null +++ b/bazel/doc/ci.md @@ -0,0 +1,37 @@ +# CI: Makefile/Docker testing +To test the build on various distro, I'm using docker containers and a Makefile for orchestration. + +pros: +* You are independent of third party CI runner config (e.g. github actions runners or Travis-CI VM images). +* You can run it locally on your linux system. +* Most CI provide runner with docker and Makefile installed (e.g. tarvis-ci [minimal images](https://docs.travis-ci.com/user/languages/minimal-and-generic/). + +cons: +* Only GNU/Linux distro supported. +* Could take few GiB (~30 GiB for all distro and all languages) + * ~500MiB OS + C++/CMake tools, + * ~150 MiB Python, + * ~400 MiB dotnet-sdk, + * ~400 MiB java-jdk. + +# Usage +To get the help simply type: +```sh +make +``` + +note: you can also use from top directory +```sh +make --directory=bazel +``` + +## Example +For example to test inside an `Ubuntu` container: +```sh +make ubuntu_test +``` + +# Docker Layers +Dockerfile is splitted in several stages. + +![docker](docker.svg) diff --git a/bazel/doc/docker.dot b/bazel/doc/docker.dot new file mode 100644 index 0000000000..681c60ed52 --- /dev/null +++ b/bazel/doc/docker.dot @@ -0,0 +1,45 @@ +@startdot +digraph DockerDeps { + //rankdir=BT; + rankdir=TD; + node [shape=cylinder, style="rounded,filled", color=black, fillcolor=royalblue]; + DISTRO_IMG [label=":latest"]; + PKG [label="packages\ne.g. bazel, g++", shape=box3d]; + SRC [label="git repo", shape=folder]; + + subgraph clusterDockerfile { + ENV_IMG [label="ortools/bazel:_env\nenv"]; + DEVEL_IMG [label="ortools/bazel:_devel\ndevel"]; + BUILD_IMG [label="ortools/bazel:_build\nbuild"]; + TEST_IMG [label="ortools/bazel:_test\ntest"]; + + edge [color=black]; + ENV_IMG -> DEVEL_IMG; + DEVEL_IMG -> BUILD_IMG; + BUILD_IMG -> TEST_IMG; + + color=royalblue; + label = "docker//Dockerfile"; + } + DISTRO_IMG -> ENV_IMG; + PKG -> ENV_IMG [label="install", style="dashed"]; + SRC -> DEVEL_IMG [label="copy", style="dashed"]; + + subgraph clusterCache { + node [shape=note, style="rounded,filled", color=black, fillcolor=royalblue]; + ENV_TAR [label="docker_env.tar"]; + DEVEL_TAR [label="docker_devel.tar"]; + BUILD_TAR [label="docker_build.tar"]; + TEST_TAR [label="docker_test.tar"]; + + edge [color=red]; + ENV_IMG -> ENV_TAR [label="make save__env"]; + DEVEL_IMG -> DEVEL_TAR [label="make save__devel"]; + BUILD_IMG -> BUILD_TAR [label="make save__build"]; + TEST_IMG -> TEST_TAR [label="make save__test"]; + + color=royalblue; + label = "cache//"; + } +} +@enddot diff --git a/bazel/doc/docker.svg b/bazel/doc/docker.svg new file mode 100644 index 0000000000..ef3a75a91e --- /dev/null +++ b/bazel/doc/docker.svg @@ -0,0 +1,176 @@ + + + + + + +DockerDeps + + +clusterDockerfile + +docker/<distro>/Dockerfile + + +clusterCache + +cache/<distro>/ + + + +DISTRO_IMG + + +<distro>:latest + + + +ENV_IMG + + +ortools/bazel:<distro>_env +env + + + +DISTRO_IMG->ENV_IMG + + + + + +PKG + + + + +packages +e.g. bazel, g++ + + + +PKG->ENV_IMG + + +install + + + +SRC + +git repo + + + +DEVEL_IMG + + +ortools/bazel:<distro>_devel +devel + + + +SRC->DEVEL_IMG + + +copy + + + +ENV_IMG->DEVEL_IMG + + + + + +ENV_TAR + + + +docker_env.tar + + + +ENV_IMG->ENV_TAR + + +make save_<distro>_env + + + +BUILD_IMG + + +ortools/bazel:<distro>_build +build + + + +DEVEL_IMG->BUILD_IMG + + + + + +DEVEL_TAR + + + +docker_devel.tar + + + +DEVEL_IMG->DEVEL_TAR + + +make save_<distro>_devel + + + +TEST_IMG + + +ortools/bazel:<distro>_test +test + + + +BUILD_IMG->TEST_IMG + + + + + +BUILD_TAR + + + +docker_build.tar + + + +BUILD_IMG->BUILD_TAR + + +make save_<distro>_build + + + +TEST_TAR + + + +docker_test.tar + + + +TEST_IMG->TEST_TAR + + +make save_<distro>_test + + + diff --git a/bazel/doc/generate_image.sh b/bazel/doc/generate_image.sh new file mode 100755 index 0000000000..cd1743a7f1 --- /dev/null +++ b/bazel/doc/generate_image.sh @@ -0,0 +1,8 @@ +#/usr/bin/env bash +set -ex + +rm -f *.svg *.png +for i in *.dot; do + #plantuml -Tpng "$i"; + plantuml -Tsvg "$i"; +done diff --git a/bazel/docker/alpine/Dockerfile b/bazel/docker/alpine/Dockerfile new file mode 100644 index 0000000000..e0d59cd882 --- /dev/null +++ b/bazel/docker/alpine/Dockerfile @@ -0,0 +1,19 @@ +# Create a virtual environment with all tools installed +# ref: https://hub.docker.com/_/alpine +FROM alpine:edge AS env +LABEL maintainer="corentinl@google.com" +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN apk add --no-cache git build-base linux-headers zlib-dev +RUN apk add --no-cache -X http://dl-cdn.alpinelinux.org/alpine/edge/testing bazel +CMD [ "/bin/sh" ] + +FROM env AS devel +WORKDIR /home/lib +COPY . . + +FROM devel as build +RUN bazel build --curses=no --copt='-Wno-sign-compare' //...:all + +FROM build as test +RUN bazel test -c opt --curses=no --copt='-Wno-sign-compare' //...:all diff --git a/bazel/docker/archlinux/Dockerfile b/bazel/docker/archlinux/Dockerfile new file mode 100644 index 0000000000..822e0ebfff --- /dev/null +++ b/bazel/docker/archlinux/Dockerfile @@ -0,0 +1,18 @@ +# Create a virtual environment with all tools installed +# ref: https://hub.docker.com/_/archlinux/ +FROM archlinux/base AS env +LABEL maintainer="corentinl@google.com" +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN pacman -Syu --noconfirm git base-devel bazel +CMD [ "/bin/bash" ] + +FROM env AS devel +WORKDIR /home/lib +COPY . . + +FROM devel as build +RUN bazel build --curses=no --copt='-Wno-sign-compare' //...:all + +FROM build as test +RUN bazel test -c opt --curses=no --copt='-Wno-sign-compare' //...:all diff --git a/bazel/docker/centos/Dockerfile b/bazel/docker/centos/Dockerfile new file mode 100644 index 0000000000..389678b96d --- /dev/null +++ b/bazel/docker/centos/Dockerfile @@ -0,0 +1,30 @@ +# Create a virtual environment with all tools installed +# ref: https://hub.docker.com/_/centos +FROM centos:latest AS env +LABEL maintainer="corentinl@google.com" +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN yum -y update \ +&& yum -y install git wget zlib-devel \ +&& yum -y groupinstall "Development Tools" \ +&& yum clean all \ +&& rm -rf /var/cache/yum +# Install bazel +# see: https://docs.bazel.build/versions/master/install-redhat.html#installing-on-centos-7 +RUN cd /etc/yum.repos.d \ +&& wget https://copr.fedorainfracloud.org/coprs/vbatts/bazel/repo/epel-8/vbatts-bazel-epel-8.repo \ +&& yum -y update \ +&& yum -y install bazel \ +&& yum clean all \ +&& rm -rf /var/cache/yum +CMD [ "/usr/bin/bash" ] + +FROM env AS devel +WORKDIR /home/lib +COPY . . + +FROM devel as build +RUN bazel build --curses=no --copt='-Wno-sign-compare' //...:all + +FROM build as test +RUN bazel test -c opt --curses=no --copt='-Wno-sign-compare' //...:all diff --git a/bazel/docker/debian/Dockerfile b/bazel/docker/debian/Dockerfile new file mode 100644 index 0000000000..5f95e985ba --- /dev/null +++ b/bazel/docker/debian/Dockerfile @@ -0,0 +1,30 @@ +# Create a virtual environment with all tools installed +# ref: https://hub.docker.com/_/debian +FROM debian:latest AS env +LABEL maintainer="corentinl@google.com" +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN apt-get update -qq \ +&& apt-get install -yq git wget curl libssl-dev build-essential \ +&& apt-get install -yq python3-dev python3-pip \ +&& apt-get install -yq default-jdk \ +&& apt-get clean \ +&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +# Install Bazel +RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - +RUN echo "deb [arch=amd64] https://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list +RUN apt-get update -qq \ +&& apt-get install -yq bazel \ +&& apt-get clean \ +&& rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +CMD [ "/bin/bash" ] + +FROM env AS devel +WORKDIR /home/lib +COPY . . + +FROM devel as build +RUN bazel build --curses=no --copt='-Wno-sign-compare' //...:all + +FROM build as test +RUN bazel test -c opt --curses=no --copt='-Wno-sign-compare' //...:all diff --git a/bazel/docker/fedora/Dockerfile b/bazel/docker/fedora/Dockerfile new file mode 100644 index 0000000000..7bfe4012ec --- /dev/null +++ b/bazel/docker/fedora/Dockerfile @@ -0,0 +1,25 @@ +# Create a virtual environment with all tools installed +# ref: https://hub.docker.com/_/fedora +FROM fedora:latest AS env +LABEL maintainer="corentinl@google.com" +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN dnf -y update \ +&& dnf -y install git \ +&& dnf -y groupinstall "Development Tools" \ +&& dnf -y install gcc-c++ zlib-devel \ +&& dnf -y install dnf-plugins-core \ +&& dnf -y copr enable vbatts/bazel \ +&& dnf -y install bazel \ +&& dnf clean all +CMD [ "/usr/bin/bash" ] + +FROM env AS devel +WORKDIR /home/lib +COPY . . + +FROM devel as build +RUN bazel build --curses=no --copt='-Wno-sign-compare' //...:all + +FROM build as test +RUN bazel test -c opt --curses=no --copt='-Wno-sign-compare' //...:all diff --git a/bazel/docker/opensuse/Dockerfile b/bazel/docker/opensuse/Dockerfile new file mode 100644 index 0000000000..8ab0ee11e5 --- /dev/null +++ b/bazel/docker/opensuse/Dockerfile @@ -0,0 +1,21 @@ +# Create a virtual environment with all tools installed +# ref: https://hub.docker.com/r/opensuse/tumbleweed +FROM opensuse/tumbleweed AS env +LABEL maintainer="corentinl@google.com" +# Install system build dependencies +ENV PATH=/usr/local/bin:$PATH +RUN zypper update -y \ +&& zypper install -y git gcc gcc-c++ zlib-devel bazel \ +&& zypper clean -a +ENV CC=gcc CXX=g++ +CMD [ "/usr/bin/bash" ] + +FROM env AS devel +WORKDIR /home/lib +COPY . . + +FROM devel as build +RUN bazel build --curses=no --copt='-Wno-sign-compare' //...:all + +FROM build as test +RUN bazel test -c opt --curses=no --copt='-Wno-sign-compare' //...:all diff --git a/bazel/bazel.Dockerfile b/bazel/docker/ubuntu/Dockerfile similarity index 90% rename from bazel/bazel.Dockerfile rename to bazel/docker/ubuntu/Dockerfile index 299adc6b61..0c9dd4e0e0 100644 --- a/bazel/bazel.Dockerfile +++ b/bazel/docker/ubuntu/Dockerfile @@ -1,7 +1,7 @@ # Create a virtual environment with all tools installed # ref: https://hub.docker.com/_/ubuntu FROM ubuntu:rolling AS env -LABEL maintainer="mizux.dev@gmail.com" +LABEL maintainer="corentinl@google.com" # Install system build dependencies RUN apt-get update -qq \ && apt-get install -yq git wget curl libssl-dev build-essential \ @@ -18,12 +18,11 @@ RUN apt-get update -qq \ && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* CMD [ "/bin/bash" ] -# Create lib directory +FROM env AS devel WORKDIR /home/lib -# Bundle lib source COPY . . -FROM env as build +FROM devel as build RUN bazel build --curses=no --copt='-Wno-sign-compare' //...:all FROM build as test