From e319b18d593c4ea15ae1420db05c7ca4e3e9ba69 Mon Sep 17 00:00:00 2001 From: Eduardo Almeida Date: Thu, 21 Jul 2022 17:07:12 +0100 Subject: [PATCH] ci: Refactor before_script segments --- utils/tests/gitlab-ci-alpine.yml | 6 ++- utils/tests/gitlab-ci-clang.yml | 17 ++++--- utils/tests/gitlab-ci-cppyy.yml | 31 ++++++++---- utils/tests/gitlab-ci-doc.yml | 5 +- utils/tests/gitlab-ci-fedora.yml | 13 ++++- utils/tests/gitlab-ci-gcc.yml | 15 +++++- utils/tests/gitlab-ci-test.yml | 47 +++-------------- utils/tests/gitlab-ci-ubuntu.yml | 87 +++++++++++++++++++++++--------- utils/tests/gitlab-ci.yml | 12 +++-- 9 files changed, 145 insertions(+), 88 deletions(-) diff --git a/utils/tests/gitlab-ci-alpine.yml b/utils/tests/gitlab-ci-alpine.yml index f5c475c95..e2a5b1312 100644 --- a/utils/tests/gitlab-ci-alpine.yml +++ b/utils/tests/gitlab-ci-alpine.yml @@ -20,6 +20,10 @@ weekly-build-alpine-latest-default: before_script: - apk update - apk upgrade - - apk add bash ccache cmake g++ iproute2 ninja python3 linux-headers + - apk add + g++ cmake ninja ccache + python3 + bash + iproute2 linux-headers variables: MODE: default diff --git a/utils/tests/gitlab-ci-clang.yml b/utils/tests/gitlab-ci-clang.yml index d94b58d67..e0d47d52a 100644 --- a/utils/tests/gitlab-ci-clang.yml +++ b/utils/tests/gitlab-ci-clang.yml @@ -15,13 +15,19 @@ - $RELEASE == "weekly" - $CLANG_BUILD_ENABLE == "True" before_script: - - apt-get --quiet update --yes - - apt-get --quiet install cmake ninja-build gsl-bin libgsl-dev ccache sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-0 libgtk-3-dev libboost-all-dev libgsl23 ssh --yes - - apt-get --quiet install software-properties-common gnupg wget --yes + - apt update + - apt install -y ca-certificates gnupg software-properties-common wget - echo -en "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-$CLANG main \n deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-$CLANG main" > /etc/apt/sources.list.d/llvm.list - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - - - apt-get --quiet update --yes - - apt-get --quiet install ca-certificates build-essential clang-$CLANG --yes + - apt update + - apt install -y + clang-$CLANG cmake ninja-build ccache build-essential + libboost-all-dev + libgtk-3-0 libgtk-3-dev + libgsl-dev libgsl23 gsl-bin + libsqlite3-dev sqlite3 + libxml2 libxml2-dev + ssh variables: COMPILER: clang++-$CLANG @@ -119,4 +125,3 @@ weekly-build-clang-11-optimized: variables: CLANG: 11 MODE: optimized - diff --git a/utils/tests/gitlab-ci-cppyy.yml b/utils/tests/gitlab-ci-cppyy.yml index 0986f651e..d9d28f705 100644 --- a/utils/tests/gitlab-ci-cppyy.yml +++ b/utils/tests/gitlab-ci-cppyy.yml @@ -7,8 +7,12 @@ cppyy-22.04: - nsnam - linux before_script: - - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get install -y g++ cmake ninja-build ccache libgsl-dev libgtk-3-dev libboost-dev wget git python3 python3-pip + - apt update + - DEBIAN_FRONTEND=noninteractive apt install -y + g++ cmake ninja-build ccache + python3 python3-pip + libboost-dev libgsl-dev libgtk-3-dev + git wget - pip install cppyy matplotlib numpy script: - ./ns3 configure -G Ninja --enable-python-bindings @@ -29,7 +33,6 @@ cppyy-22.04: - ./ns3 run ./utils/python-unit-tests.py timeout: 9h - cppyy-20.04: image: ubuntu:20.04 # python 3.8 only: @@ -39,8 +42,12 @@ cppyy-20.04: - nsnam - linux before_script: - - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get install -y g++ cmake ninja-build ccache libgsl-dev libgtk-3-dev libboost-dev wget git python3 python3-pip + - apt update + - DEBIAN_FRONTEND=noninteractive apt install -y + g++ cmake ninja-build ccache + python3 python3-pip + libboost-dev libgsl-dev libgtk-3-dev + git wget - pip install cppyy script: - ./ns3 configure -G Ninja --enable-python-bindings @@ -60,11 +67,15 @@ cppyy-18.04: - nsnam - linux before_script: - - apt-get update - - apt-get install -y software-properties-common - - add-apt-repository ppa:ubuntu-toolchain-r/test --yes - - apt-get update - - DEBIAN_FRONTEND=noninteractive apt-get install -y cmake ninja-build ccache libgsl-dev libgtk-3-dev libboost-dev wget git g++-9 gcc-9 python3 python3-pip llvm-dev libclang-dev + - apt update + - apt install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test -y + - apt update + - DEBIAN_FRONTEND=noninteractive apt install -y + g++-9 gcc-9 cmake ninja-build ccache libclang-dev llvm-dev + python3 python3-pip + libboost-dev libgsl-dev libgtk-3-dev + git wget - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 800 --slave /usr/bin/g++ g++ /usr/bin/g++-9 - pip3 install cppyy-cling cppyy-backend CPyCppyy cppyy # older python version may install dependencies in the wrong order script: diff --git a/utils/tests/gitlab-ci-doc.yml b/utils/tests/gitlab-ci-doc.yml index 2c958b83b..1777554b6 100644 --- a/utils/tests/gitlab-ci-doc.yml +++ b/utils/tests/gitlab-ci-doc.yml @@ -5,8 +5,9 @@ stage: documentation before_script: - pacman -Syu --noconfirm - - pacman -Sy base-devel ninja cmake python ccache gsl gtk3 boost --noconfirm - - pacman -Sy git doxygen graphviz imagemagick dia texlive-core texlive-bin texlive-latexextra python-sphinx --noconfirm + base-devel ninja cmake python ccache gsl gtk3 boost + git + doxygen graphviz imagemagick dia texlive-core texlive-bin texlive-latexextra python-sphinx # no policy is a good policy. - mv /etc/ImageMagick-7/policy.xml /etc/ImageMagick-7/policy.xml.bak # - sed -i "s/EPS,PDF,//g" /etc/ImageMagick-7/policy.xml diff --git a/utils/tests/gitlab-ci-fedora.yml b/utils/tests/gitlab-ci-fedora.yml index d6989287b..dca32a427 100644 --- a/utils/tests/gitlab-ci-fedora.yml +++ b/utils/tests/gitlab-ci-fedora.yml @@ -15,7 +15,18 @@ - $FEDORA == "True" before_script: - dnf update --assumeyes - - dnf install gcc gcc-c++ cmake ninja-build python python-devel ccache gsl gsl-devel gtk3 gtk3-devel sqlite sqlite-devel libxml2 libxml2-devel openmpi openmpi-devel boost-devel pygobject3-devel python3-gobject gobject-introspection-devel goocanvas2-devel graphviz-devel graphviz ipython --assumeyes + - dnf install --assumeyes + gcc-c++ cmake ccache ninja-build + boost-devel + gtk3 gtk3-devel + gsl gsl-devel + sqlite sqlite-devel + libxml2 libxml2-devel + openmpi openmpi-devel + goocanvas2-devel + python python-devel + pygobject3-devel python3-gobject gobject-introspection-devel + graphviz-devel graphviz ipython - source /etc/profile.d/modules.sh - module load mpi/openmpi-x86_64 variables: diff --git a/utils/tests/gitlab-ci-gcc.yml b/utils/tests/gitlab-ci-gcc.yml index 98e8c52f5..34dd57629 100644 --- a/utils/tests/gitlab-ci-gcc.yml +++ b/utils/tests/gitlab-ci-gcc.yml @@ -14,9 +14,20 @@ - $RELEASE == "weekly" - $GCC_BUILD_ENABLE == "True" before_script: - - apt-get --quiet update --yes + - apt update - update-alternatives --install /usr/bin/gfortran gfortran /usr/local/bin/gfortran 999 - - apt-get --quiet install gcc g++ cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev gsl-bin libgsl0-dev ssh --yes + - DEBIAN_FRONTEND=noninteractive apt install -y + g++ cmake ninja-build ccache + python3 python3-dev + libgsl0-dev gsl-bin + libboost-all-dev + libgtk-3-dev + libfl-dev + libxml2 libxml2-dev + libopenmpi-dev openmpi-bin openmpi-common openmpi-doc + libsqlite3-dev sqlite3 + qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools + bison flex ssh variables: COMPILER: g++ ENABLE_MPI: --enable-mpi diff --git a/utils/tests/gitlab-ci-test.yml b/utils/tests/gitlab-ci-test.yml index d24edd302..eb021e770 100644 --- a/utils/tests/gitlab-ci-test.yml +++ b/utils/tests/gitlab-ci-test.yml @@ -7,6 +7,12 @@ .base-test: stage: test + image: archlinux + before_script: + - pacman -Syu --noconfirm + base-devel cmake ninja ccache valgrind + python + boost gsl gtk3 script: - mkdir -p $CCACHE_BASEDIR_VALUE - export CCACHE_BASEDIR=${PWD} @@ -22,6 +28,7 @@ paths: - $CCACHE_BASEDIR_VALUE/ variables: + COMPILER: g++ CCACHE_BASEDIR_VALUE: ns-3-ccache-storage VALGRIND_FLAG: "" FULLNESS: "" @@ -29,9 +36,7 @@ # Run the test.py script with files compiled in debug mode daily-test-debug: extends: .base-test - image: archlinux variables: - COMPILER: g++ MODE: debug only: variables: @@ -39,45 +44,30 @@ daily-test-debug: tags: - nsnam - linux - before_script: - - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm # Run the test.py script with files compiled in default mode daily-test-default: extends: .base-test - image: archlinux variables: - COMPILER: g++ MODE: default only: variables: - $RELEASE == "daily" - before_script: - - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm # Run the test.py script with files compiled in optimized mode daily-test-optimized: extends: .base-test - image: archlinux variables: - COMPILER: g++ MODE: optimized only: variables: - $RELEASE == "daily" - before_script: - - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm ### Valgrind tests # Run the test.py script with files compiled in optimized mode + valgrind (daily) daily-test-optimized-valgrind: extends: .base-test - image: archlinux variables: - COMPILER: g++ MODE: optimized VALGRIND_FLAG: -g only: @@ -86,16 +76,11 @@ daily-test-optimized-valgrind: tags: - nsnam - linux - before_script: - - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm # Run the test.py script with files compiled in debug mode weekly-test-debug-valgrind: extends: .base-test - image: archlinux variables: - COMPILER: g++ MODE: debug VALGRIND_FLAG: -g only: @@ -104,16 +89,11 @@ weekly-test-debug-valgrind: tags: - nsnam - linux - before_script: - - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm # Run the test.py script with files compiled in default mode weekly-test-default-valgrind: extends: .base-test - image: archlinux variables: - COMPILER: g++ MODE: default VALGRIND_FLAG: -g only: @@ -122,16 +102,11 @@ weekly-test-default-valgrind: tags: - nsnam - linux - before_script: - - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm # Run the test.py script with files compiled in optimized mode weekly-test-optimized-valgrind: extends: .base-test - image: archlinux variables: - COMPILER: g++ MODE: optimized VALGRIND_FLAG: -g only: @@ -140,16 +115,11 @@ weekly-test-optimized-valgrind: tags: - nsnam - linux - before_script: - - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm # Do a check for the TAKES_FOREVER jobs, only in optimized mode weekly-test-takes-forever-optimized: extends: .base-test - image: archlinux variables: - COMPILER: g++ MODE: optimized FULLNESS: "-f TAKES_FOREVER" only: @@ -158,6 +128,3 @@ weekly-test-takes-forever-optimized: tags: - nsnam - linux - before_script: - - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm diff --git a/utils/tests/gitlab-ci-ubuntu.yml b/utils/tests/gitlab-ci-ubuntu.yml index a90a3db6c..cb27e5c1f 100644 --- a/utils/tests/gitlab-ci-ubuntu.yml +++ b/utils/tests/gitlab-ci-ubuntu.yml @@ -14,10 +14,21 @@ - $RELEASE == "weekly" - $UBUNTU == "True" before_script: - - apt-get --quiet update --yes - - apt-get --quiet upgrade --yes - - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata - - apt-get --quiet install gcc g++ cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev gsl-bin libgsl-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev $LIBGSL ssh --yes + - apt update + - apt upgrade -y + - DEBIAN_FRONTEND=noninteractive apt install -y + g++ cmake ninja-build ccache + python3 python3-dev + gsl-bin libgsl-dev $LIBGSL + libboost-all-dev + libgtk-3-dev + libfl-dev + libxml2 libxml2-dev + libopenmpi-dev openmpi-bin openmpi-common openmpi-doc + libsqlite3-dev sqlite3 + qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools + bison flex ssh + variables: COMPILER: g++ @@ -26,13 +37,23 @@ weekly-build-ubuntu-18.04-debug: extends: .weekly-build-ubuntu image: ubuntu:18.04 before_script: - - apt-get --quiet update --yes - - apt-get --quiet upgrade --yes - - apt-get --quiet install software-properties-common --yes - - add-apt-repository ppa:ubuntu-toolchain-r/test --yes - - apt-get --quiet update --yes - - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata - - apt-get --quiet install gcc-9 g++-9 cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev gsl-bin libgsl-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev $LIBGSL ssh --yes + - apt update + - apt upgrade -y + - apt install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test -y + - apt update + - DEBIAN_FRONTEND=noninteractive apt install -y + g++-9 cmake ninja-build ccache + python3 python3-dev + gsl-bin libgsl-dev $LIBGSL + libboost-all-dev + libgtk-3-dev + libfl-dev + libxml2 libxml2-dev + libopenmpi-dev openmpi-bin openmpi-common openmpi-doc + libsqlite3-dev sqlite3 + qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools + bison flex ssh - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7 - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 800 --slave /usr/bin/g++ g++ /usr/bin/g++-9 variables: @@ -43,12 +64,22 @@ weekly-build-ubuntu-18.04-default: extends: .weekly-build-ubuntu image: ubuntu:18.04 before_script: - - apt-get --quiet update --yes - - apt-get --quiet install software-properties-common --yes - - add-apt-repository ppa:ubuntu-toolchain-r/test --yes - - apt-get --quiet update --yes - - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata - - apt-get --quiet install gcc-9 g++-9 cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev gsl-bin libgsl-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev $LIBGSL ssh --yes + - apt update + - apt install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test -y + - apt update + - DEBIAN_FRONTEND=noninteractive apt install -y + g++-9 cmake ninja-build ccache + python3 python3-dev + gsl-bin libgsl-dev $LIBGSL + libboost-all-dev + libgtk-3-dev + libfl-dev + libxml2 libxml2-dev + libopenmpi-dev openmpi-bin openmpi-common openmpi-doc + libsqlite3-dev sqlite3 + qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools + bison flex ssh - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7 - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 800 --slave /usr/bin/g++ g++ /usr/bin/g++-9 variables: @@ -59,12 +90,22 @@ weekly-build-ubuntu-18.04-optimized: extends: .weekly-build-ubuntu image: ubuntu:18.04 before_script: - - apt-get --quiet update --yes - - apt-get --quiet install software-properties-common --yes - - add-apt-repository ppa:ubuntu-toolchain-r/test --yes - - apt-get --quiet update --yes - - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata - - apt-get --quiet install gcc-9 g++-9 cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev gsl-bin libgsl-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev $LIBGSL ssh --yes + - apt update + - apt install -y software-properties-common + - add-apt-repository ppa:ubuntu-toolchain-r/test -y + - apt update + - DEBIAN_FRONTEND=noninteractive apt install -y + g++-9 cmake ninja-build ccache + python3 python3-dev + gsl-bin libgsl-dev $LIBGSL + libboost-all-dev + libgtk-3-dev + libfl-dev + libxml2 libxml2-dev + libopenmpi-dev openmpi-bin openmpi-common openmpi-doc + libsqlite3-dev sqlite3 + qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools + bison flex ssh - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7 - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 800 --slave /usr/bin/g++ g++ /usr/bin/g++-9 variables: diff --git a/utils/tests/gitlab-ci.yml b/utils/tests/gitlab-ci.yml index ac5c5a40b..4688f2eac 100644 --- a/utils/tests/gitlab-ci.yml +++ b/utils/tests/gitlab-ci.yml @@ -72,7 +72,9 @@ per-commit-compile-debug: MODE: debug before_script: - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost ninja cmake --noconfirm + base-devel cmake ninja ccache + python + boost gsl gtk3 per-commit-compile-default: extends: .base-build @@ -87,7 +89,9 @@ per-commit-compile-default: MODE: default before_script: - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost ninja cmake --noconfirm + base-devel cmake ninja ccache + python + boost gsl gtk3 per-commit-compile-optimized: extends: .base-build @@ -102,7 +106,9 @@ per-commit-compile-optimized: MODE: optimized before_script: - pacman -Syu --noconfirm - - pacman -Sy base-devel python ccache gsl gtk3 boost ninja cmake --noconfirm + base-devel cmake ninja ccache + python + boost gsl gtk3 # Weekly jobs for other distribution and compilers include: