ci: Refactor before_script segments

2022-07-21 17:07:12 +01:00
parent 71119bbb6b
commit e319b18d59
9 changed files with 145 additions and 88 deletions
--- a/utils/tests/gitlab-ci-alpine.yml
+++ b/utils/tests/gitlab-ci-alpine.yml
@@ -20,6 +20,10 @@ weekly-build-alpine-latest-default:
  before_script:
    - apk update
    - apk upgrade
-    - apk add bash ccache cmake g++ iproute2 ninja python3 linux-headers
+    - apk add
+      g++ cmake ninja ccache
+      python3
+      bash
+      iproute2 linux-headers
  variables:
    MODE: default
--- a/utils/tests/gitlab-ci-clang.yml
+++ b/utils/tests/gitlab-ci-clang.yml
@@ -15,13 +15,19 @@
      - $RELEASE == "weekly"
      - $CLANG_BUILD_ENABLE == "True"
  before_script:
-    - apt-get --quiet update --yes
-    - apt-get --quiet install cmake ninja-build gsl-bin libgsl-dev ccache sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-0 libgtk-3-dev libboost-all-dev libgsl23 ssh --yes
-    - apt-get --quiet install software-properties-common gnupg wget --yes
+    - apt update
+    - apt install -y ca-certificates gnupg software-properties-common wget
    - echo -en "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-$CLANG main \n deb-src http://apt.llvm.org/bionic/ llvm-toolchain-bionic-$CLANG main" > /etc/apt/sources.list.d/llvm.list
    - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add -
-    - apt-get --quiet update --yes
-    - apt-get --quiet install ca-certificates build-essential clang-$CLANG --yes
+    - apt update
+    - apt install -y
+      clang-$CLANG cmake ninja-build ccache build-essential
+      libboost-all-dev
+      libgtk-3-0 libgtk-3-dev
+      libgsl-dev libgsl23 gsl-bin
+      libsqlite3-dev sqlite3
+      libxml2 libxml2-dev
+      ssh
  variables:
    COMPILER: clang++-$CLANG

@@ -119,4 +125,3 @@ weekly-build-clang-11-optimized:
  variables:
    CLANG: 11
    MODE: optimized
-
--- a/utils/tests/gitlab-ci-cppyy.yml
+++ b/utils/tests/gitlab-ci-cppyy.yml
@@ -7,8 +7,12 @@ cppyy-22.04:
    - nsnam
    - linux
  before_script:
-    - apt-get update
-    - DEBIAN_FRONTEND=noninteractive apt-get install -y g++ cmake ninja-build ccache libgsl-dev libgtk-3-dev libboost-dev wget git python3 python3-pip
+    - apt update
+    - DEBIAN_FRONTEND=noninteractive apt install -y
+      g++ cmake ninja-build ccache
+      python3 python3-pip
+      libboost-dev libgsl-dev libgtk-3-dev
+      git wget
    - pip install cppyy matplotlib numpy
  script:
    - ./ns3 configure -G Ninja --enable-python-bindings
@@ -29,7 +33,6 @@ cppyy-22.04:
    - ./ns3 run ./utils/python-unit-tests.py
  timeout: 9h

-
 cppyy-20.04:
  image: ubuntu:20.04 # python 3.8
  only:
@@ -39,8 +42,12 @@ cppyy-20.04:
    - nsnam
    - linux
  before_script:
-    - apt-get update
-    - DEBIAN_FRONTEND=noninteractive apt-get install -y g++ cmake ninja-build ccache libgsl-dev libgtk-3-dev libboost-dev wget git python3 python3-pip
+    - apt update
+    - DEBIAN_FRONTEND=noninteractive apt install -y
+      g++ cmake ninja-build ccache
+      python3 python3-pip
+      libboost-dev libgsl-dev libgtk-3-dev
+      git wget
    - pip install cppyy
  script:
    - ./ns3 configure -G Ninja --enable-python-bindings
@@ -60,11 +67,15 @@ cppyy-18.04:
    - nsnam
    - linux
  before_script:
-    - apt-get update
-    - apt-get install -y software-properties-common
-    - add-apt-repository ppa:ubuntu-toolchain-r/test --yes
-    - apt-get update
-    - DEBIAN_FRONTEND=noninteractive apt-get install -y cmake ninja-build ccache libgsl-dev libgtk-3-dev libboost-dev wget git g++-9 gcc-9 python3 python3-pip llvm-dev libclang-dev
+    - apt update
+    - apt install -y software-properties-common
+    - add-apt-repository ppa:ubuntu-toolchain-r/test -y
+    - apt update
+    - DEBIAN_FRONTEND=noninteractive apt install -y
+      g++-9 gcc-9 cmake ninja-build ccache libclang-dev llvm-dev
+      python3 python3-pip
+      libboost-dev libgsl-dev libgtk-3-dev
+      git wget
    - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 800 --slave /usr/bin/g++ g++ /usr/bin/g++-9
    - pip3 install cppyy-cling cppyy-backend CPyCppyy cppyy # older python version may install dependencies in the wrong order
  script:
--- a/utils/tests/gitlab-ci-doc.yml
+++ b/utils/tests/gitlab-ci-doc.yml
@@ -5,8 +5,9 @@
  stage: documentation
  before_script:
    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel ninja cmake python ccache gsl gtk3 boost --noconfirm
-    - pacman -Sy git doxygen graphviz imagemagick dia texlive-core texlive-bin texlive-latexextra python-sphinx --noconfirm
+      base-devel ninja cmake python ccache gsl gtk3 boost
+      git
+      doxygen graphviz imagemagick dia texlive-core texlive-bin texlive-latexextra python-sphinx
    # no policy is a good policy.
    - mv /etc/ImageMagick-7/policy.xml /etc/ImageMagick-7/policy.xml.bak
 #    - sed -i "s/EPS,PDF,//g" /etc/ImageMagick-7/policy.xml
--- a/utils/tests/gitlab-ci-fedora.yml
+++ b/utils/tests/gitlab-ci-fedora.yml
@@ -15,7 +15,18 @@
      - $FEDORA == "True"
  before_script:
    - dnf update --assumeyes
-    - dnf install gcc gcc-c++ cmake ninja-build python python-devel ccache gsl gsl-devel gtk3 gtk3-devel sqlite sqlite-devel libxml2 libxml2-devel openmpi openmpi-devel boost-devel pygobject3-devel python3-gobject gobject-introspection-devel goocanvas2-devel graphviz-devel graphviz ipython --assumeyes
+    - dnf install --assumeyes
+      gcc-c++ cmake ccache ninja-build
+      boost-devel
+      gtk3 gtk3-devel
+      gsl gsl-devel
+      sqlite sqlite-devel
+      libxml2 libxml2-devel
+      openmpi openmpi-devel
+      goocanvas2-devel
+      python python-devel
+      pygobject3-devel python3-gobject gobject-introspection-devel
+      graphviz-devel graphviz ipython
    - source /etc/profile.d/modules.sh
    - module load mpi/openmpi-x86_64
  variables:
--- a/utils/tests/gitlab-ci-gcc.yml
+++ b/utils/tests/gitlab-ci-gcc.yml
@@ -14,9 +14,20 @@
      - $RELEASE == "weekly"
      - $GCC_BUILD_ENABLE == "True"
  before_script:
-    - apt-get --quiet update --yes
+    - apt update
    - update-alternatives --install /usr/bin/gfortran gfortran /usr/local/bin/gfortran 999
-    - apt-get --quiet install gcc g++ cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev gsl-bin libgsl0-dev ssh --yes
+    - DEBIAN_FRONTEND=noninteractive apt install -y
+      g++ cmake ninja-build ccache
+      python3 python3-dev
+      libgsl0-dev gsl-bin
+      libboost-all-dev
+      libgtk-3-dev
+      libfl-dev
+      libxml2 libxml2-dev
+      libopenmpi-dev openmpi-bin openmpi-common openmpi-doc
+      libsqlite3-dev sqlite3
+      qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools
+      bison flex ssh
  variables:
    COMPILER: g++
    ENABLE_MPI: --enable-mpi
--- a/utils/tests/gitlab-ci-test.yml
+++ b/utils/tests/gitlab-ci-test.yml
@@ -7,6 +7,12 @@

 .base-test:
  stage: test
+  image: archlinux
+  before_script:
+    - pacman -Syu --noconfirm
+      base-devel cmake ninja ccache valgrind
+      python
+      boost gsl gtk3
  script:
    - mkdir -p $CCACHE_BASEDIR_VALUE
    - export CCACHE_BASEDIR=${PWD}
@@ -22,6 +28,7 @@
    paths:
      - $CCACHE_BASEDIR_VALUE/
  variables:
+    COMPILER: g++
    CCACHE_BASEDIR_VALUE: ns-3-ccache-storage
    VALGRIND_FLAG: ""
    FULLNESS: ""
@@ -29,9 +36,7 @@
 # Run the test.py script with files compiled in debug mode
 daily-test-debug:
  extends: .base-test
-  image: archlinux
  variables:
-    COMPILER: g++
    MODE: debug
  only:
    variables:
@@ -39,45 +44,30 @@ daily-test-debug:
  tags:
    - nsnam
    - linux
-  before_script:
-    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm

 # Run the test.py script with files compiled in default mode
 daily-test-default:
  extends: .base-test
-  image: archlinux
  variables:
-    COMPILER: g++
    MODE: default
  only:
    variables:
      - $RELEASE == "daily"
-  before_script:
-    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm

 # Run the test.py script with files compiled in optimized mode
 daily-test-optimized:
  extends: .base-test
-  image: archlinux
  variables:
-    COMPILER: g++
    MODE: optimized
  only:
    variables:
      - $RELEASE == "daily"
-  before_script:
-    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm

 ### Valgrind tests
 # Run the test.py script with files compiled in optimized mode + valgrind (daily)
 daily-test-optimized-valgrind:
  extends: .base-test
-  image: archlinux
  variables:
-    COMPILER: g++
    MODE: optimized
    VALGRIND_FLAG: -g
  only:
@@ -86,16 +76,11 @@ daily-test-optimized-valgrind:
  tags:
    - nsnam
    - linux
-  before_script:
-    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm

 # Run the test.py script with files compiled in debug mode
 weekly-test-debug-valgrind:
  extends: .base-test
-  image: archlinux
  variables:
-    COMPILER: g++
    MODE: debug
    VALGRIND_FLAG: -g
  only:
@@ -104,16 +89,11 @@ weekly-test-debug-valgrind:
  tags:
    - nsnam
    - linux
-  before_script:
-    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm

 # Run the test.py script with files compiled in default mode
 weekly-test-default-valgrind:
  extends: .base-test
-  image: archlinux
  variables:
-    COMPILER: g++
    MODE: default
    VALGRIND_FLAG: -g
  only:
@@ -122,16 +102,11 @@ weekly-test-default-valgrind:
  tags:
    - nsnam
    - linux
-  before_script:
-    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm

 # Run the test.py script with files compiled in optimized mode
 weekly-test-optimized-valgrind:
  extends: .base-test
-  image: archlinux
  variables:
-    COMPILER: g++
    MODE: optimized
    VALGRIND_FLAG: -g
  only:
@@ -140,16 +115,11 @@ weekly-test-optimized-valgrind:
  tags:
    - nsnam
    - linux
-  before_script:
-    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm

 # Do a check for the TAKES_FOREVER jobs, only in optimized mode
 weekly-test-takes-forever-optimized:
  extends: .base-test
-  image: archlinux
  variables:
-    COMPILER: g++
    MODE: optimized
    FULLNESS: "-f TAKES_FOREVER"
  only:
@@ -158,6 +128,3 @@ weekly-test-takes-forever-optimized:
  tags:
    - nsnam
    - linux
-  before_script:
-    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost valgrind ninja cmake --noconfirm
--- a/utils/tests/gitlab-ci-ubuntu.yml
+++ b/utils/tests/gitlab-ci-ubuntu.yml
@@ -14,10 +14,21 @@
      - $RELEASE == "weekly"
      - $UBUNTU == "True"
  before_script:
-    - apt-get --quiet update --yes
-    - apt-get --quiet upgrade --yes
-    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata
-    - apt-get --quiet install gcc g++ cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev gsl-bin libgsl-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev $LIBGSL ssh --yes
+    - apt update
+    - apt upgrade -y
+    - DEBIAN_FRONTEND=noninteractive apt install -y
+      g++ cmake ninja-build ccache
+      python3 python3-dev
+      gsl-bin libgsl-dev $LIBGSL
+      libboost-all-dev
+      libgtk-3-dev
+      libfl-dev
+      libxml2 libxml2-dev
+      libopenmpi-dev openmpi-bin openmpi-common openmpi-doc
+      libsqlite3-dev sqlite3
+      qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools
+      bison flex ssh
+
  variables:
    COMPILER: g++

@@ -26,13 +37,23 @@ weekly-build-ubuntu-18.04-debug:
  extends: .weekly-build-ubuntu
  image: ubuntu:18.04
  before_script:
-    - apt-get --quiet update --yes
-    - apt-get --quiet upgrade --yes
-    - apt-get --quiet install software-properties-common --yes
-    - add-apt-repository ppa:ubuntu-toolchain-r/test --yes
-    - apt-get --quiet update --yes
-    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata
-    - apt-get --quiet install gcc-9 g++-9 cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev gsl-bin libgsl-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev $LIBGSL ssh --yes
+    - apt update
+    - apt upgrade -y
+    - apt install -y software-properties-common
+    - add-apt-repository ppa:ubuntu-toolchain-r/test -y
+    - apt update
+    - DEBIAN_FRONTEND=noninteractive apt install -y
+      g++-9 cmake ninja-build ccache
+      python3 python3-dev
+      gsl-bin libgsl-dev $LIBGSL
+      libboost-all-dev
+      libgtk-3-dev
+      libfl-dev
+      libxml2 libxml2-dev
+      libopenmpi-dev openmpi-bin openmpi-common openmpi-doc
+      libsqlite3-dev sqlite3
+      qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools
+      bison flex ssh
    - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7
    - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 800 --slave /usr/bin/g++ g++ /usr/bin/g++-9
  variables:
@@ -43,12 +64,22 @@ weekly-build-ubuntu-18.04-default:
  extends: .weekly-build-ubuntu
  image: ubuntu:18.04
  before_script:
-    - apt-get --quiet update --yes
-    - apt-get --quiet install software-properties-common --yes
-    - add-apt-repository ppa:ubuntu-toolchain-r/test --yes
-    - apt-get --quiet update --yes
-    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata
-    - apt-get --quiet install gcc-9 g++-9 cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev gsl-bin libgsl-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev $LIBGSL ssh --yes
+    - apt update
+    - apt install -y software-properties-common
+    - add-apt-repository ppa:ubuntu-toolchain-r/test -y
+    - apt update
+    - DEBIAN_FRONTEND=noninteractive apt install -y
+      g++-9 cmake ninja-build ccache
+      python3 python3-dev
+      gsl-bin libgsl-dev $LIBGSL
+      libboost-all-dev
+      libgtk-3-dev
+      libfl-dev
+      libxml2 libxml2-dev
+      libopenmpi-dev openmpi-bin openmpi-common openmpi-doc
+      libsqlite3-dev sqlite3
+      qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools
+      bison flex ssh
    - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7
    - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 800 --slave /usr/bin/g++ g++ /usr/bin/g++-9
  variables:
@@ -59,12 +90,22 @@ weekly-build-ubuntu-18.04-optimized:
  extends: .weekly-build-ubuntu
  image: ubuntu:18.04
  before_script:
-    - apt-get --quiet update --yes
-    - apt-get --quiet install software-properties-common --yes
-    - add-apt-repository ppa:ubuntu-toolchain-r/test --yes
-    - apt-get --quiet update --yes
-    - DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata
-    - apt-get --quiet install gcc-9 g++-9 cmake ninja-build python3 python3-dev qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools openmpi-bin openmpi-common openmpi-doc libopenmpi-dev gsl-bin libgsl-dev ccache flex bison libfl-dev sqlite3 libsqlite3-dev libxml2 libxml2-dev libgtk-3-dev libboost-all-dev $LIBGSL ssh --yes
+    - apt update
+    - apt install -y software-properties-common
+    - add-apt-repository ppa:ubuntu-toolchain-r/test -y
+    - apt update
+    - DEBIAN_FRONTEND=noninteractive apt install -y
+      g++-9 cmake ninja-build ccache
+      python3 python3-dev
+      gsl-bin libgsl-dev $LIBGSL
+      libboost-all-dev
+      libgtk-3-dev
+      libfl-dev
+      libxml2 libxml2-dev
+      libopenmpi-dev openmpi-bin openmpi-common openmpi-doc
+      libsqlite3-dev sqlite3
+      qtbase5-dev qtchooser qt5-qmake qtbase5-dev-tools
+      bison flex ssh
    - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-7 700 --slave /usr/bin/g++ g++ /usr/bin/g++-7
    - update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 800 --slave /usr/bin/g++ g++ /usr/bin/g++-9
  variables:
--- a/utils/tests/gitlab-ci.yml
+++ b/utils/tests/gitlab-ci.yml
@@ -72,7 +72,9 @@ per-commit-compile-debug:
    MODE: debug
  before_script:
    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost ninja cmake --noconfirm
+      base-devel cmake ninja ccache
+      python
+      boost gsl gtk3

 per-commit-compile-default:
  extends: .base-build
@@ -87,7 +89,9 @@ per-commit-compile-default:
    MODE: default
  before_script:
    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost ninja cmake --noconfirm
+      base-devel cmake ninja ccache
+      python
+      boost gsl gtk3

 per-commit-compile-optimized:
  extends: .base-build
@@ -102,7 +106,9 @@ per-commit-compile-optimized:
    MODE: optimized
  before_script:
    - pacman -Syu --noconfirm
-    - pacman -Sy base-devel python ccache gsl gtk3 boost ninja cmake --noconfirm
+      base-devel cmake ninja ccache
+      python
+      boost gsl gtk3

 # Weekly jobs for other distribution and compilers
 include: