From 455cfda2d527feb9b634a35608c9619e6915b698 Mon Sep 17 00:00:00 2001 From: Tommaso Pecorella Date: Tue, 9 May 2023 18:12:54 +0200 Subject: [PATCH] ci: use artifacts to speedup test jobs --- utils/tests/gitlab-ci-alpine.yml | 6 +-- utils/tests/gitlab-ci-clang.yml | 48 +++++++-------------- utils/tests/gitlab-ci-code-linting.yml | 1 + utils/tests/gitlab-ci-cppyy.yml | 1 + utils/tests/gitlab-ci-daily.yml | 36 ++++++++-------- utils/tests/gitlab-ci-doc.yml | 1 + utils/tests/gitlab-ci-fedora.yml | 24 ++++------- utils/tests/gitlab-ci-gcc.yml | 60 +++++++++----------------- utils/tests/gitlab-ci-per-commit.yml | 14 ++---- utils/tests/gitlab-ci-ubuntu.yml | 36 ++++++---------- utils/tests/gitlab-ci.yml | 19 +++++--- 11 files changed, 97 insertions(+), 149 deletions(-) diff --git a/utils/tests/gitlab-ci-alpine.yml b/utils/tests/gitlab-ci-alpine.yml index 7fb14aa77..78341652f 100644 --- a/utils/tests/gitlab-ci-alpine.yml +++ b/utils/tests/gitlab-ci-alpine.yml @@ -47,9 +47,7 @@ weekly-alpine-latest-default-test: image: alpine:latest stage: test needs: ["weekly-alpine-latest-default"] - cache: - key: "ccache-weekly-alpine-latest-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-alpine-latest-default variables: MODE: default diff --git a/utils/tests/gitlab-ci-clang.yml b/utils/tests/gitlab-ci-clang.yml index 686f58f06..19401ee66 100644 --- a/utils/tests/gitlab-ci-clang.yml +++ b/utils/tests/gitlab-ci-clang.yml @@ -96,10 +96,8 @@ weekly-test-clang-6-default: extends: .weekly-clang-ubuntu18.04 stage: test needs: ["weekly-build-clang-6-default"] - cache: - key: "ccache-weekly-build-clang-6-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-clang-6-default variables: CLANG: "6.0" MODE: default @@ -108,10 +106,8 @@ weekly-test-clang-6-optimized: extends: .weekly-clang-ubuntu18.04 stage: test needs: ["weekly-build-clang-6-optimized"] - cache: - key: "ccache-weekly-build-clang-6-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-clang-6-optimized variables: CLANG: "6.0" MODE: optimized @@ -145,10 +141,8 @@ weekly-test-clang-10-default: extends: .weekly-clang-ubuntu18.04 stage: test needs: ["weekly-build-clang-10-default"] - cache: - key: "ccache-weekly-build-clang-10-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-clang-10-default variables: CLANG: 10 MODE: default @@ -157,10 +151,8 @@ weekly-test-clang-10-optimized: extends: .weekly-clang-ubuntu18.04 stage: test needs: ["weekly-build-clang-10-optimized"] - cache: - key: "ccache-weekly-build-clang-10-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-clang-10-optimized variables: CLANG: 10 MODE: optimized @@ -194,10 +186,8 @@ weekly-test-clang-11-default: extends: .weekly-clang-ubuntu22.04 stage: test needs: ["weekly-build-clang-11-default"] - cache: - key: "ccache-weekly-build-clang-11-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-clang-11-default variables: CLANG: 11 MODE: default @@ -206,10 +196,8 @@ weekly-test-clang-11-optimized: extends: .weekly-clang-ubuntu22.04 stage: test needs: ["weekly-build-clang-11-optimized"] - cache: - key: "ccache-weekly-build-clang-11-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-clang-11-optimized variables: CLANG: 11 MODE: optimized @@ -243,10 +231,8 @@ weekly-test-clang-14-default: extends: .weekly-clang-ubuntu22.04 stage: test needs: ["weekly-build-clang-14-default"] - cache: - key: "ccache-weekly-build-clang-14-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-clang-14-default variables: CLANG: 14 MODE: default @@ -255,10 +241,8 @@ weekly-test-clang-14-optimized: extends: .weekly-clang-ubuntu22.04 stage: test needs: ["weekly-build-clang-14-optimized"] - cache: - key: "ccache-weekly-build-clang-14-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-clang-14-optimized variables: CLANG: 14 MODE: optimized diff --git a/utils/tests/gitlab-ci-code-linting.yml b/utils/tests/gitlab-ci-code-linting.yml index d37906ed4..98a106691 100644 --- a/utils/tests/gitlab-ci-code-linting.yml +++ b/utils/tests/gitlab-ci-code-linting.yml @@ -66,6 +66,7 @@ clang-tidy: # Check job results - (! egrep -A 3 "error:|warning:|note:" $CLANG_TIDY_OUTPUT) - echo "No clang-tidy errors found" + dependencies: [] artifacts: paths: - $CLANG_TIDY_OUTPUT diff --git a/utils/tests/gitlab-ci-cppyy.yml b/utils/tests/gitlab-ci-cppyy.yml index b4f8f009f..f363ef3a2 100644 --- a/utils/tests/gitlab-ci-cppyy.yml +++ b/utils/tests/gitlab-ci-cppyy.yml @@ -4,6 +4,7 @@ cppyy-22.04: - if: $RELEASE == "weekly" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' needs: ["weekly-slow-jobs"] + dependencies: [] tags: - nsnam - linux diff --git a/utils/tests/gitlab-ci-daily.yml b/utils/tests/gitlab-ci-daily.yml index 112e6588e..f425c439f 100644 --- a/utils/tests/gitlab-ci-daily.yml +++ b/utils/tests/gitlab-ci-daily.yml @@ -68,6 +68,7 @@ daily-build-test-debug: - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true needs: ["daily-jobs"] + dependencies: [] stage: build variables: MODE: debug @@ -82,8 +83,8 @@ daily-build-default: - if: $RELEASE == "daily" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["daily-jobs"] stage: build + needs: ["daily-jobs"] variables: MODE: default @@ -93,12 +94,10 @@ daily-test-default: - if: $RELEASE == "daily" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["daily-build-default"] stage: test - cache: - key: "ccache-daily-build-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + needs: ["daily-build-default"] + dependencies: + - daily-build-default variables: MODE: default @@ -109,8 +108,8 @@ daily-build-optimized: - if: $RELEASE == "daily" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["daily-jobs"] stage: build + needs: ["daily-jobs"] variables: MODE: optimized @@ -120,12 +119,10 @@ daily-test-optimized: - if: $RELEASE == "daily" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["daily-build-optimized"] stage: test - cache: - key: "ccache-daily-build-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + needs: ["daily-build-optimized"] + dependencies: + - daily-build-optimized variables: MODE: optimized @@ -137,8 +134,9 @@ daily-build-test-optimized-valgrind: - if: $RELEASE == "daily" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["daily-jobs"] stage: build + needs: ["daily-jobs"] + dependencies: [] variables: MODE: optimized VALGRIND_FLAG: -g @@ -154,8 +152,9 @@ weekly-build-test-debug-valgrind: - if: $RELEASE == "weekly" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["weekly-slow-jobs"] stage: build + needs: ["weekly-slow-jobs"] + dependencies: [] variables: MODE: debug VALGRIND_FLAG: -g @@ -171,8 +170,9 @@ weekly-build-test-default-valgrind: - if: $RELEASE == "weekly" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["weekly-slow-jobs"] stage: build + needs: ["weekly-slow-jobs"] + dependencies: [] variables: MODE: default VALGRIND_FLAG: -g @@ -188,8 +188,9 @@ weekly-build-test-optimized-valgrind: - if: $RELEASE == "weekly" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["weekly-slow-jobs"] stage: build + needs: ["weekly-slow-jobs"] + dependencies: [] variables: MODE: optimized VALGRIND_FLAG: -g @@ -205,8 +206,9 @@ weekly-build-test-takes-forever-optimized: - if: $RELEASE == "weekly" - if: $CI_PIPELINE_SOURCE == 'merge_request_event' allow_failure: true - needs: ["weekly-slow-jobs"] stage: build + needs: ["weekly-slow-jobs"] + dependencies: [] variables: MODE: optimized FULLNESS: "-f TAKES_FOREVER" diff --git a/utils/tests/gitlab-ci-doc.yml b/utils/tests/gitlab-ci-doc.yml index fc76b60d6..3005d24e4 100644 --- a/utils/tests/gitlab-ci-doc.yml +++ b/utils/tests/gitlab-ci-doc.yml @@ -18,6 +18,7 @@ image: archlinux timeout: 2h interruptible: true + dependencies: [] # artifacts: # paths: # - public diff --git a/utils/tests/gitlab-ci-fedora.yml b/utils/tests/gitlab-ci-fedora.yml index 8f202fd0b..c46cfb90a 100644 --- a/utils/tests/gitlab-ci-fedora.yml +++ b/utils/tests/gitlab-ci-fedora.yml @@ -76,10 +76,8 @@ weekly-test-fedora-36-default: image: fedora:36 stage: test needs: ["weekly-build-fedora-36-default"] - cache: - key: "ccache-weekly-build-fedora-36-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-fedora-36-default variables: MODE: default @@ -88,10 +86,8 @@ weekly-test-fedora-36-optimized: image: fedora:36 stage: test needs: ["weekly-build-fedora-36-optimized"] - cache: - key: "ccache-weekly-build-fedora-36-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-fedora-36-optimized variables: MODE: optimized @@ -126,10 +122,8 @@ weekly-test-fedora-37-default: image: fedora:37 stage: test needs: ["weekly-build-fedora-37-default"] - cache: - key: "ccache-weekly-build-fedora-37-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-fedora-37-default variables: MODE: default @@ -138,9 +132,7 @@ weekly-test-fedora-37-optimized: image: fedora:37 stage: test needs: ["weekly-build-fedora-37-optimized"] - cache: - key: "ccache-weekly-build-fedora-37-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-fedora-37-optimized variables: MODE: optimized diff --git a/utils/tests/gitlab-ci-gcc.yml b/utils/tests/gitlab-ci-gcc.yml index e441e2ce7..b9c836147 100644 --- a/utils/tests/gitlab-ci-gcc.yml +++ b/utils/tests/gitlab-ci-gcc.yml @@ -80,10 +80,8 @@ weekly-test-gcc-8-default: image: ubuntu:20.04 stage: test needs: ["weekly-build-gcc-8-default"] - cache: - key: "ccache-weekly-build-gcc-8-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-8-default variables: MODE: default COMPILER: g++-8 @@ -94,10 +92,8 @@ weekly-test-gcc-8-optimized: image: ubuntu:20.04 stage: test needs: ["weekly-build-gcc-8-optimized"] - cache: - key: "ccache-weekly-build-gcc-8-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-8-optimized variables: MODE: optimized COMPILER: g++-8 @@ -139,10 +135,8 @@ weekly-test-gcc-9-default: image: ubuntu:20.04 stage: test needs: ["weekly-build-gcc-9-default"] - cache: - key: "ccache-weekly-build-gcc-9-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-9-default variables: MODE: default COMPILER: g++-9 @@ -153,10 +147,8 @@ weekly-test-gcc-9-optimized: image: ubuntu:20.04 stage: test needs: ["weekly-build-gcc-9-optimized"] - cache: - key: "ccache-weekly-build-gcc-9-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-9-optimized variables: MODE: optimized COMPILER: g++-9 @@ -198,10 +190,8 @@ weekly-test-gcc-10-default: image: ubuntu:20.04 stage: test needs: ["weekly-build-gcc-10-default"] - cache: - key: "ccache-weekly-build-gcc-10-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-10-default variables: MODE: default COMPILER: g++-10 @@ -212,10 +202,8 @@ weekly-test-gcc-10-optimized: image: ubuntu:20.04 stage: test needs: ["weekly-build-gcc-10-optimized"] - cache: - key: "ccache-weekly-build-gcc-10-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-10-optimized variables: MODE: optimized COMPILER: g++-10 @@ -257,10 +245,8 @@ weekly-test-gcc-11-default: image: ubuntu:22.04 stage: test needs: ["weekly-build-gcc-11-default"] - cache: - key: "ccache-weekly-build-gcc-11-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-11-default variables: MODE: default COMPILER: g++-11 @@ -271,10 +257,8 @@ weekly-test-gcc-11-optimized: image: ubuntu:22.04 stage: test needs: ["weekly-build-gcc-11-optimized"] - cache: - key: "ccache-weekly-build-gcc-11-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-11-optimized variables: MODE: optimized COMPILER: g++-11 @@ -316,10 +300,8 @@ weekly-test-gcc-12-default: image: ubuntu:22.04 stage: test needs: ["weekly-build-gcc-12-default"] - cache: - key: "ccache-weekly-build-gcc-12-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-12-default variables: MODE: default COMPILER: g++-12 @@ -330,10 +312,8 @@ weekly-test-gcc-12-optimized: image: ubuntu:22.04 stage: test needs: ["weekly-build-gcc-12-optimized"] - cache: - key: "ccache-weekly-build-gcc-12-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-gcc-12-optimized variables: MODE: optimized COMPILER: g++-12 diff --git a/utils/tests/gitlab-ci-per-commit.yml b/utils/tests/gitlab-ci-per-commit.yml index 3c401b15a..cda1cad4c 100644 --- a/utils/tests/gitlab-ci-per-commit.yml +++ b/utils/tests/gitlab-ci-per-commit.yml @@ -52,11 +52,8 @@ per-commit-gcc-default-test: extends: .base-per-commit-compile stage: test needs: ["per-commit-gcc-default"] - cache: - # Reuse the ccache from the per-commit-gcc-default job - key: "ccache-per-commit-gcc-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - per-commit-gcc-default variables: MODE: default COMPILER: g++ @@ -65,11 +62,8 @@ per-commit-gcc-optimized-test: extends: .base-per-commit-compile stage: test needs: ["per-commit-gcc-optimized"] - cache: - # Reuse the ccache from the per-commit-gcc-optimized job - key: "ccache-per-commit-gcc-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - per-commit-gcc-optimized variables: MODE: optimized COMPILER: g++ diff --git a/utils/tests/gitlab-ci-ubuntu.yml b/utils/tests/gitlab-ci-ubuntu.yml index 5fbba3a19..d2e6fad6c 100644 --- a/utils/tests/gitlab-ci-ubuntu.yml +++ b/utils/tests/gitlab-ci-ubuntu.yml @@ -82,10 +82,8 @@ weekly-test-ubuntu-18.04-default: image: ubuntu:18.04 stage: test needs: ["weekly-build-ubuntu-18.04-default"] - cache: - key: "ccache-weekly-build-ubuntu-18.04-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-ubuntu-18.04-default variables: MODE: default LIBGSL: libgsl23 @@ -96,10 +94,8 @@ weekly-test-ubuntu-18.04-optimized: image: ubuntu:18.04 stage: test needs: ["weekly-build-ubuntu-18.04-optimized"] - cache: - key: "ccache-weekly-build-ubuntu-18.04-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-ubuntu-18.04-optimized variables: MODE: optimized LIBGSL: libgsl23 @@ -163,10 +159,8 @@ weekly-test-ubuntu-22.04-default: image: ubuntu:22.04 stage: test needs: ["weekly-build-ubuntu-22.04-default"] - cache: - key: "ccache-weekly-build-ubuntu-22.04-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-ubuntu-22.04-default variables: MODE: default LIBGSL: libgsl27 @@ -177,10 +171,8 @@ weekly-test-ubuntu-22.04-optimized: image: ubuntu:22.04 stage: test needs: ["weekly-build-ubuntu-22.04-optimized"] - cache: - key: "ccache-weekly-build-ubuntu-22.04-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-ubuntu-22.04-optimized variables: MODE: optimized LIBGSL: libgsl27 @@ -222,10 +214,8 @@ weekly-test-ubuntu-latest-default: image: ubuntu:latest stage: test needs: ["weekly-build-ubuntu-latest-default"] - cache: - key: "ccache-weekly-build-ubuntu-latest-default" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-ubuntu-latest-default variables: MODE: default LIBGSL: libgsl27 @@ -236,10 +226,8 @@ weekly-test-ubuntu-latest-optimized: image: ubuntu:latest stage: test needs: ["weekly-build-ubuntu-latest-optimized"] - cache: - key: "ccache-weekly-build-ubuntu-latest-optimized" - paths: - - $CCACHE_BASEDIR_VALUE/ + dependencies: + - weekly-build-ubuntu-latest-optimized variables: MODE: optimized LIBGSL: libgsl27 diff --git a/utils/tests/gitlab-ci.yml b/utils/tests/gitlab-ci.yml index 319ee696e..d222d1678 100644 --- a/utils/tests/gitlab-ci.yml +++ b/utils/tests/gitlab-ci.yml @@ -45,18 +45,20 @@ workflow: - export MPI_CI=1 - CXX=$COMPILER ./ns3 configure -d $MODE -GNinja --enable-examples --enable-tests --enable-asserts $ENABLE_MPI - ccache -z - - ./ns3 build - - if [[ "$CI_JOB_STAGE" == "build" ]] && [[ "`./utils/ccache-miss-rate.py`" != "0" ]]; - then `touch $CCACHE_BASEDIR_VALUE/tests-$CI_COMMIT_REF_SLUG.txt`; + - if [[ "$CI_JOB_STAGE" == "build" ]]; + then ./ns3 build; fi - - if [[ "$CI_JOB_STAGE" == "test" ]] && [[ "$MODE" != "debug" ]] && [[ -f $CCACHE_BASEDIR_VALUE/tests-$CI_COMMIT_REF_SLUG.txt ]]; + - if [[ "$CI_JOB_STAGE" == "build" ]] && [[ "`./utils/ccache-miss-rate.py`" != "0" ]]; + then `touch build/tests-$CI_COMMIT_REF_SLUG.txt`; + fi + - if [[ "$CI_JOB_STAGE" == "test" ]] && [[ "$MODE" != "debug" ]] && [[ -f build/tests-$CI_COMMIT_REF_SLUG.txt ]]; then ./test.py -n; if [[ $? == 0 ]]; - then `rm $CCACHE_BASEDIR_VALUE/tests-$CI_COMMIT_REF_SLUG.txt` || true; + then `rm build/tests-$CI_COMMIT_REF_SLUG.txt` || true; fi; fi - ccache -s - - ./ns3 clean + # - ./ns3 clean cache: # Use separate key for each (debug/default/optimized) jobs because # they run in parallel and will otherwise overwrite each other @@ -72,6 +74,11 @@ workflow: timeout: 12h variables: CCACHE_BASEDIR_VALUE: ns-3-ccache-storage + artifacts: + expire_in: 2 day + paths: + - build/ + - .lock-* # Weekly jobs for other distribution and compilers include: