F5/unison
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ci: use artifacts to speedup test jobs

Browse Source
This commit is contained in:
Tommaso Pecorella
2023-05-09 18:12:54 +02:00
committed by Tommaso Pecorella
parent 679529ccb4
commit 455cfda2d5
11 changed files with 97 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
utils/tests/gitlab-ci-alpine.yml
View File

@@ -47,9 +47,7 @@ weekly-alpine-latest-default-test:
image: alpine:latest
stage: test
needs: ["weekly-alpine-latest-default"]
cache:
key: "ccache-weekly-alpine-latest-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-alpine-latest-default
variables:
MODE: default

48
utils/tests/gitlab-ci-clang.yml
View File

@@ -96,10 +96,8 @@ weekly-test-clang-6-default:
extends: .weekly-clang-ubuntu18.04
stage: test
needs: ["weekly-build-clang-6-default"]
cache:
key: "ccache-weekly-build-clang-6-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-clang-6-default
variables:
CLANG: "6.0"
MODE: default
@@ -108,10 +106,8 @@ weekly-test-clang-6-optimized:
extends: .weekly-clang-ubuntu18.04
stage: test
needs: ["weekly-build-clang-6-optimized"]
cache:
key: "ccache-weekly-build-clang-6-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-clang-6-optimized
variables:
CLANG: "6.0"
MODE: optimized
@@ -145,10 +141,8 @@ weekly-test-clang-10-default:
extends: .weekly-clang-ubuntu18.04
stage: test
needs: ["weekly-build-clang-10-default"]
cache:
key: "ccache-weekly-build-clang-10-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-clang-10-default
variables:
CLANG: 10
MODE: default
@@ -157,10 +151,8 @@ weekly-test-clang-10-optimized:
extends: .weekly-clang-ubuntu18.04
stage: test
needs: ["weekly-build-clang-10-optimized"]
cache:
key: "ccache-weekly-build-clang-10-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-clang-10-optimized
variables:
CLANG: 10
MODE: optimized
@@ -194,10 +186,8 @@ weekly-test-clang-11-default:
extends: .weekly-clang-ubuntu22.04
stage: test
needs: ["weekly-build-clang-11-default"]
cache:
key: "ccache-weekly-build-clang-11-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-clang-11-default
variables:
CLANG: 11
MODE: default
@@ -206,10 +196,8 @@ weekly-test-clang-11-optimized:
extends: .weekly-clang-ubuntu22.04
stage: test
needs: ["weekly-build-clang-11-optimized"]
cache:
key: "ccache-weekly-build-clang-11-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-clang-11-optimized
variables:
CLANG: 11
MODE: optimized
@@ -243,10 +231,8 @@ weekly-test-clang-14-default:
extends: .weekly-clang-ubuntu22.04
stage: test
needs: ["weekly-build-clang-14-default"]
cache:
key: "ccache-weekly-build-clang-14-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-clang-14-default
variables:
CLANG: 14
MODE: default
@@ -255,10 +241,8 @@ weekly-test-clang-14-optimized:
extends: .weekly-clang-ubuntu22.04
stage: test
needs: ["weekly-build-clang-14-optimized"]
cache:
key: "ccache-weekly-build-clang-14-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-clang-14-optimized
variables:
CLANG: 14
MODE: optimized

1
utils/tests/gitlab-ci-code-linting.yml
View File

@@ -66,6 +66,7 @@ clang-tidy:
# Check job results
- (! egrep -A 3 "error:|warning:|note:" $CLANG_TIDY_OUTPUT)
- echo "No clang-tidy errors found"
dependencies: []
artifacts:
paths:
- $CLANG_TIDY_OUTPUT

1
utils/tests/gitlab-ci-cppyy.yml
View File

@@ -4,6 +4,7 @@ cppyy-22.04:
- if: $RELEASE == "weekly"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
needs: ["weekly-slow-jobs"]
dependencies: []
tags:
- nsnam
- linux

36
utils/tests/gitlab-ci-daily.yml
View File

@@ -68,6 +68,7 @@ daily-build-test-debug:
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["daily-jobs"]
dependencies: []
stage: build
variables:
MODE: debug
@@ -82,8 +83,8 @@ daily-build-default:
- if: $RELEASE == "daily"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["daily-jobs"]
stage: build
needs: ["daily-jobs"]
variables:
MODE: default
@@ -93,12 +94,10 @@ daily-test-default:
- if: $RELEASE == "daily"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["daily-build-default"]
stage: test
cache:
key: "ccache-daily-build-default"
paths:
- $CCACHE_BASEDIR_VALUE/
needs: ["daily-build-default"]
dependencies:
- daily-build-default
variables:
MODE: default
@@ -109,8 +108,8 @@ daily-build-optimized:
- if: $RELEASE == "daily"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["daily-jobs"]
stage: build
needs: ["daily-jobs"]
variables:
MODE: optimized
@@ -120,12 +119,10 @@ daily-test-optimized:
- if: $RELEASE == "daily"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["daily-build-optimized"]
stage: test
cache:
key: "ccache-daily-build-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
needs: ["daily-build-optimized"]
dependencies:
- daily-build-optimized
variables:
MODE: optimized
@@ -137,8 +134,9 @@ daily-build-test-optimized-valgrind:
- if: $RELEASE == "daily"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["daily-jobs"]
stage: build
needs: ["daily-jobs"]
dependencies: []
variables:
MODE: optimized
VALGRIND_FLAG: -g
@@ -154,8 +152,9 @@ weekly-build-test-debug-valgrind:
- if: $RELEASE == "weekly"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["weekly-slow-jobs"]
stage: build
needs: ["weekly-slow-jobs"]
dependencies: []
variables:
MODE: debug
VALGRIND_FLAG: -g
@@ -171,8 +170,9 @@ weekly-build-test-default-valgrind:
- if: $RELEASE == "weekly"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["weekly-slow-jobs"]
stage: build
needs: ["weekly-slow-jobs"]
dependencies: []
variables:
MODE: default
VALGRIND_FLAG: -g
@@ -188,8 +188,9 @@ weekly-build-test-optimized-valgrind:
- if: $RELEASE == "weekly"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["weekly-slow-jobs"]
stage: build
needs: ["weekly-slow-jobs"]
dependencies: []
variables:
MODE: optimized
VALGRIND_FLAG: -g
@@ -205,8 +206,9 @@ weekly-build-test-takes-forever-optimized:
- if: $RELEASE == "weekly"
- if: $CI_PIPELINE_SOURCE == 'merge_request_event'
allow_failure: true
needs: ["weekly-slow-jobs"]
stage: build
needs: ["weekly-slow-jobs"]
dependencies: []
variables:
MODE: optimized
FULLNESS: "-f TAKES_FOREVER"

1
utils/tests/gitlab-ci-doc.yml
View File

@@ -18,6 +18,7 @@
image: archlinux
timeout: 2h
interruptible: true
dependencies: []
# artifacts:
# paths:
# - public

24
utils/tests/gitlab-ci-fedora.yml
View File

@@ -76,10 +76,8 @@ weekly-test-fedora-36-default:
image: fedora:36
stage: test
needs: ["weekly-build-fedora-36-default"]
cache:
key: "ccache-weekly-build-fedora-36-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-fedora-36-default
variables:
MODE: default
@@ -88,10 +86,8 @@ weekly-test-fedora-36-optimized:
image: fedora:36
stage: test
needs: ["weekly-build-fedora-36-optimized"]
cache:
key: "ccache-weekly-build-fedora-36-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-fedora-36-optimized
variables:
MODE: optimized
@@ -126,10 +122,8 @@ weekly-test-fedora-37-default:
image: fedora:37
stage: test
needs: ["weekly-build-fedora-37-default"]
cache:
key: "ccache-weekly-build-fedora-37-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-fedora-37-default
variables:
MODE: default
@@ -138,9 +132,7 @@ weekly-test-fedora-37-optimized:
image: fedora:37
stage: test
needs: ["weekly-build-fedora-37-optimized"]
cache:
key: "ccache-weekly-build-fedora-37-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-fedora-37-optimized
variables:
MODE: optimized

60
utils/tests/gitlab-ci-gcc.yml
View File

@@ -80,10 +80,8 @@ weekly-test-gcc-8-default:
image: ubuntu:20.04
stage: test
needs: ["weekly-build-gcc-8-default"]
cache:
key: "ccache-weekly-build-gcc-8-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-8-default
variables:
MODE: default
COMPILER: g++-8
@@ -94,10 +92,8 @@ weekly-test-gcc-8-optimized:
image: ubuntu:20.04
stage: test
needs: ["weekly-build-gcc-8-optimized"]
cache:
key: "ccache-weekly-build-gcc-8-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-8-optimized
variables:
MODE: optimized
COMPILER: g++-8
@@ -139,10 +135,8 @@ weekly-test-gcc-9-default:
image: ubuntu:20.04
stage: test
needs: ["weekly-build-gcc-9-default"]
cache:
key: "ccache-weekly-build-gcc-9-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-9-default
variables:
MODE: default
COMPILER: g++-9
@@ -153,10 +147,8 @@ weekly-test-gcc-9-optimized:
image: ubuntu:20.04
stage: test
needs: ["weekly-build-gcc-9-optimized"]
cache:
key: "ccache-weekly-build-gcc-9-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-9-optimized
variables:
MODE: optimized
COMPILER: g++-9
@@ -198,10 +190,8 @@ weekly-test-gcc-10-default:
image: ubuntu:20.04
stage: test
needs: ["weekly-build-gcc-10-default"]
cache:
key: "ccache-weekly-build-gcc-10-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-10-default
variables:
MODE: default
COMPILER: g++-10
@@ -212,10 +202,8 @@ weekly-test-gcc-10-optimized:
image: ubuntu:20.04
stage: test
needs: ["weekly-build-gcc-10-optimized"]
cache:
key: "ccache-weekly-build-gcc-10-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-10-optimized
variables:
MODE: optimized
COMPILER: g++-10
@@ -257,10 +245,8 @@ weekly-test-gcc-11-default:
image: ubuntu:22.04
stage: test
needs: ["weekly-build-gcc-11-default"]
cache:
key: "ccache-weekly-build-gcc-11-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-11-default
variables:
MODE: default
COMPILER: g++-11
@@ -271,10 +257,8 @@ weekly-test-gcc-11-optimized:
image: ubuntu:22.04
stage: test
needs: ["weekly-build-gcc-11-optimized"]
cache:
key: "ccache-weekly-build-gcc-11-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-11-optimized
variables:
MODE: optimized
COMPILER: g++-11
@@ -316,10 +300,8 @@ weekly-test-gcc-12-default:
image: ubuntu:22.04
stage: test
needs: ["weekly-build-gcc-12-default"]
cache:
key: "ccache-weekly-build-gcc-12-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-12-default
variables:
MODE: default
COMPILER: g++-12
@@ -330,10 +312,8 @@ weekly-test-gcc-12-optimized:
image: ubuntu:22.04
stage: test
needs: ["weekly-build-gcc-12-optimized"]
cache:
key: "ccache-weekly-build-gcc-12-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-gcc-12-optimized
variables:
MODE: optimized
COMPILER: g++-12

14
utils/tests/gitlab-ci-per-commit.yml
View File

@@ -52,11 +52,8 @@ per-commit-gcc-default-test:
extends: .base-per-commit-compile
stage: test
needs: ["per-commit-gcc-default"]
cache:
# Reuse the ccache from the per-commit-gcc-default job
key: "ccache-per-commit-gcc-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- per-commit-gcc-default
variables:
MODE: default
COMPILER: g++
@@ -65,11 +62,8 @@ per-commit-gcc-optimized-test:
extends: .base-per-commit-compile
stage: test
needs: ["per-commit-gcc-optimized"]
cache:
# Reuse the ccache from the per-commit-gcc-optimized job
key: "ccache-per-commit-gcc-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- per-commit-gcc-optimized
variables:
MODE: optimized
COMPILER: g++

36
utils/tests/gitlab-ci-ubuntu.yml
View File

@@ -82,10 +82,8 @@ weekly-test-ubuntu-18.04-default:
image: ubuntu:18.04
stage: test
needs: ["weekly-build-ubuntu-18.04-default"]
cache:
key: "ccache-weekly-build-ubuntu-18.04-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-ubuntu-18.04-default
variables:
MODE: default
LIBGSL: libgsl23
@@ -96,10 +94,8 @@ weekly-test-ubuntu-18.04-optimized:
image: ubuntu:18.04
stage: test
needs: ["weekly-build-ubuntu-18.04-optimized"]
cache:
key: "ccache-weekly-build-ubuntu-18.04-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-ubuntu-18.04-optimized
variables:
MODE: optimized
LIBGSL: libgsl23
@@ -163,10 +159,8 @@ weekly-test-ubuntu-22.04-default:
image: ubuntu:22.04
stage: test
needs: ["weekly-build-ubuntu-22.04-default"]
cache:
key: "ccache-weekly-build-ubuntu-22.04-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-ubuntu-22.04-default
variables:
MODE: default
LIBGSL: libgsl27
@@ -177,10 +171,8 @@ weekly-test-ubuntu-22.04-optimized:
image: ubuntu:22.04
stage: test
needs: ["weekly-build-ubuntu-22.04-optimized"]
cache:
key: "ccache-weekly-build-ubuntu-22.04-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-ubuntu-22.04-optimized
variables:
MODE: optimized
LIBGSL: libgsl27
@@ -222,10 +214,8 @@ weekly-test-ubuntu-latest-default:
image: ubuntu:latest
stage: test
needs: ["weekly-build-ubuntu-latest-default"]
cache:
key: "ccache-weekly-build-ubuntu-latest-default"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-ubuntu-latest-default
variables:
MODE: default
LIBGSL: libgsl27
@@ -236,10 +226,8 @@ weekly-test-ubuntu-latest-optimized:
image: ubuntu:latest
stage: test
needs: ["weekly-build-ubuntu-latest-optimized"]
cache:
key: "ccache-weekly-build-ubuntu-latest-optimized"
paths:
- $CCACHE_BASEDIR_VALUE/
dependencies:
- weekly-build-ubuntu-latest-optimized
variables:
MODE: optimized
LIBGSL: libgsl27

19
utils/tests/gitlab-ci.yml
View File

@@ -45,18 +45,20 @@ workflow:
- export MPI_CI=1
- CXX=$COMPILER ./ns3 configure -d $MODE -GNinja --enable-examples --enable-tests --enable-asserts $ENABLE_MPI
- ccache -z
- ./ns3 build
- if [[ "$CI_JOB_STAGE" == "build" ]] && [[ "`./utils/ccache-miss-rate.py`" != "0" ]];
then `touch $CCACHE_BASEDIR_VALUE/tests-$CI_COMMIT_REF_SLUG.txt`;
- if [[ "$CI_JOB_STAGE" == "build" ]];
then ./ns3 build;
fi
- if [[ "$CI_JOB_STAGE" == "test" ]] && [[ "$MODE" != "debug" ]] && [[ -f $CCACHE_BASEDIR_VALUE/tests-$CI_COMMIT_REF_SLUG.txt ]];
- if [[ "$CI_JOB_STAGE" == "build" ]] && [[ "`./utils/ccache-miss-rate.py`" != "0" ]];
then `touch build/tests-$CI_COMMIT_REF_SLUG.txt`;
fi
- if [[ "$CI_JOB_STAGE" == "test" ]] && [[ "$MODE" != "debug" ]] && [[ -f build/tests-$CI_COMMIT_REF_SLUG.txt ]];
then ./test.py -n;
if [[ $? == 0 ]];
then `rm $CCACHE_BASEDIR_VALUE/tests-$CI_COMMIT_REF_SLUG.txt` || true;
then `rm build/tests-$CI_COMMIT_REF_SLUG.txt` || true;
fi;
fi
- ccache -s
- ./ns3 clean
# - ./ns3 clean
cache:
# Use separate key for each (debug/default/optimized) jobs because
# they run in parallel and will otherwise overwrite each other
@@ -72,6 +74,11 @@ workflow:
timeout: 12h
variables:
CCACHE_BASEDIR_VALUE: ns-3-ccache-storage
artifacts:
expire_in: 2 day
paths:
- build/
- .lock-*
# Weekly jobs for other distribution and compilers
include: