Merge "Update protoc to v3.7.0 for voltha-protos"
diff --git a/jjb/bbsim-validation.yaml b/jjb/bbsim-validation.yaml
index 7fd5c19..831ba42 100644
--- a/jjb/bbsim-validation.yaml
+++ b/jjb/bbsim-validation.yaml
@@ -31,7 +31,7 @@
parameters:
- string:
name: executorNode
- default: 'ubuntu16.04-basebuild-4c-8g'
+ default: 'qct-pod2-node1'
description: 'Name of the Jenkins node to run the job on'
- string:
@@ -54,6 +54,16 @@
default: '16'
description: 'Number of ONUs per pon port'
+ - string:
+ name: EmulationMode
+ default: '--set emulation_mode=default'
+ description: 'Emulation for BBSIM (both|aaa|default)'
+
+ - string:
+ name: TestTimeout
+ default: '300s'
+ description: 'timeout for each test case (increase as you scale up)'
+
project-type: pipeline
concurrent: false
diff --git a/jjb/defaults.yaml b/jjb/defaults.yaml
index 4863e0e..b1f4948 100644
--- a/jjb/defaults.yaml
+++ b/jjb/defaults.yaml
@@ -78,7 +78,7 @@
# matching repos that should be version tagged by the version-tag job
# (basically the same as imagebuilder projects + helm charts + tools
- version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|tt-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter|pyvoltha|voltha-adtran-adapter|voltha-openolt-adapter|voltha-openonu-adapter|plyxproto|voltha-protos)$'
+ version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|tt-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter|pyvoltha|voltha-adtran-adapter|voltha-openolt-adapter|voltha-openonu-adapter|plyxproto|voltha-protos|alpine-grpc-base)$'
# PyPI related publishing variables
#
@@ -92,6 +92,11 @@
# in the job once testing is completed
pypi-index: 'testpypi'
+ # Name of the Docker repository (usually on DockerHub) that is the
+ # destination for images to be pushed to after building.
+ # Default is invalid - must be specified on every job.
+ docker-repo: 'default-and-invalid'
+
# for matching files with file-include-regexp
all-files-regexp: '.*'
doc-files-regexp: '^docs/.*'
diff --git a/jjb/docker-publish.yaml b/jjb/docker-publish.yaml
new file mode 100644
index 0000000..9495c04
--- /dev/null
+++ b/jjb/docker-publish.yaml
@@ -0,0 +1,53 @@
+---
+# generic docker image building + dockerhub publishing tasks
+
+- job-template:
+ id: docker-publish
+ name: 'docker-publish_{project}'
+ description: |
+ Created by {id} job-template from ci-management/jjb/docker-publish.yaml
+
+ triggers:
+ - cord-infra-gerrit-trigger-merge:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: '^{project}$'
+ branch-regexp: '{all-branches-regexp}'
+ file-include-regexp: '{all-files-regexp}'
+ dependency-jobs: '{dependency-jobs}'
+
+ properties:
+ - cord-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: '30'
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ parameters:
+ - string:
+ name: executorNode
+ default: 'ubuntu16.04-basebuild-1c-2g'
+ description: 'Name of the Jenkins node to run the job on'
+
+ - string:
+ name: gitUrl
+ default: '{gerrit-server-url}/{project}'
+ description: 'URL to the git repo'
+
+ - string:
+ name: dockerRepo
+ default: '{docker-repo}'
+ description: "Docker repository to push to ('opencord', 'xosproject', etc.)"
+
+ # AWS CPU arch names: `x86_64` `arm64` (which don't align to vendor names... *sigh*)
+ - string:
+ name: dockerArchList
+ default: 'x86_64'
+ description: "List of architectures to build containers on, pipe separated (nonfunctional currently)"
+
+ project-type: pipeline
+ concurrent: true
+
+ dsl: !include-raw-escape: pipeline/docker-publish.groovy
diff --git a/jjb/pipeline/bbsim-validation.groovy b/jjb/pipeline/bbsim-validation.groovy
index 6da8132..f9a079c 100644
--- a/jjb/pipeline/bbsim-validation.groovy
+++ b/jjb/pipeline/bbsim-validation.groovy
@@ -42,40 +42,18 @@
}
}
- stage('minikube') {
+ stage('Clean up') {
steps {
- /* see https://github.com/kubernetes/minikube/#linux-continuous-integration-without-vm-support */
- sh '''
- export MINIKUBE_WANTUPDATENOTIFICATION=false
- export MINIKUBE_WANTREPORTERRORPROMPT=false
- export CHANGE_MINIKUBE_NONE_USER=true
- export MINIKUBE_HOME=$HOME
- mkdir -p $HOME/.kube || true
- touch $HOME/.kube/config
- export KUBECONFIG=$HOME/.kube/config
- sudo -E /usr/bin/minikube start --vm-driver=none
- '''
- script {
- timeout(3) {
- waitUntil {
- sleep 5
- def kc_ret = sh script: "kubectl get po", returnStatus: true
- return (kc_ret == 0);
- }
- }
- }
- }
- }
-
- stage('helm') {
- steps {
- sh '''
- helm init
- sleep 60
- helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
- helm repo add cord https://charts.opencord.org
- helm repo update
- '''
+ sh """
+ rm -rf voltha-bbsim/
+ rm -rf pod-configs/
+ rm -rf cord/helm-charts/helm-repo-tools/
+ for hchart in \$(helm list -q | grep -E -v 'docker-registry|mavenrepo|ponnet');
+ do
+ echo "Purging chart: \${hchart}"
+ helm delete --purge "\${hchart}"
+ done
+ """
}
}
@@ -84,7 +62,6 @@
sh '''
git clone https://github.com/opencord/voltha-bbsim
cd voltha-bbsim/
- make docker
docker images | grep bbsim
'''
}
@@ -105,7 +82,7 @@
helm-repo-tools/wait_for_pods.sh
helm upgrade --install etcd-operator --version 0.8.3 stable/etcd-operator
- sleep 120
+ sleep 60
JOBS_TIMEOUT=900 ./helm-repo-tools/wait_for_jobs.sh
helm dep up voltha
@@ -136,7 +113,7 @@
kubectl get pods
helm list
- helm install --set images.bbsim.tag=latest --set images.bbsim.pullPolicy=IfNotPresent --set onus_per_pon_port=${params.OnuCount} bbsim -n bbsim
+ helm install --set images.bbsim.tag=latest --set images.bbsim.pullPolicy=IfNotPresent --set onus_per_pon_port=${params.OnuCount} ${params.EmulationMode} bbsim -n bbsim
for hchart in \$(helm list -q);
do
echo "## 'helm status' for chart: \${hchart} ##"
@@ -162,8 +139,8 @@
sh """
#!/usr/bin/env bash
set -eu -o pipefail
- pushd cord/test/cord-tester/src/test/cord-api/Tests/BBSim/
- robot -e notready -v number_of_onus:${params.OnuCount} BBSIMScale.robot || true
+ pushd cord/test/cord-tester/src/test/cord-api/Tests/BBSim
+ robot -e serviceinstances -e onosdhcp -e notready -v number_of_onus:${params.OnuCount} -v timeout:${params.TestTimeout} BBSIMScale.robot || true
"""
}
}
@@ -200,14 +177,6 @@
echo "# removing helm deployments"
kubectl get pods
helm list
-
- for hchart in \$(helm list -q);
- do
- echo "## Purging chart: \${hchart} ##"
- helm delete --purge "\${hchart}"
- done
-
- sudo minikube delete
'''
step([$class: 'RobotPublisher',
disableArchiveOutput: false,
diff --git a/jjb/pipeline/docker-publish.groovy b/jjb/pipeline/docker-publish.groovy
new file mode 100644
index 0000000..89eef34
--- /dev/null
+++ b/jjb/pipeline/docker-publish.groovy
@@ -0,0 +1,97 @@
+/* docker-publish pipeline */
+pipeline {
+
+ /* no label, executor is determined by JJB */
+ agent {
+ label "${params.executorNode}"
+ }
+ stages {
+
+ stage('checkout') {
+ steps {
+ checkout([
+ $class: 'GitSCM',
+ userRemoteConfigs: [[
+ url: "${params.gitUrl}",
+ name: "${params.GERRIT_PATCHSET_REVISION}",
+ ]],
+ extensions: [
+ [$class: 'WipeWorkspace'],
+ [$class: 'RelativeTargetDirectory', relativeTargetDir: "${params.GERRIT_PROJECT}"],
+ [$class: 'CloneOption', depth: 0, noTags: false, reference: '', shallow: false],
+ ],
+ ])
+ script {
+ def git_tags={ shell "git tag -l --points-at HEAD" }
+ }
+ }
+ }
+
+ stage('build'){
+ steps {
+ sh """
+ #!/usr/bin/env bash
+ set -eu -o pipefail
+
+ # checked out in a subdir so the log can be in WORKSPACE
+ cd "$GERRIT_PROJECT"
+
+ export DOCKER_REPOSITORY="$dockerRepo/"
+
+ # Build w/branch
+ echo "Building image with branch"
+ make DOCKER_TAG="$GERRIT_BRANCH" docker-build 2>&1 | tee > "$WORKSPACE/docker-build.log"
+
+ # Build w/tags if they exist
+ if [ -n "$git_tags" ]
+ then
+ for tag in $git_tags
+ do
+ echo "Building image with tag (should reuse cached layers)"
+ make DOCKER_TAG="$tag" docker-build
+ done
+ fi
+ """
+ }
+ }
+
+ stage('push'){
+ steps {
+ script {
+ withDockerRegistry([credentialsId: 'docker-artifact-push-credentials']) {
+ sh """
+ #!/usr/bin/env bash
+ set -eu -o pipefail
+
+ # checked out in a subdir so the log can be in WORKSPACE
+ cd "$GERRIT_PROJECT"
+
+ export DOCKER_REPOSITORY="$dockerRepo/"
+
+ # Push w/branch
+ echo "Pushing image with branch"
+ make DOCKER_TAG="$GERRIT_BRANCH" docker-push 2>&1 | tee > "$WORKSPACE/docker-push.log"
+
+ # Push w/tags if they exist
+ if [ -n "$git_tags" ]
+ then
+ for tag in $git_tags
+ do
+ echo "Pushing image with tag (should reuse cached layers)"
+ make DOCKER_TAG="$tag" docker-push
+ done
+ fi
+ """
+ }
+ }
+ }
+ }
+ }
+
+ post {
+ always {
+ archiveArtifacts artifacts: 'docker-build.log', fingerprint: true
+ deleteDir()
+ }
+ }
+}
diff --git a/jjb/pipeline/voltha-publish.groovy b/jjb/pipeline/voltha-publish.groovy
index ecd6f25..a07d4f2 100644
--- a/jjb/pipeline/voltha-publish.groovy
+++ b/jjb/pipeline/voltha-publish.groovy
@@ -32,8 +32,12 @@
#!/usr/bin/env bash
pushd cord/incubator/voltha
- VOLTHA_BUILD=docker DOCKER_CACHE_ARG=--no-cache TAG=${params.manifestBranch} make build
-
+ if [ "${params.manifestBranch}" != "master" ]
+ then
+ VOLTHA_BUILD=docker DOCKER_CACHE_ARG=--no-cache TAG=${params.manifestBranch} make build
+ else
+ VOLTHA_BUILD=docker DOCKER_CACHE_ARG=--no-cache make build
+ fi
popd
"""
}
@@ -46,8 +50,12 @@
#!/usr/bin/env bash
pushd cord/incubator/voltha
- VOLTHA_BUILD=docker TAG=${params.manifestBranch} TARGET_REPOSITORY=voltha/ TARGET_TAG=${params.manifestBranch} make push
-
+ if [ "${params.manifestBranch}" != "master" ]
+ then
+ VOLTHA_BUILD=docker TAG=${params.manifestBranch} TARGET_REPOSITORY=voltha/ TARGET_TAG=${params.manifestBranch} make push
+ else
+ VOLTHA_BUILD=docker TARGET_REPOSITORY=voltha/ make push
+ fi
popd
"""
}
diff --git a/jjb/verify/alpine-grpc-base.yaml b/jjb/verify/alpine-grpc-base.yaml
index 0c823cc..7197480 100644
--- a/jjb/verify/alpine-grpc-base.yaml
+++ b/jjb/verify/alpine-grpc-base.yaml
@@ -8,6 +8,7 @@
jobs:
- 'verify-alpine-grpc-base-jobs':
branch-regexp: '{all-branches-regexp}'
+ - 'publish-alpine-grpc-base-jobs'
- job-group:
name: 'verify-alpine-grpc-base-jobs'
@@ -15,3 +16,9 @@
- 'verify-licensed'
- 'tag-collision-reject':
dependency-jobs: 'verify_alpine-grpc-base_licensed'
+
+- job-group:
+ name: 'publish-alpine-grpc-base-jobs'
+ jobs:
+ - 'docker-publish':
+ docker-repo: "xosproject"
diff --git a/jjb/voltha-publish.yaml b/jjb/voltha-publish.yaml
index 8ec5c41..0e03be5 100644
--- a/jjb/voltha-publish.yaml
+++ b/jjb/voltha-publish.yaml
@@ -8,7 +8,7 @@
jobs:
- 'voltha-publish':
project-regexp: 'voltha'
- branch-regexp: 'voltha-1.6'
+ branch-regexp: '^(master|voltha-1.6)$'
executor-node: 'imagebuilder'
dependency-jobs: ''