Merge "job for new Flex OCP QA POD"
diff --git a/jjb/defaults.yaml b/jjb/defaults.yaml
index 4ec37b7..de250ce 100644
--- a/jjb/defaults.yaml
+++ b/jjb/defaults.yaml
@@ -77,7 +77,7 @@
# matching repos that should be version tagged by the version-tag job
# (basically the same as imagebuilder projects + helm charts + tools
- version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter)$'
+ version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter|pyvoltha|voltha-adtran-adapter|voltha-openolt-adapter|voltha-openonu-adapter)$'
# for matching files with file-include-regexp
all-files-regexp: '.*'
diff --git a/jjb/pipeline/bbsim-validation.groovy b/jjb/pipeline/bbsim-validation.groovy
index 14392c9..1cca12a 100644
--- a/jjb/pipeline/bbsim-validation.groovy
+++ b/jjb/pipeline/bbsim-validation.groovy
@@ -117,17 +117,17 @@
JOBS_TIMEOUT=900 ./helm-repo-tools/wait_for_pods.sh
helm dep up xos-core
- helm install xos-core -n xos-core -f $WORKSPACE/pod-configs/kubernetes-configs/${params.deploymentConfig}
+ helm install xos-core -n xos-core
helm dep update xos-profiles/seba-services
- helm install xos-profiles/seba-services -n seba-services -f $WORKSPACE/pod-configs/kubernetes-configs/${params.deploymentConfig}
+ helm install xos-profiles/seba-services -n seba-services
JOBS_TIMEOUT=900 ./helm-repo-tools/wait_for_pods.sh
- helm dep update xos-profiles/base-kubernetes
- helm install xos-profiles/base-kubernetes -n base-kubernetes -f $WORKSPACE/pod-configs/kubernetes-configs/${params.deploymentConfig}
+ helm dep update xos-profiles/base-kubernetes
+ helm install xos-profiles/base-kubernetes -n base-kubernetes
helm dep update workflows/att-workflow
- helm install workflows/att-workflow -n att-workflow -f $WORKSPACE/pod-configs/kubernetes-configs/${params.deploymentConfig}
+ helm install workflows/att-workflow -n att-workflow --set att-workflow-driver.kafkaService=cord-kafka
# wait for services to load
JOBS_TIMEOUT=900 ./helm-repo-tools/wait_for_jobs.sh
@@ -181,8 +181,6 @@
kubectl logs bbsim-api-test --namespace default
kubectl get pods --all-namespaces
- curl http://127.0.0.1:30125/api/v1/devices || true
-
# copy robot logs
if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs
cp -r /tmp/helm_test_bbsim_logs_*/*ml ./RobotLogs
diff --git a/jjb/pipeline/siab-test.groovy b/jjb/pipeline/siab-test.groovy
new file mode 100644
index 0000000..4a5cac0
--- /dev/null
+++ b/jjb/pipeline/siab-test.groovy
@@ -0,0 +1,128 @@
+/* seba-in-a-box build+test */
+
+pipeline {
+
+ /* no label, executor is determined by JJB */
+ agent {
+ label "${params.executorNode}"
+ }
+
+ stages {
+
+ stage ("Clean workspace") {
+ steps {
+ sh 'cd $WORKSPACE; rm -rf *'
+ }
+ }
+
+ stage('repo') {
+ steps {
+ checkout(changelog: false, \
+ poll: false,
+ scm: [$class: 'RepoScm', \
+ manifestRepositoryUrl: "https://gerrit.opencord.org/manifest.git", \
+ manifestBranch: "master", \
+ currentBranch: true, \
+ destinationDir: 'cord', \
+ forceSync: true,
+ resetFirst: true, \
+ quiet: true, \
+ jobs: 4, \
+ showAllChanges: true] \
+ )
+ }
+ }
+
+ stage('patch') {
+ steps {
+ sh """
+ pushd $WORKSPACE/cord
+ repo download helm-charts "${gerritChangeNumber}/${gerritPatchsetNumber}"
+ popd
+ """
+ }
+ }
+
+
+ stage ('Reset Kubeadm') {
+ steps {
+ sh """
+ pushd $WORKSPACE/cord/automation-tools/seba-in-a-box
+ make reset-kubeadm
+ popd
+ """
+ }
+ }
+
+ stage ('Install SEBA') {
+ steps {
+ sh """
+ pushd $WORKSPACE/cord/automation-tools/seba-in-a-box
+ make
+ popd
+ """
+ }
+ }
+
+ stage ('Run E2E Tests') {
+ steps {
+ sh """
+ pushd $WORKSPACE/cord/automation-tools/seba-in-a-box
+ make run-tests ${params.Test_Tags} || true
+ popd
+ """
+ }
+ }
+
+ stage ('Display Kafka Events') {
+ steps {
+ sh """
+ pushd $WORKSPACE/cord/automation-tools/seba-in-a-box
+ CORD_KAFKA_IP=\$(kubectl exec cord-kafka-0 -- ip a | grep -oE "([0-9]{1,3}\\.){3}[0-9]{1,3}\\b" | grep 192)
+ kafkacat -e -C -b \$CORD_KAFKA_IP -t onu.events -f 'Topic %t [%p] at offset %o: key %k: %s\n >0'
+ kafkacat -e -C -b \$CORD_KAFKA_IP -t authentication.events -f 'Topic %t [%p] at offset %o: key %k: %s\n >0'
+ kafkacat -e -C -b \$CORD_KAFKA_IP -t dhcp.events -f 'Topic %t [%p] at offset %o: key %k: %s\n >0'
+ popd
+ """
+ }
+ }
+
+ stage('Publish') {
+ steps {
+ sh """
+ if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs
+ cp -r $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/WorkflowValidations/*ml ./RobotLogs
+ """
+ step([$class: 'RobotPublisher',
+ disableArchiveOutput: false,
+ logFileName: 'RobotLogs/log*.html',
+ otherFiles: '',
+ outputFileName: 'RobotLogs/output*.xml',
+ outputPath: '.',
+ passThreshold: 100,
+ reportFileName: 'RobotLogs/report*.html',
+ unstableThreshold: 0]);
+ }
+ }
+
+ }
+
+ post {
+ always {
+ sh '''
+ sudo cp /var/log/containers/*.log $WORKSPACE/
+ sudo chown cord:cord $WORKSPACE/*log
+ '''
+ archiveArtifacts artifacts: '*.log'
+ step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "andy@opennetworking.org, kailash@opennetworking.org", sendToIndividuals: false])
+ }
+ failure {
+ sh '''
+ curl -X GET -u karaf:karaf http://127.0.0.1:30120/onos/v1/devices
+ curl -X GET -u karaf:karaf http://127.0.0.1:30120/onos/v1/devices/of:0000000000000001/ports
+ curl -X GET http://127.0.0.1:30125/api/v1/devices
+ curl -X GET http://127.0.0.1:30125/api/v1/logical_devices
+ '''
+ }
+ }
+}
diff --git a/jjb/pipeline/workflow-validations.groovy b/jjb/pipeline/workflow-validations.groovy
index 1db4652..850ae8d 100644
--- a/jjb/pipeline/workflow-validations.groovy
+++ b/jjb/pipeline/workflow-validations.groovy
@@ -78,7 +78,7 @@
}
- stage('install/test att-workflow') {
+ stage('install profile') {
steps {
sh """
#!/usr/bin/env bash
@@ -95,10 +95,10 @@
helm install xos-core -n xos-core
helm dep update xos-profiles/seba-services
- helm install \${helm_install_args} xos-profiles/seba-services
+ helm install xos-profiles/seba-services
JOBS_TIMEOUT=900 ./helm-repo-tools/wait_for_jobs.sh
helm dep update workflows/att-workflow
- helm install workflows/att-workflow -n att-workflow
+ helm install workflows/att-workflow --set att-workflow-driver.kafkaService=cord-kafka -n att-workflow
helm repo add incubator http://storage.googleapis.com/kubernetes-charts-incubator
@@ -114,7 +114,27 @@
echo "## 'helm status' for chart: \${hchart} ##"
helm status "\${hchart}"
done
- helm test --timeout 1000 att-workflow
+ popd
+
+ """
+ }
+ }
+ stage('test data-model scale') {
+ steps {
+ sh """
+ #!/usr/bin/env bash
+ set -eu -o pipefail
+ CORD_KAFKA_IP=\$(kubectl exec cord-kafka-0 -- ip a | grep -oE "([0-9]{1,3}\\.){3}[0-9]{1,3}\\b" | grep 172)
+ pushd cord/test/cord-tester/src/test/cord-api/
+ source setup_venv.sh
+ cd Tests/XosScaleValidations
+ robot --variable xos_chameleon_url:127.0.0.1 \
+ --variable xos_chameleon_port:30006 \
+ --variable cord_kafka:\$CORD_KAFKA_IP \
+ --variable num_olts:${params.Olts} \
+ --variable num_onus:${params.Onus} \
+ --variable num_pon_ports:${params.PonPorts} \
+ xos-scale-att-workflow.robot || true
popd
"""
@@ -124,14 +144,11 @@
post {
always {
sh '''
-
- kubectl logs att-workflow-api-test --namespace default
kubectl get pods --all-namespaces
# copy robot logs
if [ -d RobotLogs ]; then rm -r RobotLogs; fi; mkdir RobotLogs
- cp -r /tmp/helm_test_attworkflow_logs_*/*ml ./RobotLogs
-
+ cp -r $WORKSPACE/cord/test/cord-tester/src/test/cord-api/Tests/XosScaleValidations/*ml ./RobotLogs
echo "# removing helm deployments"
kubectl get pods
helm list
diff --git a/jjb/shell/ansiblelint.sh b/jjb/shell/ansiblelint.sh
index f97d186..930c09f 100755
--- a/jjb/shell/ansiblelint.sh
+++ b/jjb/shell/ansiblelint.sh
@@ -30,7 +30,9 @@
while IFS= read -r -d '' yf
do
echo "==> CHECKING: ${yf}"
- ansible-lint -p "${yf}"
+ # Ignore line length limit (E204) as it can't (as of v4.0.1) be
+ # overridden with the skip_ansible_lint tag
+ ansible-lint -x 204 -p "${yf}"
rc=$?
if [[ $rc != 0 ]]; then
echo "==> LINTING FAIL: ${yf}"
diff --git a/jjb/shell/licensecheck.sh b/jjb/shell/licensecheck.sh
index f34f78e..977495c 100755
--- a/jjb/shell/licensecheck.sh
+++ b/jjb/shell/licensecheck.sh
@@ -90,6 +90,8 @@
! -name "*.xtarget" \
! -name "*ignore" \
! -name "*rc" \
+ ! -name "*_pb2.py" \
+ ! -name "*_pb2_grpc.py" \
! -name "Dockerfile" \
! -name "Dockerfile.*" \
! -name "Makefile" \
diff --git a/jjb/siab-e2e.yaml b/jjb/siab-e2e.yaml
index 1549e08..b5b1935 100644
--- a/jjb/siab-e2e.yaml
+++ b/jjb/siab-e2e.yaml
@@ -52,7 +52,7 @@
cd cord/automation-tools/seba-in-a-box
[ -e /usr/bin/kubeadm ] && make reset-kubeadm
- make && make run-tests TESTTAGS=test1
+ make && make run-tests
publishers:
- email:
diff --git a/jjb/siab.yaml b/jjb/siab.yaml
index e3baf22..2a87e90 100644
--- a/jjb/siab.yaml
+++ b/jjb/siab.yaml
@@ -9,6 +9,7 @@
jobs:
- 'siab-stable'
- 'siab-latest'
+ - 'siab-test'
- job-template:
id: 'siab-stable'
@@ -100,3 +101,58 @@
TZ=America/Los_Angeles
59 */2 * * *
+
+- job-template:
+ id: 'siab-test'
+ name: 'seba-in-a-box-build-test'
+
+ description: |
+ <!-- Managed by Jenkins Job Builder -->
+ Created by {id} job-template from ci-management/jjb/siba.yaml <br /><br />
+ E2E Validation for Seba-in-a-Box
+
+ properties:
+ - cord-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - lf-infra-wrappers:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ parameters:
+ - string:
+ name: executorNode
+ default: 'qct-pod1-node2'
+ description: 'Name of the Jenkins node to run the job on'
+
+
+ - string:
+ name: gerritChangeNumber
+ default: '$GERRIT_CHANGE_NUMBER'
+ description: 'Changeset number in Gerrit'
+
+ - string:
+ name: gerritPatchsetNumber
+ default: '$GERRIT_PATCHSET_NUMBER'
+ description: 'PatchSet number in Gerrit'
+
+ - string:
+ name: Test_Tags
+ default: 'TESTTAGS=test1'
+ description: ''
+
+ project-type: pipeline
+ concurrent: false
+
+ dsl: !include-raw-escape: pipeline/siab-test.groovy
+
+ triggers:
+ - cord-infra-gerrit-trigger-patchset:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: 'helm-charts'
+ branch-regexp: 'master'
+ dependency-jobs: '{dependency-jobs}'
+ file-include-regexp: '{all-files-regexp}'
+
diff --git a/jjb/verify/pyvoltha.yaml b/jjb/verify/pyvoltha.yaml
new file mode 100644
index 0000000..6cfc474
--- /dev/null
+++ b/jjb/verify/pyvoltha.yaml
@@ -0,0 +1,21 @@
+---
+# verification jobs for 'pyvoltha' repo
+
+- project:
+ name: pyvoltha
+ project: '{name}'
+
+ jobs:
+ - 'verify-pyvoltha-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+
+- job-group:
+ name: 'verify-pyvoltha-jobs'
+ jobs:
+ - 'verify-licensed'
+ - 'tag-collision-reject':
+ dependency-jobs: 'verify_pyvoltha_licensed'
+# - 'verify-sonarqube':
+# dependency-jobs: 'verify_pyvoltha_tag-collision'
+# - 'python-unit-test':
+# dependency-jobs: 'verify_pyvoltha_sonarqube'
diff --git a/jjb/verify/voltha-adtran-adapter.yaml b/jjb/verify/voltha-adtran-adapter.yaml
new file mode 100644
index 0000000..b648a24
--- /dev/null
+++ b/jjb/verify/voltha-adtran-adapter.yaml
@@ -0,0 +1,21 @@
+---
+# verification jobs for 'voltha-adtran-adapter' repo
+
+- project:
+ name: voltha-adtran-adapter
+ project: '{name}'
+
+ jobs:
+ - 'verify-voltha-adtran-adapter-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+
+- job-group:
+ name: 'verify-voltha-adtran-adapter-jobs'
+ jobs:
+ - 'verify-licensed'
+ - 'tag-collision-reject':
+ dependency-jobs: 'verify_voltha-adtran-adapter_licensed'
+# - 'verify-sonarqube':
+# dependency-jobs: 'verify_voltha-adtran-adapter_tag-collision'
+# - 'python-unit-test':
+# dependency-jobs: 'verify_voltha-adtran-adapter_sonarqube'
diff --git a/jjb/verify/voltha-openolt-adapter.yaml b/jjb/verify/voltha-openolt-adapter.yaml
new file mode 100644
index 0000000..4508349
--- /dev/null
+++ b/jjb/verify/voltha-openolt-adapter.yaml
@@ -0,0 +1,21 @@
+---
+# verification jobs for 'voltha-openolt-adapter' repo
+
+- project:
+ name: voltha-openolt-adapter
+ project: '{name}'
+
+ jobs:
+ - 'verify-voltha-openolt-adapter-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+
+- job-group:
+ name: 'verify-voltha-openolt-adapter-jobs'
+ jobs:
+ - 'verify-licensed'
+ - 'tag-collision-reject':
+ dependency-jobs: 'verify_voltha-openolt-adapter_licensed'
+# - 'verify-sonarqube':
+# dependency-jobs: 'verify_voltha-openolt-adapter_tag-collision'
+# - 'python-unit-test':
+# dependency-jobs: 'verify_voltha-openolt-adapter_sonarqube'
diff --git a/jjb/verify/voltha-openonu-adapter.yaml b/jjb/verify/voltha-openonu-adapter.yaml
new file mode 100644
index 0000000..9eca5dc
--- /dev/null
+++ b/jjb/verify/voltha-openonu-adapter.yaml
@@ -0,0 +1,21 @@
+---
+# verification jobs for 'voltha-openonu-adapter' repo
+
+- project:
+ name: voltha-openonu-adapter
+ project: '{name}'
+
+ jobs:
+ - 'verify-voltha-openonu-adapter-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+
+- job-group:
+ name: 'verify-voltha-openonu-adapter-jobs'
+ jobs:
+ - 'verify-licensed'
+ - 'tag-collision-reject':
+ dependency-jobs: 'verify_voltha-openonu-adapter_licensed'
+# - 'verify-sonarqube':
+# dependency-jobs: 'verify_voltha-openonu-adapter_tag-collision'
+# - 'python-unit-test':
+# dependency-jobs: 'verify_voltha-openonu-adapter_sonarqube'
diff --git a/jjb/verify/voltha-protos.yaml b/jjb/verify/voltha-protos.yaml
new file mode 100644
index 0000000..42b6b95
--- /dev/null
+++ b/jjb/verify/voltha-protos.yaml
@@ -0,0 +1,22 @@
+---
+# verification jobs for 'voltha-protos' repo
+
+- project:
+ name: voltha-protos
+ project: '{name}'
+
+ jobs:
+ - 'verify-voltha-protos-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+
+- job-group:
+ name: 'verify-voltha-protos-jobs'
+ jobs:
+ - 'verify-licensed'
+ - 'tag-collision-reject':
+ dependency-jobs: 'verify_voltha-protos_licensed'
+# - 'verify-sonarqube':
+# dependency-jobs: 'verify_voltha-protos_tag-collision'
+ - 'python-unit-test':
+ dependency-jobs: 'verify_voltha-protos_tag-collision'
+# dependency-jobs: 'verify_voltha-protos_sonarqube'
diff --git a/jjb/workflow-validations.yaml b/jjb/workflow-validations.yaml
index f86fdc3..522970b 100644
--- a/jjb/workflow-validations.yaml
+++ b/jjb/workflow-validations.yaml
@@ -11,7 +11,7 @@
- job-template:
id: 'wf-validation'
- name: 'ATT-Event-Workflow-Validation'
+ name: 'att-workflow-xos-data-model-scale-test'
description: |
<!-- Managed by Jenkins Job Builder -->
@@ -44,6 +44,21 @@
default: 'master'
description: 'Name of the repo branch to use'
+ - string:
+ name: Olts
+ default: '10'
+ description: 'Number of OLTs to create'
+
+ - string:
+ name: Onus
+ default: '1'
+ description: 'Number of ONUs to create'
+
+ - string:
+ name: PonPorts
+ default: '10'
+ description: 'Number of Pon Ports'
+
project-type: pipeline
concurrent: false
@@ -52,4 +67,4 @@
triggers:
- timed: |
TZ=America/Los_Angeles
- H 10 * * *
+ H 0,12 * * *
diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh
index 17d68e0..7cd3988 100644
--- a/packer/provision/basebuild.sh
+++ b/packer/provision/basebuild.sh
@@ -86,16 +86,21 @@
# install basic sofware requirements
apt-get install -y \
"docker-ce=17.06*" \
+ apt-transport-https \
ansible \
build-essential \
bzip2 \
curl \
+ ebtables \
+ ethtool \
git \
golang-1.10-go \
httpie \
- "kubeadm=1.11.3-*" \
- "kubelet=1.11.3-*" \
- "kubectl=1.11.3-*" \
+ jq \
+ kafkacat \
+ "kubeadm=1.12.4-*" \
+ "kubelet=1.12.4-*" \
+ "kubectl=1.12.4-*" \
less \
libmysqlclient-dev \
libpcap-dev \
@@ -240,6 +245,12 @@
jenkins ALL=(ALL) NOPASSWD:SETENV: CMDS
EOF
+ # remove apparmor
+ service apparmor stop
+ update-rc.d -f apparmor remove
+ apt-get remove apparmor-utils libapparmor-perl apparmor
+ update-grub
+
# clean up
apt-get clean
apt-get purge -y
diff --git a/packer/provision/baseline.sh b/packer/provision/baseline.sh
index e34d7b8..9c08069 100644
--- a/packer/provision/baseline.sh
+++ b/packer/provision/baseline.sh
@@ -228,6 +228,7 @@
apt-get upgrade -m
apt-get dist-upgrade -m
+ apt-get update -m
ensure_ubuntu_install unzip xz-utils puppet git libxml-xpath-perl
# install Java 7