Merge "Have jflint fail invalid declarative pipeline groovy"
diff --git a/jjb/api-test.yaml b/jjb/api-test.yaml
index 27d877d..740fbd8 100644
--- a/jjb/api-test.yaml
+++ b/jjb/api-test.yaml
@@ -13,7 +13,7 @@
gerrit-server-name: '{gerrit-server-name}'
project-regexp: '^{project}$'
branch-regexp: '{branch-regexp}'
- file-include-regexp: '{code-files-regexp}'
+ file-include-regexp: '{api-test-files-regexp}'
dependency-jobs: '{dependency-jobs}'
properties:
diff --git a/jjb/cord-macros.yaml b/jjb/cord-macros.yaml
index 4d05c2f..68b057d 100644
--- a/jjb/cord-macros.yaml
+++ b/jjb/cord-macros.yaml
@@ -119,3 +119,25 @@
file-paths:
- compare-type: REG_EXP
pattern: '{file-include-regexp}'
+
+# wrapper to provide pypi config file
+
+- wrapper:
+ name: cord-pypi-wrapper
+ wrappers:
+ - mask-passwords
+ - timeout:
+ type: absolute
+ timeout: '{build-timeout}'
+ timeout-var: 'BUILD_TIMEOUT'
+ fail: true
+ - timestamps
+ - ssh-agent-credentials:
+ users:
+ - '{jenkins-ssh-credential}'
+ - config-file-provider:
+ files:
+ - file-id: pypirc
+ target: '$HOME/.pypirc'
+ - file-id: pipconf
+ target: '$HOME/.config/pip/pip.conf'
diff --git a/jjb/cord-test/att-workflow.yaml b/jjb/cord-test/att-workflow.yaml
index 4562eb1..3f280cf 100644
--- a/jjb/cord-test/att-workflow.yaml
+++ b/jjb/cord-test/att-workflow.yaml
@@ -106,3 +106,11 @@
profile: 'att-workflow'
branch: 'cord-6.1'
Jenkinsfile: 'Jenkinsfile-attworkflow-test'
+
+ # flex ocp pod test job - seba-1.0/cord-6.1 release
+ - 'build_pod_test':
+ testvm: 'qa-testvm-pod'
+ config-pod: 'flex-ocp-cord-voltha16'
+ profile: 'att-workflow'
+ branch: 'cord-6.1'
+ Jenkinsfile: 'Jenkinsfile-attworkflow-test'
diff --git a/jjb/cord-test/mcord.yaml b/jjb/cord-test/mcord.yaml
index 97cafe7..5fd830a 100644
--- a/jjb/cord-test/mcord.yaml
+++ b/jjb/cord-test/mcord.yaml
@@ -16,6 +16,7 @@
profile: 'mcord'
branch: 'master'
Jenkinsfile: 'Jenkinsfile-mcord-local-build'
+
- 'build_mcord_pod_manual':
testvm: 'onf-build'
config-pod: 'onf-mcord-pod2'
@@ -23,3 +24,10 @@
branch: 'master'
configurePod: false
Jenkinsfile: 'Jenkinsfile-mcord-remote-build'
+
+ - 'build_mcord_pod_manual':
+ testvm: 'mcord-dt'
+ config-pod: 'mcord-barcellona-remote-dt'
+ profile: 'mcord'
+ branch: 'master'
+ Jenkinsfile: 'Jenkinsfile-mcord-remote-build'
diff --git a/jjb/cord-test/nightly-build-pipeline.yaml b/jjb/cord-test/nightly-build-pipeline.yaml
index 5d16e45..debaa61 100644
--- a/jjb/cord-test/nightly-build-pipeline.yaml
+++ b/jjb/cord-test/nightly-build-pipeline.yaml
@@ -781,7 +781,7 @@
- job-template:
- name: 'build_{profile}_{config-pod}_{branch}_manual'
+ name: 'build_{config-pod}'
id: build_mcord_pod_manual
description: |
<!-- Managed by Jenkins Job Builder -->
diff --git a/jjb/defaults.yaml b/jjb/defaults.yaml
index de250ce..684e084 100644
--- a/jjb/defaults.yaml
+++ b/jjb/defaults.yaml
@@ -77,7 +77,19 @@
# matching repos that should be version tagged by the version-tag job
# (basically the same as imagebuilder projects + helm charts + tools
- version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter|pyvoltha|voltha-adtran-adapter|voltha-openolt-adapter|voltha-openonu-adapter)$'
+ version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter|pyvoltha|voltha-adtran-adapter|voltha-openolt-adapter|voltha-openonu-adapter|plyxproto)$'
+
+ # PyPI related publishing variables
+ #
+ # Pipe-separated list of directories relative to $WORKSPACE containing
+ # python modules to publish to PyPI. Default is current directory which is
+ # correct for simple modules
+ pypi-module-dirs: '.'
+
+ # Which PyPI index to use. Corresponds to values in ~/.pypirc
+ # 'pypi' and 'testpypi' are current options. Default to testpypi, override
+ # in the job once testing is completed
+ pypi-index: 'testpypi'
# for matching files with file-include-regexp
all-files-regexp: '.*'
@@ -87,9 +99,9 @@
# tests. Have to also ignore the commit message on negative match,
# per: https://issues.jenkins-ci.org/browse/JENKINS-19891
code-files-regexp: '^(?!docs|VERSION|\/COMMIT_MSG).*$'
+ api-test-files-regexp: '^(?!docs|VERSION|\/COMMIT_MSG|Jenkinsfile-*).*$'
-
- # Jenkins SSH host doc publisher
+ # Jenkins SSH host doc publisher
docs-ssh-host: 'guide.opencord.org'
docs-ssh-host-key: 'guide.opencord.org,52.9.82.207 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFHwOY3/8GucdKzBngH/FC08nHac/RJ/OheZp2+5EpOPXZG9zQW2YUbXH5A9pO76lI5CG3z3+huG62xEGo99UQU='
diff --git a/jjb/license-check.yaml b/jjb/license-check.yaml
index 7dc0528..d1f282f 100644
--- a/jjb/license-check.yaml
+++ b/jjb/license-check.yaml
@@ -56,7 +56,13 @@
project-type: pipeline
concurrent: true
- dsl: !include-raw-escape: pipeline/license-check.groovy
+ pipeline-scm:
+ script-path: 'Jenkinsfile-mcord-license-check'
+ scm:
+ - git:
+ url: '{gerrit-server-url}/cord-tester'
+ branches:
+ - 'master'
triggers:
- timed: |
diff --git a/jjb/pipeline/all-xos-api-test-helm.groovy b/jjb/pipeline/all-xos-api-test-helm.groovy
index 26d9518..8efdba4 100644
--- a/jjb/pipeline/all-xos-api-test-helm.groovy
+++ b/jjb/pipeline/all-xos-api-test-helm.groovy
@@ -126,7 +126,7 @@
XOS_VERSION=\$(cat cord/orchestration/xos/VERSION)
# update services
- for df in cord/orchestration/xos_services/*/Dockerfile.synchronizer cord/orchestration/profiles/*/Dockerfile.synchronizer
+ for df in cord/orchestration/xos_services/*/Dockerfile.synchronizer
do
df_contents=\$(cat "\$df")
if [[ "\$df_contents" =~ "FROM xosproject/xos-synchronizer-base:\$XOS_MAJOR" ||
diff --git a/jjb/pipeline/license-check.groovy b/jjb/pipeline/license-check.groovy
deleted file mode 100644
index 38320f4..0000000
--- a/jjb/pipeline/license-check.groovy
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-repos = params.repos.split(",")
-
-node ("${TestNodeName}") {
- timeout (100) {
- try {
- stage ("Cleanup") {
- sh returnStdout: true, script: "rm -rf *"
- }
- stage ("Add private keys") {
- withCredentials([sshUserPrivateKey(credentialsId: '315e1f56-7193-464e-8af1-97bf7b1ee541', keyFileVariable: 'KEY')]) {
- sh returnStdout: true, script: """
- chmod 600 $KEY && eval `ssh-agent -s` && ssh-add $KEY &&
- ssh-keyscan github.com >> ~/.ssh/known_hosts
- """
- }
- }
- for(int i=0; i < repos.size(); i++) {
- stage ("Check repos ${repos[i]}") {
- checkRepo(repos[i])
- }
- }
- currentBuild.result = 'SUCCESS'
- } catch (err) {
- currentBuild.result = 'FAILURE'
- step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
- }
- echo "RESULT: ${currentBuild.result}"
- }
-}
-
-def checkRepo(repo) {
- sh returnStdout: true, script: """
- git clone -b ${branch} git@github.com:omec-project/${repo}.git
- """
- hub_detect("--detect.source.path=${repo} --detect.project.name=${prefix}-${repo} --detect.project.version.name=${branch} --snippet-matching --full-snippet-scan")
-}
diff --git a/jjb/pipeline/xos-synchronizer-update.groovy b/jjb/pipeline/xos-synchronizer-update.groovy
index 570cc5f..13ae1f7 100644
--- a/jjb/pipeline/xos-synchronizer-update.groovy
+++ b/jjb/pipeline/xos-synchronizer-update.groovy
@@ -122,7 +122,7 @@
XOS_VERSION=\$(cat cord/orchestration/xos/VERSION)
# update services
- for df in cord/orchestration/xos_services/*/Dockerfile.synchronizer cord/orchestration/profiles/*/Dockerfile.synchronizer
+ for df in cord/orchestration/xos_services/*/Dockerfile.synchronizer
do
df_contents=\$(cat "\$df")
if [[ "\$df_contents" =~ "FROM xosproject/xos-synchronizer-base:\$XOS_MAJOR" ||
diff --git a/jjb/pypi-publish.yaml b/jjb/pypi-publish.yaml
new file mode 100644
index 0000000..b2076e9
--- /dev/null
+++ b/jjb/pypi-publish.yaml
@@ -0,0 +1,51 @@
+---
+# publishing Python modules to PyPI
+
+- job-template:
+ id: pypi-publish
+ name: 'pypi-publish_{project}'
+ description: |
+ Created by {id} job-template from ci-management/jjb/pypi-publish.yaml
+ When a patch is merged, publish python modules to PyPI
+
+ triggers:
+ - cord-infra-gerrit-trigger-merge:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: '{project-regexp}'
+ branch-regexp: '{branch-regexp}'
+ file-include-regexp: '{all-files-regexp}'
+ dependency-jobs: '{dependency-jobs}'
+
+ properties:
+ - cord-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - cord-pypi-wrapper:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+ scm:
+ - lf-infra-gerrit-scm:
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+ refspec: ''
+ branch: '$GERRIT_BRANCH'
+ submodule-recursive: 'false'
+ choosing-strategy: 'gerrit'
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ node: '{build-node}'
+ project-type: freestyle
+ concurrent: true
+
+ builders:
+ - inject:
+ properties-content: |
+ PYPI_INDEX={pypi-index}
+ PYPI_MODULE_DIRS={pypi-module-dirs}
+
+ - shell: !include-raw-escape: shell/pypi-publish.sh
+
+
+
diff --git a/jjb/python-unit.yaml b/jjb/python-unit.yaml
index 1a20728..18bdb96 100644
--- a/jjb/python-unit.yaml
+++ b/jjb/python-unit.yaml
@@ -61,9 +61,9 @@
publishers:
- junit:
- results: "**/nose2-junit.xml"
+ results: "**/*junit.xml,**/*results.xml,**/*report.xml"
- cobertura:
- report-file: "**/coverage.xml"
+ report-file: "**/*coverage.xml"
targets:
- files:
healthy: 80
diff --git a/jjb/shell/pypi-publish.sh b/jjb/shell/pypi-publish.sh
new file mode 100755
index 0000000..0f451cc
--- /dev/null
+++ b/jjb/shell/pypi-publish.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# pypi-publish.sh - Publishes Python modules to PyPI
+#
+# Makes the following assumptions:
+# - PyPI credentials are populated in ~/.pypirc
+# - git repo is tagged with a SEMVER released version. If not, exit.
+# - If required, Environmental variables are set for:
+# PYPI_INDEX - name of PyPI index to use (see contents of ~/.pypirc)
+# PYPI_MODULE_DIRS - pipe-separated list of modules to be uploaded
+
+set -eu -o pipefail
+
+echo "Using twine version:"
+twine --version
+
+pypi_success=0
+
+# environmental vars
+WORKSPACE=${WORKSPACE:-.}
+PYPI_INDEX=${PYPI_INDEX:-testpypi}
+PYPI_MODULE_DIRS=${PYPI_MODULE_DIRS:-.}
+
+# check that we're on a semver released version
+GIT_VERSION=$(git tag -l --points-at HEAD)
+
+if [[ "$GIT_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]
+then
+ echo "git has a SemVer released version tag: '$GIT_VERSION', publishing to PyPI"
+else
+ echo "No SemVer released version tag found, exiting..."
+ exit 0
+fi
+
+# iterate over $PYPI_MODULE_DIRS
+# field separator is pipe character
+IFS=$'|'
+for pymod in $PYPI_MODULE_DIRS
+do
+ pymoddir="$WORKSPACE/$pymod"
+
+ if [ ! -f "$pymoddir/setup.py" ]
+ then
+ echo "Directory with python module not found at '$pymoddir'"
+ pypi_success=1
+ else
+ pushd "$pymoddir"
+
+ echo "Building python module in '$pymoddir'"
+ # Create source distribution
+ python setup.py sdist
+
+ # Upload to PyPI
+ echo "Uploading to PyPI"
+ twine upload -r "$PYPI_INDEX" dist/*
+
+ popd
+ fi
+done
+
+exit $pypi_success
diff --git a/jjb/verify/plyxproto.yaml b/jjb/verify/plyxproto.yaml
new file mode 100644
index 0000000..4eb2c32
--- /dev/null
+++ b/jjb/verify/plyxproto.yaml
@@ -0,0 +1,29 @@
+---
+# verification jobs for 'plyxproto' repo
+
+- project:
+ name: plyxproto
+ project: '{name}'
+
+ jobs:
+ - 'verify-plyxproto-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+ - 'post-merge-plyxproto-jobs':
+ branch-regexp: '{supported-branches-regexp}'
+ pypi-index: 'pypi'
+
+- job-group:
+ name: 'verify-plyxproto-jobs'
+ jobs:
+ - 'verify-licensed'
+ - 'tag-collision-reject':
+ dependency-jobs: 'verify_plyxproto_licensed'
+ - 'python-unit-test':
+ dependency-jobs: 'verify_plyxproto_sonarqube'
+
+- job-group:
+ name: 'post-merge-plyxproto-jobs'
+ jobs:
+ - 'pypi-publish':
+ project-regexp: '^{name}$'
+ dependency-jobs: 'version-tag'
diff --git a/jjb/verify/xos.yaml b/jjb/verify/xos.yaml
index b2b56d9..8d4e360 100644
--- a/jjb/verify/xos.yaml
+++ b/jjb/verify/xos.yaml
@@ -29,10 +29,7 @@
jobs:
- 'verify-sonarqube'
- 'xos-unit-test':
- dependency-jobs: 'verify_xos_sonarqube'
- 'synchronizer-update':
- dependency-jobs: 'verify_xos_unit-test'
- 'api-test':
- dependency-jobs: 'verify_xos_synchronizer-update'
pipeline_script: 'all-xos-api-test-helm.groovy'
diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh
index 7cd3988..bf3038d 100644
--- a/packer/provision/basebuild.sh
+++ b/packer/provision/basebuild.sh
@@ -148,6 +148,7 @@
robotframework-requests \
robotframework-sshlibrary \
tox \
+ twine \
virtualenv
# end of pip install list