Merge "Have jflint fail invalid declarative pipeline groovy"
diff --git a/jjb/api-test.yaml b/jjb/api-test.yaml
index 27d877d..740fbd8 100644
--- a/jjb/api-test.yaml
+++ b/jjb/api-test.yaml
@@ -13,7 +13,7 @@
           gerrit-server-name: '{gerrit-server-name}'
           project-regexp: '^{project}$'
           branch-regexp: '{branch-regexp}'
-          file-include-regexp: '{code-files-regexp}'
+          file-include-regexp: '{api-test-files-regexp}'
           dependency-jobs: '{dependency-jobs}'
 
     properties:
diff --git a/jjb/cord-macros.yaml b/jjb/cord-macros.yaml
index 4d05c2f..68b057d 100644
--- a/jjb/cord-macros.yaml
+++ b/jjb/cord-macros.yaml
@@ -119,3 +119,25 @@
               file-paths:
                 - compare-type: REG_EXP
                   pattern: '{file-include-regexp}'
+
+# wrapper to provide pypi config file
+
+- wrapper:
+    name: cord-pypi-wrapper
+    wrappers:
+      - mask-passwords
+      - timeout:
+          type: absolute
+          timeout: '{build-timeout}'
+          timeout-var: 'BUILD_TIMEOUT'
+          fail: true
+      - timestamps
+      - ssh-agent-credentials:
+          users:
+            - '{jenkins-ssh-credential}'
+      - config-file-provider:
+          files:
+            - file-id: pypirc
+              target: '$HOME/.pypirc'
+            - file-id: pipconf
+              target: '$HOME/.config/pip/pip.conf'
diff --git a/jjb/cord-test/att-workflow.yaml b/jjb/cord-test/att-workflow.yaml
index 4562eb1..3f280cf 100644
--- a/jjb/cord-test/att-workflow.yaml
+++ b/jjb/cord-test/att-workflow.yaml
@@ -106,3 +106,11 @@
          profile: 'att-workflow'
          branch: 'cord-6.1'
          Jenkinsfile: 'Jenkinsfile-attworkflow-test'
+
+      # flex ocp pod test job - seba-1.0/cord-6.1 release
+      - 'build_pod_test':
+         testvm: 'qa-testvm-pod'
+         config-pod: 'flex-ocp-cord-voltha16'
+         profile: 'att-workflow'
+         branch: 'cord-6.1'
+         Jenkinsfile: 'Jenkinsfile-attworkflow-test'
diff --git a/jjb/cord-test/mcord.yaml b/jjb/cord-test/mcord.yaml
index 97cafe7..5fd830a 100644
--- a/jjb/cord-test/mcord.yaml
+++ b/jjb/cord-test/mcord.yaml
@@ -16,6 +16,7 @@
          profile: 'mcord'
          branch: 'master'
          Jenkinsfile: 'Jenkinsfile-mcord-local-build'
+
       - 'build_mcord_pod_manual':
          testvm: 'onf-build'
          config-pod: 'onf-mcord-pod2'
@@ -23,3 +24,10 @@
          branch: 'master'
          configurePod: false
          Jenkinsfile: 'Jenkinsfile-mcord-remote-build'
+
+      - 'build_mcord_pod_manual':
+         testvm: 'mcord-dt'
+         config-pod: 'mcord-barcellona-remote-dt'
+         profile: 'mcord'
+         branch: 'master'
+         Jenkinsfile: 'Jenkinsfile-mcord-remote-build'
diff --git a/jjb/cord-test/nightly-build-pipeline.yaml b/jjb/cord-test/nightly-build-pipeline.yaml
index 5d16e45..debaa61 100644
--- a/jjb/cord-test/nightly-build-pipeline.yaml
+++ b/jjb/cord-test/nightly-build-pipeline.yaml
@@ -781,7 +781,7 @@
 
 
 - job-template:
-    name: 'build_{profile}_{config-pod}_{branch}_manual'
+    name: 'build_{config-pod}'
     id: build_mcord_pod_manual
     description: |
                   <!-- Managed by Jenkins Job Builder -->
diff --git a/jjb/defaults.yaml b/jjb/defaults.yaml
index de250ce..684e084 100644
--- a/jjb/defaults.yaml
+++ b/jjb/defaults.yaml
@@ -77,7 +77,19 @@
 
     # matching repos that should be version tagged by the version-tag job
     # (basically the same as imagebuilder projects + helm charts + tools
-    version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter|pyvoltha|voltha-adtran-adapter|voltha-openolt-adapter|voltha-openonu-adapter)$'
+    version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter|pyvoltha|voltha-adtran-adapter|voltha-openolt-adapter|voltha-openonu-adapter|plyxproto)$'
+
+    # PyPI related publishing variables
+    #
+    # Pipe-separated list of directories relative to $WORKSPACE containing
+    # python modules to publish to PyPI. Default is current directory which is
+    # correct for simple modules
+    pypi-module-dirs: '.'
+
+    # Which PyPI index to use. Corresponds to values in ~/.pypirc
+    # 'pypi' and 'testpypi' are current options. Default to testpypi, override
+    # in the job once testing is completed
+    pypi-index: 'testpypi'
 
     # for matching files with file-include-regexp
     all-files-regexp: '.*'
@@ -87,9 +99,9 @@
     # tests.  Have to also ignore the commit message on negative match,
     # per: https://issues.jenkins-ci.org/browse/JENKINS-19891
     code-files-regexp: '^(?!docs|VERSION|\/COMMIT_MSG).*$'
+    api-test-files-regexp: '^(?!docs|VERSION|\/COMMIT_MSG|Jenkinsfile-*).*$'
 
-
-    # Jenkins SSH host doc publisher
+      # Jenkins SSH host doc publisher
     docs-ssh-host: 'guide.opencord.org'
     docs-ssh-host-key: 'guide.opencord.org,52.9.82.207 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFHwOY3/8GucdKzBngH/FC08nHac/RJ/OheZp2+5EpOPXZG9zQW2YUbXH5A9pO76lI5CG3z3+huG62xEGo99UQU='
 
diff --git a/jjb/license-check.yaml b/jjb/license-check.yaml
index 7dc0528..d1f282f 100644
--- a/jjb/license-check.yaml
+++ b/jjb/license-check.yaml
@@ -56,7 +56,13 @@
     project-type: pipeline
     concurrent: true
 
-    dsl: !include-raw-escape: pipeline/license-check.groovy
+    pipeline-scm:
+      script-path: 'Jenkinsfile-mcord-license-check'
+      scm:
+        - git:
+           url: '{gerrit-server-url}/cord-tester'
+           branches:
+             - 'master'
 
     triggers:
        - timed: |
diff --git a/jjb/pipeline/all-xos-api-test-helm.groovy b/jjb/pipeline/all-xos-api-test-helm.groovy
index 26d9518..8efdba4 100644
--- a/jjb/pipeline/all-xos-api-test-helm.groovy
+++ b/jjb/pipeline/all-xos-api-test-helm.groovy
@@ -126,7 +126,7 @@
            XOS_VERSION=\$(cat cord/orchestration/xos/VERSION)
 
            # update services
-           for df in cord/orchestration/xos_services/*/Dockerfile.synchronizer cord/orchestration/profiles/*/Dockerfile.synchronizer
+           for df in cord/orchestration/xos_services/*/Dockerfile.synchronizer
            do
              df_contents=\$(cat "\$df")
              if [[ "\$df_contents" =~ "FROM xosproject/xos-synchronizer-base:\$XOS_MAJOR" ||
diff --git a/jjb/pipeline/license-check.groovy b/jjb/pipeline/license-check.groovy
deleted file mode 100644
index 38320f4..0000000
--- a/jjb/pipeline/license-check.groovy
+++ /dev/null
@@ -1,48 +0,0 @@
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-repos = params.repos.split(",")
-
-node ("${TestNodeName}") {
-    timeout (100) {
-        try {
-            stage ("Cleanup") {
-                sh returnStdout: true, script: "rm -rf *"
-            }
-            stage ("Add private keys") {
-                withCredentials([sshUserPrivateKey(credentialsId: '315e1f56-7193-464e-8af1-97bf7b1ee541', keyFileVariable: 'KEY')]) {
-                    sh returnStdout: true, script: """
-                    chmod 600 $KEY && eval `ssh-agent -s` && ssh-add $KEY &&
-                    ssh-keyscan github.com >> ~/.ssh/known_hosts
-                    """
-                }
-            }
-            for(int i=0; i < repos.size(); i++) {
-                stage ("Check repos ${repos[i]}") {
-                    checkRepo(repos[i])
-                }
-            }
-            currentBuild.result = 'SUCCESS'
-        } catch (err) {
-            currentBuild.result = 'FAILURE'
-            step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: "${notificationEmail}", sendToIndividuals: false])
-        }
-        echo "RESULT: ${currentBuild.result}"
-    }
-}
-
-def checkRepo(repo) {
-        sh returnStdout: true, script: """
-            git clone -b ${branch} git@github.com:omec-project/${repo}.git
-        """
-        hub_detect("--detect.source.path=${repo} --detect.project.name=${prefix}-${repo} --detect.project.version.name=${branch} --snippet-matching --full-snippet-scan")
-}
diff --git a/jjb/pipeline/xos-synchronizer-update.groovy b/jjb/pipeline/xos-synchronizer-update.groovy
index 570cc5f..13ae1f7 100644
--- a/jjb/pipeline/xos-synchronizer-update.groovy
+++ b/jjb/pipeline/xos-synchronizer-update.groovy
@@ -122,7 +122,7 @@
            XOS_VERSION=\$(cat cord/orchestration/xos/VERSION)
 
            # update services
-           for df in cord/orchestration/xos_services/*/Dockerfile.synchronizer cord/orchestration/profiles/*/Dockerfile.synchronizer
+           for df in cord/orchestration/xos_services/*/Dockerfile.synchronizer
            do
              df_contents=\$(cat "\$df")
              if [[ "\$df_contents" =~ "FROM xosproject/xos-synchronizer-base:\$XOS_MAJOR" ||
diff --git a/jjb/pypi-publish.yaml b/jjb/pypi-publish.yaml
new file mode 100644
index 0000000..b2076e9
--- /dev/null
+++ b/jjb/pypi-publish.yaml
@@ -0,0 +1,51 @@
+---
+# publishing Python modules to PyPI
+
+- job-template:
+    id: pypi-publish
+    name: 'pypi-publish_{project}'
+    description: |
+      Created by {id} job-template from ci-management/jjb/pypi-publish.yaml
+      When a patch is merged, publish python modules to PyPI
+
+    triggers:
+      - cord-infra-gerrit-trigger-merge:
+          gerrit-server-name: '{gerrit-server-name}'
+          project-regexp: '{project-regexp}'
+          branch-regexp: '{branch-regexp}'
+          file-include-regexp: '{all-files-regexp}'
+          dependency-jobs: '{dependency-jobs}'
+
+    properties:
+      - cord-infra-properties:
+          build-days-to-keep: '{build-days-to-keep}'
+          artifact-num-to-keep: '{artifact-num-to-keep}'
+
+    wrappers:
+      - cord-pypi-wrapper:
+          build-timeout: '{build-timeout}'
+          jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+    scm:
+      - lf-infra-gerrit-scm:
+          git-url: '$GIT_URL/$GERRIT_PROJECT'
+          refspec: ''
+          branch: '$GERRIT_BRANCH'
+          submodule-recursive: 'false'
+          choosing-strategy: 'gerrit'
+          jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+    node: '{build-node}'
+    project-type: freestyle
+    concurrent: true
+
+    builders:
+      - inject:
+          properties-content: |
+            PYPI_INDEX={pypi-index}
+            PYPI_MODULE_DIRS={pypi-module-dirs}
+
+      - shell: !include-raw-escape: shell/pypi-publish.sh
+
+
+
diff --git a/jjb/python-unit.yaml b/jjb/python-unit.yaml
index 1a20728..18bdb96 100644
--- a/jjb/python-unit.yaml
+++ b/jjb/python-unit.yaml
@@ -61,9 +61,9 @@
 
     publishers:
       - junit:
-          results: "**/nose2-junit.xml"
+          results: "**/*junit.xml,**/*results.xml,**/*report.xml"
       - cobertura:
-          report-file: "**/coverage.xml"
+          report-file: "**/*coverage.xml"
           targets:
             - files:
                 healthy: 80
diff --git a/jjb/shell/pypi-publish.sh b/jjb/shell/pypi-publish.sh
new file mode 100755
index 0000000..0f451cc
--- /dev/null
+++ b/jjb/shell/pypi-publish.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# pypi-publish.sh - Publishes Python modules to PyPI
+#
+# Makes the following assumptions:
+# - PyPI credentials are populated in ~/.pypirc
+# - git repo is tagged with a SEMVER released version. If not, exit.
+# - If required, Environmental variables are set for:
+#     PYPI_INDEX - name of PyPI index to use (see contents of ~/.pypirc)
+#     PYPI_MODULE_DIRS - pipe-separated list of modules to be uploaded
+
+set -eu -o pipefail
+
+echo "Using twine version:"
+twine --version
+
+pypi_success=0
+
+# environmental vars
+WORKSPACE=${WORKSPACE:-.}
+PYPI_INDEX=${PYPI_INDEX:-testpypi}
+PYPI_MODULE_DIRS=${PYPI_MODULE_DIRS:-.}
+
+# check that we're on a semver released version
+GIT_VERSION=$(git tag -l --points-at HEAD)
+
+if [[ "$GIT_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]
+then
+  echo "git has a SemVer released version tag: '$GIT_VERSION', publishing to PyPI"
+else
+  echo "No SemVer released version tag found, exiting..."
+  exit 0
+fi
+
+# iterate over $PYPI_MODULE_DIRS
+# field separator is pipe character
+IFS=$'|'
+for pymod in $PYPI_MODULE_DIRS
+do
+  pymoddir="$WORKSPACE/$pymod"
+
+  if [ ! -f "$pymoddir/setup.py" ]
+  then
+    echo "Directory with python module not found at '$pymoddir'"
+    pypi_success=1
+  else
+    pushd "$pymoddir"
+
+    echo "Building python module in '$pymoddir'"
+    # Create source distribution
+    python setup.py sdist
+
+    # Upload to PyPI
+    echo "Uploading to PyPI"
+    twine upload -r "$PYPI_INDEX" dist/*
+
+    popd
+  fi
+done
+
+exit $pypi_success
diff --git a/jjb/verify/plyxproto.yaml b/jjb/verify/plyxproto.yaml
new file mode 100644
index 0000000..4eb2c32
--- /dev/null
+++ b/jjb/verify/plyxproto.yaml
@@ -0,0 +1,29 @@
+---
+# verification jobs for 'plyxproto' repo
+
+- project:
+    name: plyxproto
+    project: '{name}'
+
+    jobs:
+      - 'verify-plyxproto-jobs':
+          branch-regexp: '{supported-branches-regexp}'
+      - 'post-merge-plyxproto-jobs':
+          branch-regexp: '{supported-branches-regexp}'
+          pypi-index: 'pypi'
+
+- job-group:
+    name: 'verify-plyxproto-jobs'
+    jobs:
+      - 'verify-licensed'
+      - 'tag-collision-reject':
+          dependency-jobs: 'verify_plyxproto_licensed'
+      - 'python-unit-test':
+          dependency-jobs: 'verify_plyxproto_sonarqube'
+
+- job-group:
+    name: 'post-merge-plyxproto-jobs'
+    jobs:
+      - 'pypi-publish':
+          project-regexp: '^{name}$'
+          dependency-jobs: 'version-tag'
diff --git a/jjb/verify/xos.yaml b/jjb/verify/xos.yaml
index b2b56d9..8d4e360 100644
--- a/jjb/verify/xos.yaml
+++ b/jjb/verify/xos.yaml
@@ -29,10 +29,7 @@
     jobs:
       - 'verify-sonarqube'
       - 'xos-unit-test':
-          dependency-jobs: 'verify_xos_sonarqube'
       - 'synchronizer-update':
-          dependency-jobs: 'verify_xos_unit-test'
       - 'api-test':
-          dependency-jobs: 'verify_xos_synchronizer-update'
           pipeline_script: 'all-xos-api-test-helm.groovy'
 
diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh
index 7cd3988..bf3038d 100644
--- a/packer/provision/basebuild.sh
+++ b/packer/provision/basebuild.sh
@@ -148,6 +148,7 @@
         robotframework-requests \
         robotframework-sshlibrary \
         tox \
+        twine \
         virtualenv
         # end of pip install list