[SEBA-451]
Automation for publishing python modules to PyPI
Change-Id: If93f927ba61838b7e72b3a447a3144c5bfa417ef
diff --git a/jjb/cord-macros.yaml b/jjb/cord-macros.yaml
index 4d05c2f..68b057d 100644
--- a/jjb/cord-macros.yaml
+++ b/jjb/cord-macros.yaml
@@ -119,3 +119,25 @@
file-paths:
- compare-type: REG_EXP
pattern: '{file-include-regexp}'
+
+# wrapper to provide pypi config file
+
+- wrapper:
+ name: cord-pypi-wrapper
+ wrappers:
+ - mask-passwords
+ - timeout:
+ type: absolute
+ timeout: '{build-timeout}'
+ timeout-var: 'BUILD_TIMEOUT'
+ fail: true
+ - timestamps
+ - ssh-agent-credentials:
+ users:
+ - '{jenkins-ssh-credential}'
+ - config-file-provider:
+ files:
+ - file-id: pypirc
+ target: '$HOME/.pypirc'
+ - file-id: pipconf
+ target: '$HOME/.config/pip/pip.conf'
diff --git a/jjb/defaults.yaml b/jjb/defaults.yaml
index 90973e5..edf86ea 100644
--- a/jjb/defaults.yaml
+++ b/jjb/defaults.yaml
@@ -79,6 +79,17 @@
# (basically the same as imagebuilder projects + helm charts + tools
version-tag-projects-regexp: '^(xos.*|helm-charts|automation-tools|cord-tester|chameleon|rcord|mcord|ecord|acordion|addressmanager|epc-service|exampleservice|fabric|fabric-crossconnect|globalxos|hippie-oss|hss_db|hypercache|internetemulator|kubernetes-service|monitoring|olt-service|onos-service|openstack|progran|sdn-controller|simpleexampleservice|templateservice|vEE|vEG|vBBU|venb|vHSS|vMME|vnaas|vPGWC|vPGWU|vrouter|vsg|vsg-hw|vSGW|vSM|vspgwc|vspgwu|vtn-service|vtr|att-workflow-driver|ves-agent|voltha-bbsim|openolt|sadis-server|kafka-topic-exporter|pyvoltha|voltha-adtran-adapter|voltha-openolt-adapter|voltha-openonu-adapter|plyxproto)$'
+ # Projects that build modules sent to PyPI
+ pypi-projects-regexp: '^(plyxproto)$'
+
+ # Pipe-separated list of directories relative to $WORKSPACE containing
+ # python modules to publish to PyPI.
+ pypi-module-dirs: '.'
+
+ # Which PyPI index to use. Corresponds to values in ~/.pypirc
+ # 'pypi' and 'testpypi' are current options.
+ pypi-index: 'testpypi'
+
# for matching files with file-include-regexp
all-files-regexp: '.*'
doc-files-regexp: '^docs/.*'
diff --git a/jjb/pypi-publish.yaml b/jjb/pypi-publish.yaml
new file mode 100644
index 0000000..9f34aac
--- /dev/null
+++ b/jjb/pypi-publish.yaml
@@ -0,0 +1,62 @@
+---
+# publishing Python modules to PyPI
+
+- project:
+ name: pypi-publisher
+
+ branch-regexp: '{modern-branches-regexp}'
+ project-regexp: '{pypi-projects-regexp}'
+
+ # wait to run pypi-publish job until version-tag job has tagged the repo
+ jobs:
+ - 'pypi-publish':
+ dependency-jobs: 'version-tag'
+
+- job-template:
+ id: pypi-publish
+ name: '{id}'
+ description: |
+ Created by {id} job-template from ci-management/jjb/pypi-publish.yaml
+ When a patch is merged, publish python modules to PyPI
+
+ triggers:
+ - cord-infra-gerrit-trigger-merge:
+ gerrit-server-name: '{gerrit-server-name}'
+ project-regexp: '{project-regexp}'
+ branch-regexp: '{branch-regexp}'
+ file-include-regexp: '{all-files-regexp}'
+ dependency-jobs: '{dependency-jobs}'
+
+ properties:
+ - cord-infra-properties:
+ build-days-to-keep: '{build-days-to-keep}'
+ artifact-num-to-keep: '{artifact-num-to-keep}'
+
+ wrappers:
+ - cord-pypi-wrapper:
+ build-timeout: '{build-timeout}'
+ jenkins-ssh-credential: '{gerrit-ssh-credential}'
+
+ scm:
+ - lf-infra-gerrit-scm:
+ git-url: '$GIT_URL/$GERRIT_PROJECT'
+ refspec: ''
+ branch: '$GERRIT_BRANCH'
+ submodule-recursive: 'false'
+ choosing-strategy: 'gerrit'
+ jenkins-ssh-credential: '{jenkins-ssh-credential}'
+
+ node: '{build-node}'
+ project-type: freestyle
+ concurrent: true
+
+ builders:
+ - inject:
+ properties-content:
+ PYPI_INDEX={pypi-index}
+ PYPI_MODULE_DIRS={pypi-module-dirs}
+
+ - shell: !include-raw-escape: shell/pypi-publish.sh
+
+
+
diff --git a/jjb/shell/pypi-publish.sh b/jjb/shell/pypi-publish.sh
new file mode 100755
index 0000000..0f451cc
--- /dev/null
+++ b/jjb/shell/pypi-publish.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# pypi-publish.sh - Publishes Python modules to PyPI
+#
+# Makes the following assumptions:
+# - PyPI credentials are populated in ~/.pypirc
+# - git repo is tagged with a SEMVER released version. If not, exit.
+# - If required, Environmental variables are set for:
+# PYPI_INDEX - name of PyPI index to use (see contents of ~/.pypirc)
+# PYPI_MODULE_DIRS - pipe-separated list of modules to be uploaded
+
+set -eu -o pipefail
+
+echo "Using twine version:"
+twine --version
+
+pypi_success=0
+
+# environmental vars
+WORKSPACE=${WORKSPACE:-.}
+PYPI_INDEX=${PYPI_INDEX:-testpypi}
+PYPI_MODULE_DIRS=${PYPI_MODULE_DIRS:-.}
+
+# check that we're on a semver released version
+GIT_VERSION=$(git tag -l --points-at HEAD)
+
+if [[ "$GIT_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]
+then
+ echo "git has a SemVer released version tag: '$GIT_VERSION', publishing to PyPI"
+else
+ echo "No SemVer released version tag found, exiting..."
+ exit 0
+fi
+
+# iterate over $PYPI_MODULE_DIRS
+# field separator is pipe character
+IFS=$'|'
+for pymod in $PYPI_MODULE_DIRS
+do
+ pymoddir="$WORKSPACE/$pymod"
+
+ if [ ! -f "$pymoddir/setup.py" ]
+ then
+ echo "Directory with python module not found at '$pymoddir'"
+ pypi_success=1
+ else
+ pushd "$pymoddir"
+
+ echo "Building python module in '$pymoddir'"
+ # Create source distribution
+ python setup.py sdist
+
+ # Upload to PyPI
+ echo "Uploading to PyPI"
+ twine upload -r "$PYPI_INDEX" dist/*
+
+ popd
+ fi
+done
+
+exit $pypi_success
diff --git a/packer/provision/basebuild.sh b/packer/provision/basebuild.sh
index 7cd3988..bf3038d 100644
--- a/packer/provision/basebuild.sh
+++ b/packer/provision/basebuild.sh
@@ -148,6 +148,7 @@
robotframework-requests \
robotframework-sshlibrary \
tox \
+ twine \
virtualenv
# end of pip install list