Merge "Fixing docker-publish dependency with version-tag job"
diff --git a/jjb/pipeline/voltha/master/software-upgrades.groovy b/jjb/pipeline/voltha/master/software-upgrades.groovy
index 524d9ca..92f772b 100755
--- a/jjb/pipeline/voltha/master/software-upgrades.groovy
+++ b/jjb/pipeline/voltha/master/software-upgrades.groovy
@@ -68,13 +68,19 @@
_TAG=kail-${name} kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
"""
def extraHelmFlags = extraHelmFlags.trim()
- if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
+ if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg" || "${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=1,pon=1 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
}
+ if ("${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg") {
+ extraHelmFlags = " --set global.extended_omci_support.enabled=true " + extraHelmFlags
+ }
+ if ("${name}" == "onu-software-upgrade-omci-extended-msg") {
+ extraHelmFlags = " --set omccVersion=180 " + extraHelmFlags
+ }
if ("${name}" == "onu-image-dwl-simultaneously") {
extraHelmFlags = " --set global.log_level=${logLevel.toUpperCase()},onu=2,pon=2 --set onos-classic.replicas=3,onos-classic.atomix.replicas=3 " + extraHelmFlags
}
- if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-image-dwl-simultaneously") {
+ if ("${name}" == "onos-app-upgrade" || "${name}" == "onu-software-upgrade" || "${name}" == "onu-software-upgrade-omci-extended-msg" || "${name}" == "onu-image-dwl-simultaneously") {
extraHelmFlags = " --set global.image_tag=master --set onos-classic.image.tag=master " + extraHelmFlags
}
if ("${name}" == "voltha-component-upgrade" || "${name}" == "voltha-component-rolling-upgrade") {
@@ -177,7 +183,7 @@
if [[ ${name} == 'voltha-component-rolling-upgrade' ]]; then
export TARGET=voltha-comp-rolling-upgrade-test
fi
- if [[ ${name} == 'onu-software-upgrade' ]]; then
+ if [ ${name} == 'onu-software-upgrade' ] || [ ${name} == 'onu-software-upgrade-omci-extended-msg' ]; then
export ROBOT_MISC_ARGS="-d \$ROBOT_LOGS_DIR -v image_version:${onuImageVersion.trim()} -v image_url:${onuImageUrl.trim()} -v image_vendor:${onuImageVendor.trim()} -v image_activate_on_success:${onuImageActivateOnSuccess.trim()} -v image_commit_on_success:${onuImageCommitOnSuccess.trim()} -v image_crc:${onuImageCrc.trim()} -e PowerSwitch"
export TARGET=onu-upgrade-test
fi
@@ -235,7 +241,7 @@
label "${params.buildNode}"
}
options {
- timeout(time: 150, unit: 'MINUTES')
+ timeout(time: 220, unit: 'MINUTES')
}
environment {
PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/snap/bin"
@@ -273,6 +279,7 @@
test_software_upgrade("voltha-component-upgrade")
test_software_upgrade("voltha-component-rolling-upgrade")
test_software_upgrade("onu-software-upgrade")
+ test_software_upgrade("onu-software-upgrade-omci-extended-msg")
test_software_upgrade("onu-image-dwl-simultaneously")
}
}
diff --git a/jjb/pipeline/voltha/voltha-2.10/bbsim-tests.groovy b/jjb/pipeline/voltha/voltha-2.10/bbsim-tests.groovy
new file mode 100755
index 0000000..edfcb6c
--- /dev/null
+++ b/jjb/pipeline/voltha/voltha-2.10/bbsim-tests.groovy
@@ -0,0 +1,313 @@
+// Copyright 2021-present Open Networking Foundation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// voltha-2.x e2e tests for openonu-go
+// uses bbsim to simulate OLT/ONUs
+
+library identifier: 'cord-jenkins-libraries@master',
+ retriever: modernSCM([
+ $class: 'GitSCMSource',
+ remote: 'https://gerrit.opencord.org/ci-management.git'
+])
+
+def clusterName = "kind-ci"
+
+def execute_test(testTarget, workflow, testLogging, teardown, testSpecificHelmFlags = "") {
+ def infraNamespace = "default"
+ def volthaNamespace = "voltha"
+ def logsDir = "$WORKSPACE/${testTarget}"
+ stage('Cleanup') {
+ if (teardown) {
+ timeout(15) {
+ script {
+ helmTeardown(["default", infraNamespace, volthaNamespace])
+ }
+ timeout(1) {
+ sh returnStdout: false, script: '''
+ # remove orphaned port-forward from different namespaces
+ ps aux | grep port-forw | grep -v grep | awk '{print $2}' | xargs --no-run-if-empty kill -9 || true
+ '''
+ }
+ }
+ }
+ }
+ stage('Deploy common infrastructure') {
+ sh '''
+ helm repo add onf https://charts.opencord.org
+ helm repo update
+ if [ ${withMonitoring} = true ] ; then
+ helm install nem-monitoring onf/nem-monitoring \
+ --set prometheus.alertmanager.enabled=false,prometheus.pushgateway.enabled=false \
+ --set kpi_exporter.enabled=false,dashboards.xos=false,dashboards.onos=false,dashboards.aaa=false,dashboards.voltha=false
+ fi
+ '''
+ }
+ stage('Deploy Voltha') {
+ if (teardown) {
+ timeout(10) {
+ script {
+
+ sh """
+ mkdir -p ${logsDir}
+ _TAG=kail-startup kail -n ${infraNamespace} -n ${volthaNamespace} > ${logsDir}/onos-voltha-startup-combined.log &
+ """
+
+ // if we're downloading a voltha-helm-charts patch, then install from a local copy of the charts
+ def localCharts = false
+ if (volthaHelmChartsChange != "" || gerritProject == "voltha-helm-charts") {
+ localCharts = true
+ }
+
+ // NOTE temporary workaround expose ONOS node ports
+ def localHelmFlags = extraHelmFlags.trim() + " --set global.log_level=${logLevel.toUpperCase()} " +
+ " --set onos-classic.onosSshPort=30115 " +
+ " --set onos-classic.onosApiPort=30120 " +
+ " --set onos-classic.onosOfPort=31653 " +
+ " --set onos-classic.individualOpenFlowNodePorts=true " + testSpecificHelmFlags
+
+ if (gerritProject != "") {
+ localHelmFlags = "${localHelmFlags} " + getVolthaImageFlags("${gerritProject}")
+ }
+
+ volthaDeploy([
+ infraNamespace: infraNamespace,
+ volthaNamespace: volthaNamespace,
+ workflow: workflow.toLowerCase(),
+ withMacLearning: enableMacLearning.toBoolean(),
+ extraHelmFlags: localHelmFlags,
+ localCharts: localCharts,
+ bbsimReplica: olts.toInteger(),
+ dockerRegistry: registry,
+ ])
+ }
+
+ // stop logging
+ sh """
+ P_IDS="\$(ps e -ww -A | grep "_TAG=kail-startup" | grep -v grep | awk '{print \$1}')"
+ if [ -n "\$P_IDS" ]; then
+ echo \$P_IDS
+ for P_ID in \$P_IDS; do
+ kill -9 \$P_ID
+ done
+ fi
+ cd ${logsDir}
+ gzip -k onos-voltha-startup-combined.log
+ rm onos-voltha-startup-combined.log
+ """
+ }
+ sh """
+ JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-voltha-api" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/voltha-voltha-api 55555:55555; done"&
+ JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-etcd" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-etcd 2379:2379; done"&
+ JENKINS_NODE_COOKIE="dontKillMe" _TAG="voltha-infra-kafka" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${infraNamespace} svc/voltha-infra-kafka 9092:9092; done"&
+ bbsimDmiPortFwd=50075
+ for i in {0..${olts.toInteger() - 1}}; do
+ JENKINS_NODE_COOKIE="dontKillMe" _TAG="bbsim\${i}" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n ${volthaNamespace} svc/bbsim\${i} \${bbsimDmiPortFwd}:50075; done"&
+ ((bbsimDmiPortFwd++))
+ done
+ if [ ${withMonitoring} = true ] ; then
+ JENKINS_NODE_COOKIE="dontKillMe" _TAG="nem-monitoring-prometheus-server" bash -c "while true; do kubectl port-forward --address 0.0.0.0 -n default svc/nem-monitoring-prometheus-server 31301:80; done"&
+ fi
+ ps aux | grep port-forward
+ """
+ // setting ONOS log level
+ script {
+ setOnosLogLevels([
+ onosNamespace: infraNamespace,
+ apps: [
+ 'org.opencord.dhcpl2relay',
+ 'org.opencord.olt',
+ 'org.opencord.aaa',
+ 'org.opencord.maclearner',
+ 'org.onosproject.net.flowobjective.impl.FlowObjectiveManager',
+ 'org.onosproject.net.flowobjective.impl.InOrderFlowObjectiveManager'
+ ],
+ logLevel: logLevel
+ ])
+ }
+ }
+ }
+ stage('Run test ' + testTarget + ' on ' + workflow + ' workFlow') {
+ sh """
+ if [ ${withMonitoring} = true ] ; then
+ mkdir -p $WORKSPACE/voltha-pods-mem-consumption-${workflow}
+ cd $WORKSPACE/voltha-system-tests
+ make vst_venv
+ source ./vst_venv/bin/activate || true
+ # Collect initial memory consumption
+ python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
+ fi
+ """
+ sh """
+ mkdir -p ${logsDir}
+ export ROBOT_MISC_ARGS="-d ${logsDir} ${params.extraRobotArgs} "
+ ROBOT_MISC_ARGS+="-v ONOS_SSH_PORT:30115 -v ONOS_REST_PORT:30120 -v NAMESPACE:${volthaNamespace} -v INFRA_NAMESPACE:${infraNamespace} -v container_log_dir:${logsDir} -v logging:${testLogging}"
+ export KVSTOREPREFIX=voltha/voltha_voltha
+
+ make -C $WORKSPACE/voltha-system-tests ${testTarget} || true
+ """
+ getPodsInfo("${logsDir}")
+ sh """
+ set +e
+ # collect logs collected in the Robot Framework StartLogging keyword
+ cd ${logsDir}
+ gzip *-combined.log || true
+ rm *-combined.log || true
+ """
+ sh """
+ if [ ${withMonitoring} = true ] ; then
+ cd $WORKSPACE/voltha-system-tests
+ source ./vst_venv/bin/activate || true
+ # Collect memory consumption of voltha pods once all the tests are complete
+ python scripts/mem_consumption.py -o $WORKSPACE/voltha-pods-mem-consumption-${workflow} -a 0.0.0.0:31301 -n ${volthaNamespace} || true
+ fi
+ """
+ }
+}
+
+def collectArtifacts(exitStatus) {
+ getPodsInfo("$WORKSPACE/${exitStatus}")
+ sh """
+ kubectl logs -n voltha -l app.kubernetes.io/part-of=voltha > $WORKSPACE/${exitStatus}/voltha.log || true
+ """
+ archiveArtifacts artifacts: '**/*.log,**/*.gz,**/*.txt,**/*.html,**/voltha-pods-mem-consumption-att/*,**/voltha-pods-mem-consumption-dt/*,**/voltha-pods-mem-consumption-tt/*'
+ sh '''
+ sync
+ pkill kail || true
+ which voltctl
+ md5sum $(which voltctl)
+ '''
+ step([$class: 'RobotPublisher',
+ disableArchiveOutput: false,
+ logFileName: "**/*/log*.html",
+ otherFiles: '',
+ outputFileName: "**/*/output*.xml",
+ outputPath: '.',
+ passThreshold: 100,
+ reportFileName: "**/*/report*.html",
+ unstableThreshold: 0,
+ onlyCritical: true]);
+}
+
+pipeline {
+
+ /* no label, executor is determined by JJB */
+ agent {
+ label "${params.buildNode}"
+ }
+ options {
+ timeout(time: "${timeout}", unit: 'MINUTES')
+ }
+ environment {
+ KUBECONFIG="$HOME/.kube/kind-${clusterName}"
+ VOLTCONFIG="$HOME/.volt/config"
+ PATH="$PATH:$WORKSPACE/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+ DIAGS_PROFILE="VOLTHA_PROFILE"
+ SSHPASS="karaf"
+ }
+ stages {
+ stage('Download Code') {
+ steps {
+ getVolthaCode([
+ branch: "${branch}",
+ gerritProject: "${gerritProject}",
+ gerritRefspec: "${gerritRefspec}",
+ volthaSystemTestsChange: "${volthaSystemTestsChange}",
+ volthaHelmChartsChange: "${volthaHelmChartsChange}",
+ ])
+ }
+ }
+ stage('Build patch') {
+ // build the patch only if gerritProject is specified
+ when {
+ expression {
+ return !gerritProject.isEmpty()
+ }
+ }
+ steps {
+ // NOTE that the correct patch has already been checked out
+ // during the getVolthaCode step
+ buildVolthaComponent("${gerritProject}")
+ }
+ }
+ stage('Create K8s Cluster') {
+ steps {
+ script {
+ def clusterExists = sh returnStdout: true, script: """
+ kind get clusters | grep ${clusterName} | wc -l
+ """
+ if (clusterExists.trim() == "0") {
+ createKubernetesCluster([nodes: 3, name: clusterName])
+ }
+ }
+ }
+ }
+ stage('Replace voltctl') {
+ // if the project is voltctl override the downloaded one with the built one
+ when {
+ expression {
+ return gerritProject == "voltctl"
+ }
+ }
+ steps{
+ sh """
+ mv `ls $WORKSPACE/voltctl/release/voltctl-*-linux-amd*` $WORKSPACE/bin/voltctl
+ chmod +x $WORKSPACE/bin/voltctl
+ """
+ }
+ }
+ stage('Load image in kind nodes') {
+ when {
+ expression {
+ return !gerritProject.isEmpty()
+ }
+ }
+ steps {
+ loadToKind()
+ }
+ }
+ stage('Parse and execute tests') {
+ steps {
+ script {
+ def tests = readYaml text: testTargets
+
+ for(int i = 0;i<tests.size();i++) {
+ def test = tests[i]
+ def target = test["target"]
+ def workflow = test["workflow"]
+ def flags = test["flags"]
+ def teardown = test["teardown"].toBoolean()
+ def logging = test["logging"].toBoolean()
+ def testLogging = 'False'
+ if (logging) {
+ testLogging = 'True'
+ }
+ println "Executing test ${target} on workflow ${workflow} with logging ${testLogging} and extra flags ${flags}"
+ execute_test(target, workflow, testLogging, teardown, flags)
+ }
+ }
+ }
+ }
+ }
+ post {
+ aborted {
+ collectArtifacts("aborted")
+ }
+ failure {
+ collectArtifacts("failed")
+ }
+ always {
+ collectArtifacts("always")
+ }
+ }
+}
diff --git a/jjb/repos/voltha-helm-charts.yaml b/jjb/repos/voltha-helm-charts.yaml
index a8f6d36..9b80797 100644
--- a/jjb/repos/voltha-helm-charts.yaml
+++ b/jjb/repos/voltha-helm-charts.yaml
@@ -18,6 +18,10 @@
name-extension: '-voltha-2.9'
override-branch: 'voltha-2.9'
branch-regexp: '^voltha-2.9$'
+ - 'verify-voltha-helm-charts-jobs-voltha-2.10':
+ name-extension: '-voltha-2.10'
+ override-branch: 'voltha-2.10'
+ branch-regexp: '^voltha-2.10$'
- 'verify-voltha-helm-charts-jobs-master':
branch-regexp: '^master$'
@@ -43,6 +47,12 @@
pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
- job-group:
+ name: 'verify-voltha-helm-charts-jobs-voltha-2.10'
+ jobs:
+ - 'voltha-patch-test':
+ pipeline-script: 'voltha/voltha-2.10/bbsim-tests.groovy'
+
+- job-group:
name: 'verify-voltha-helm-charts-jobs-master'
jobs:
- 'voltha-patch-test'
diff --git a/jjb/software-upgrades.yaml b/jjb/software-upgrades.yaml
index a6e903f..89a29b7 100644
--- a/jjb/software-upgrades.yaml
+++ b/jjb/software-upgrades.yaml
@@ -25,9 +25,9 @@
mcast-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/mcast-app/2.6.0-SNAPSHOT/mcast-app-2.6.0-20220318.145711-3.oar'
kafka-version: '2.9.0.SNAPSHOT'
kafka-oar-url: 'https://oss.sonatype.org/content/groups/public/org/opencord/kafka/2.9.0-SNAPSHOT/kafka-2.9.0-20220609.001209-5.oar'
- openolt-adapter-deploy-base-tag: '4.2.5'
- openonu-adapter-deploy-base-tag: '2.2.7'
- rw-core-deploy-base-tag: '3.1.5'
+ openolt-adapter-deploy-base-tag: '4.2.6'
+ openonu-adapter-deploy-base-tag: '2.2.8'
+ rw-core-deploy-base-tag: '3.1.6'
ofagent-deploy-base-tag: '2.1.2'
onu-image-version: 'BBSM_IMG_00002'
onu-image-url: 'http://bbsim0:50074/images/software-image.img'
diff --git a/jjb/verify/voltha-system-tests.yaml b/jjb/verify/voltha-system-tests.yaml
index 1b14801..9a8cc35 100644
--- a/jjb/verify/voltha-system-tests.yaml
+++ b/jjb/verify/voltha-system-tests.yaml
@@ -19,6 +19,11 @@
name-extension: '-voltha-2.9'
override-branch: 'voltha-2.9'
branch-regexp: '^voltha-2.9$'
+ - 'verify-voltha-system-tests-jobs-voltha-2.10':
+ build-node: 'ubuntu18.04-basebuild-4c-8g'
+ name-extension: '-voltha-2.10'
+ override-branch: 'voltha-2.10'
+ branch-regexp: '^voltha-2.10$'
- 'verify-voltha-system-tests-jobs-master':
build-node: 'ubuntu18.04-basebuild-4c-8g'
branch-regexp: '^master$'
@@ -47,6 +52,12 @@
pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
- job-group:
+ name: 'verify-voltha-system-tests-jobs-voltha-2.10'
+ jobs:
+ - 'voltha-patch-test':
+ pipeline-script: 'voltha/voltha-2.10/bbsim-tests.groovy'
+
+- job-group:
name: 'verify-voltha-system-tests-jobs-master'
jobs:
- 'voltha-patch-test':
diff --git a/jjb/voltha-e2e.yaml b/jjb/voltha-e2e.yaml
index 4951a3a..5327b83 100755
--- a/jjb/voltha-e2e.yaml
+++ b/jjb/voltha-e2e.yaml
@@ -256,7 +256,7 @@
logging: true
- target: mib-upload-templating-openonu-go-adapter-test
workflow: att
- flags: "--set pon=2,onu=2,controlledActivation=only-onu,injectOmciUnknownAttributes=true,injectOmciUnknownMe=true"
+ flags: "--set pon=2,onu=2,controlledActivation=only-onu,injectOmciUnknownAttributes=true,injectOmciUnknownMe=true --set global.extended_omci_support.enabled=true "
teardown: true
logging: true
- target: reconcile-openonu-go-adapter-test-att
@@ -425,7 +425,7 @@
logging: true
- target: mib-upload-templating-openonu-go-adapter-test
workflow: att
- flags: "--set pon=2,onu=2,controlledActivation=only-onu,injectOmciUnknownAttributes=true,injectOmciUnknownMe=true"
+ flags: "--set pon=2,onu=2,controlledActivation=only-onu,injectOmciUnknownAttributes=true,injectOmciUnknownMe=true --set global.extended_omci_support.enabled=true "
teardown: true
logging: true
- target: reconcile-openonu-go-adapter-test-att
@@ -712,6 +712,62 @@
timeout: 235
- 'voltha-periodic-test':
+ name: 'periodic-voltha-multiple-olts-openonu-go-test-bbsim-2.10'
+ pipeline-script: 'voltha/voltha-2.10/bbsim-tests.groovy'
+ code-branch: 'voltha-2.10'
+ extraHelmFlags: '--set onu=2,pon=2'
+ olts: 2
+ logLevel: 'DEBUG'
+ testTargets: |
+ - target: 1t1gem-openonu-go-adapter-multi-olt-test
+ workflow: att
+ flags: ""
+ teardown: true
+ logging: true
+ - target: 1t4gem-openonu-go-adapter-multi-olt-test
+ workflow: att
+ flags: ""
+ teardown: true
+ logging: true
+ - target: 1t8gem-openonu-go-adapter-multi-olt-test
+ workflow: att
+ flags: ""
+ teardown: true
+ logging: true
+ - target: reconcile-openonu-go-adapter-multi-olt-test-att
+ workflow: att
+ flags: ""
+ teardown: true
+ logging: true
+ - target: reconcile-openonu-go-adapter-multi-olt-test-dt
+ workflow: dt
+ flags: ""
+ teardown: true
+ logging: true
+ - target: reconcile-openonu-go-adapter-multi-olt-test-tt
+ workflow: tt
+ flags: ""
+ teardown: true
+ logging: true
+ - target: voltha-onu-omci-get-multiolt-kind-att
+ workflow: att
+ flags: ""
+ teardown: true
+ logging: true
+ - target: voltha-onu-omci-get-multiolt-kind-dt
+ workflow: dt
+ flags: ""
+ teardown: true
+ logging: true
+ - target: voltha-onu-omci-get-multiolt-kind-tt
+ workflow: tt
+ flags: ""
+ teardown: true
+ logging: true
+ time-trigger: "H H/23 * * *"
+ timeout: 235
+
+ - 'voltha-periodic-test':
name: 'periodic-voltha-multiple-olts-openonu-go-test-bbsim-2.9'
pipeline-script: 'voltha/voltha-2.9/bbsim-tests.groovy'
code-branch: 'voltha-2.9'
diff --git a/vars/installVoltctl.groovy b/vars/installVoltctl.groovy
index d75beb1..dad20ff 100644
--- a/vars/installVoltctl.groovy
+++ b/vars/installVoltctl.groovy
@@ -6,6 +6,8 @@
voltctlVersion = "1.6.11"
} else if (branch == "voltha-2.9") {
voltctlVersion = "1.7.4"
+ } else if (branch == "voltha-2.10") {
+ voltctlVersion = "1.7.6"
} else {
voltctlVersion = sh (
script: "curl -sSL https://api.github.com/repos/opencord/voltctl/releases/latest | jq -r .tag_name | sed -e 's/^v//g'",