updated with changes to support automated triggering of post-deploymet provisioning
diff --git a/automation/state.go b/automation/state.go
index 4b94089..696f58f 100644
--- a/automation/state.go
+++ b/automation/state.go
@@ -1,12 +1,17 @@
package main
import (
+ "bytes"
+ "encoding/json"
"fmt"
"log"
+ "net/http"
"net/url"
+ "os/exec"
"regexp"
"strconv"
"strings"
+ "time"
maas "github.com/juju/gomaasapi"
)
@@ -37,6 +42,9 @@
Verbose bool
Preview bool
AlwaysRename bool
+ ProvTracker Tracker
+ ProvisionURL string
+ ProvisionTTL time.Duration
}
// Transitions the actual map
@@ -45,24 +53,24 @@
// really be generated from the state machine chart input. Once this has been
// accomplished you should be able to determine the action to take given your
// target state and your current state.
-var Transitions = map[string]map[string]Action{
+var Transitions = map[string]map[string][]Action{
"Deployed": {
- "New": Commission,
- "Deployed": Done,
- "Ready": Aquire,
- "Allocated": Deploy,
- "Retired": AdminState,
- "Reserved": AdminState,
- "Releasing": Wait,
- "DiskErasing": Wait,
- "Deploying": Wait,
- "Commissioning": Wait,
- "Missing": Fail,
- "FailedReleasing": Fail,
- "FailedDiskErasing": Fail,
- "FailedDeployment": Fail,
- "Broken": Fail,
- "FailedCommissioning": Fail,
+ "New": []Action{Reset, Commission},
+ "Deployed": []Action{Provision, Done},
+ "Ready": []Action{Reset, Aquire},
+ "Allocated": []Action{Reset, Deploy},
+ "Retired": []Action{Reset, AdminState},
+ "Reserved": []Action{Reset, AdminState},
+ "Releasing": []Action{Reset, Wait},
+ "DiskErasing": []Action{Reset, Wait},
+ "Deploying": []Action{Reset, Wait},
+ "Commissioning": []Action{Reset, Wait},
+ "Missing": []Action{Reset, Fail},
+ "FailedReleasing": []Action{Reset, Fail},
+ "FailedDiskErasing": []Action{Reset, Fail},
+ "FailedDeployment": []Action{Reset, Fail},
+ "Broken": []Action{Reset, Fail},
+ "FailedCommissioning": []Action{Reset, Fail},
},
}
@@ -87,7 +95,11 @@
(FailedEraseDisk)->(Broken)
(Releasing)->(Ready)
(DiskErasing)->(Ready)
- (Broken)->(Ready)`
+ (Broken)->(Ready)
+ (Deployed)->(Provisioning)
+ (Provisioning)->(ProvisionError)
+ (ProvisionError)->(Provisioning)
+ (Provisioning)->(Provisioned)`
)
// updateName - changes the name of the MAAS node based on the configuration file
@@ -115,6 +127,178 @@
return nil
}
+// Reset we are at the target state, nothing to do
+var Reset = func(client *maas.MAASObject, node MaasNode, options ProcessingOptions) error {
+ if options.Verbose {
+ log.Printf("RESET: %s", node.Hostname())
+ }
+
+ if options.AlwaysRename {
+ updateNodeName(client, node, options)
+ }
+
+ options.ProvTracker.Clear(node.ID())
+
+ return nil
+}
+
+// Provision we are at the target state, nothing to do
+var Provision = func(client *maas.MAASObject, node MaasNode, options ProcessingOptions) error {
+ if options.Verbose {
+ log.Printf("CHECK PROVISION: %s", node.Hostname())
+ }
+
+ if options.AlwaysRename {
+ updateNodeName(client, node, options)
+ }
+
+ record, err := options.ProvTracker.Get(node.ID())
+ if options.Verbose {
+ log.Printf("[info] Current state of node '%s' is '%s'", node.Hostname(), record.State.String())
+ }
+ if err != nil {
+ log.Printf("[warn] unable to retrieve provisioning state of node '%s' : %s", node.Hostname(), err)
+ } else if record.State == Unprovisioned || record.State == ProvisionError {
+ var err error = nil
+ var callout *url.URL
+ log.Printf("PROVISION '%s'", node.Hostname())
+ if len(options.ProvisionURL) > 0 {
+ if options.Verbose {
+ log.Printf("[info] Provisioning callout to '%s'", options.ProvisionURL)
+ }
+ callout, err = url.Parse(options.ProvisionURL)
+ if err != nil {
+ log.Printf("[error] Failed to parse provisioning URL '%s' : %s", options.ProvisionURL, err)
+ } else {
+ ips := node.IPs()
+ ip := ""
+ if len(ips) > 0 {
+ ip = ips[0]
+ }
+ switch callout.Scheme {
+ // If the scheme is a file, then we will execute the refereced file
+ case "", "file":
+ if options.Verbose {
+ log.Printf("[info] executing local script file '%s'", callout.Path)
+ }
+ record.State = Provisioning
+ record.Timestamp = time.Now().Unix()
+ options.ProvTracker.Set(node.ID(), record)
+ err = exec.Command(callout.Path, node.ID(), node.Hostname(), ip).Run()
+ if err != nil {
+ log.Printf("[error] Failed to execute '%s' : %s", options.ProvisionURL, err)
+ } else {
+ if options.Verbose {
+ log.Printf("[info] Marking node '%s' with ID '%s' as provisioned",
+ node.Hostname(), node.ID())
+ }
+ record.State = Provisioned
+ options.ProvTracker.Set(node.ID(), record)
+ }
+
+ default:
+ if options.Verbose {
+ log.Printf("[info] POSTing to '%s'", options.ProvisionURL)
+ }
+ data := map[string]string{
+ "id": node.ID(),
+ "name": node.Hostname(),
+ "ip": ip,
+ }
+ hc := http.Client{}
+ var b []byte
+ b, err = json.Marshal(data)
+ if err != nil {
+ log.Printf("[error] Unable to marshal node data : %s", err)
+ } else {
+ var req *http.Request
+ var resp *http.Response
+ if options.Verbose {
+ log.Printf("[debug] POSTing data '%s'", string(b))
+ }
+ req, err = http.NewRequest("POST", options.ProvisionURL, bytes.NewReader(b))
+ if err != nil {
+ log.Printf("[error] Unable to construct POST request to provisioner : %s",
+ err)
+ } else {
+ req.Header.Add("Content-Type", "application/json")
+ resp, err = hc.Do(req)
+ if err != nil {
+ log.Printf("[error] Unable to process POST request : %s",
+ err)
+ } else {
+ defer resp.Body.Close()
+ if resp.StatusCode == 202 {
+ record.State = Provisioning
+ } else {
+ record.State = ProvisionError
+ }
+ options.ProvTracker.Set(node.ID(), record)
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if err != nil {
+ if options.Verbose {
+ log.Printf("[warn] Not marking node '%s' with ID '%s' as provisioned, because of error '%s'",
+ node.Hostname(), node.ID(), err)
+ record.State = ProvisionError
+ options.ProvTracker.Set(node.ID(), record)
+ }
+ }
+ } else if record.State == Provisioning && time.Since(time.Unix(record.Timestamp, 0)) > options.ProvisionTTL {
+ log.Printf("[error] Provisioning of node '%s' has passed provisioning TTL of '%v'",
+ node.Hostname(), options.ProvisionTTL)
+ record.State = ProvisionError
+ options.ProvTracker.Set(node.ID(), record)
+ } else if record.State == Provisioning {
+ callout, err := url.Parse(options.ProvisionURL)
+ if err != nil {
+ log.Printf("[error] Unable to parse provisioning URL '%s' : %s", options.ProvisionURL, err)
+ } else if callout.Scheme != "file" {
+ var req *http.Request
+ var resp *http.Response
+ if options.Verbose {
+ log.Printf("[info] Fetching provisioning state for node '%s'", node.Hostname())
+ }
+ req, err = http.NewRequest("GET", options.ProvisionURL+"/"+node.ID(), nil)
+ if err != nil {
+ log.Printf("[error] Unable to construct GET request to provisioner : %s", err)
+ } else {
+ hc := http.Client{}
+ resp, err = hc.Do(req)
+ if err != nil {
+ log.Printf("[error] Failed to quest provision state for node '%s' : %s",
+ node.Hostname(), err)
+ } else {
+ switch resp.StatusCode {
+ case 200: // OK - provisioning completed
+ if options.Verbose {
+ log.Printf("[info] Marking node '%s' with ID '%s' as provisioned",
+ node.Hostname(), node.ID())
+ }
+ record.State = Provisioned
+ options.ProvTracker.Set(node.ID(), record)
+ case 202: // Accepted - in the provisioning state
+ // Noop, presumably alread in this state
+ default: // Consider anything else an erorr
+ record.State = ProvisionError
+ options.ProvTracker.Set(node.ID(), record)
+ }
+ }
+ }
+ }
+ } else if options.Verbose {
+ log.Printf("[info] Not invoking provisioning for '%s', currned state is '%s'", node.Hostname(),
+ record.State.String())
+ }
+
+ return nil
+}
+
// Done we are at the target state, nothing to do
var Done = func(client *maas.MAASObject, node MaasNode, options ProcessingOptions) error {
// As devices are normally in the "COMPLETED" state we don't want to
@@ -145,7 +329,7 @@
myNode := nodesObj.GetSubObject(node.ID())
// Start the node with the trusty distro. This should really be looked up or
// a parameter default
- _, err := myNode.CallPost("start", url.Values {"distro_series" : []string{"trusty"}})
+ _, err := myNode.CallPost("start", url.Values{"distro_series": []string{"trusty"}})
if err != nil {
log.Printf("ERROR: DEPLOY '%s' : '%s'", node.Hostname(), err)
return err
@@ -269,10 +453,10 @@
// Attempt to turn the node off
log.Printf("POWER DOWN: %s", node.Hostname())
if !options.Preview {
- //POST /api/1.0/nodes/{system_id}/ op=stop
+ //POST /api/1.0/nodes/{system_id}/ op=stop
nodesObj := client.GetSubObject("nodes")
nodeObj := nodesObj.GetSubObject(node.ID())
- _, err := nodeObj.CallPost("stop", url.Values{"stop_mode" : []string{"soft"}})
+ _, err := nodeObj.CallPost("stop", url.Values{"stop_mode": []string{"soft"}})
if err != nil {
log.Printf("ERROR: Commission '%s' : changing power start to off : '%s'", node.Hostname(), err)
}
@@ -321,14 +505,14 @@
return nil
}
-func findAction(target string, current string) (Action, error) {
+func findActions(target string, current string) ([]Action, error) {
targets, ok := Transitions[target]
if !ok {
log.Printf("[warn] unable to find transitions to target state '%s'", target)
return nil, fmt.Errorf("Could not find transition to target state '%s'", target)
}
- action, ok := targets[current]
+ actions, ok := targets[current]
if !ok {
log.Printf("[warn] unable to find transition from current state '%s' to target state '%s'",
current, target)
@@ -336,7 +520,19 @@
current, target)
}
- return action, nil
+ return actions, nil
+}
+
+// ProcessActions
+func ProcessActions(actions []Action, client *maas.MAASObject, node MaasNode, options ProcessingOptions) error {
+ var err error
+ for _, action := range actions {
+ if err = action(client, node, options); err != nil {
+ log.Printf("[error] Error while processing action for node '%s' : %s", node.Hostname, err)
+ break
+ }
+ }
+ return err
}
// ProcessNode something
@@ -345,15 +541,15 @@
if err != nil {
return err
}
- action, err := findAction("Deployed", MaasNodeStatus(substatus).String())
+ actions, err := findActions("Deployed", MaasNodeStatus(substatus).String())
if err != nil {
return err
}
if options.Preview {
- action(client, node, options)
+ ProcessActions(actions, client, node, options)
} else {
- go action(client, node, options)
+ go ProcessActions(actions, client, node, options)
}
return nil
}