Add code to track and report job runtime stats

This commit is contained in:
Russ Magee 2023-08-06 18:01:16 -07:00
parent 20bfd92e65
commit 48a92284e8
2 changed files with 45 additions and 13 deletions

View File

@ -1,6 +1,6 @@
MAKEOPTS = $(MAKEOPTS)
GIT_COMMIT := $(shell git rev-list -1 HEAD)
VERSION := 0.2.9.3
VERSION := 0.2.9.4-dev
BUILDOPTS :=$(BUILDOPTS) -ldflags "-X main.version=$(VERSION) -X main.gitCommit=$(GIT_COMMIT)"
.PHONY: install all clean

View File

@ -42,8 +42,10 @@ const (
)
type cmdItem struct {
script string
lastStatus int
script string
lastRunStartTime time.Time
lastRunDuration time.Duration
lastStatus int
}
var (
@ -591,12 +593,12 @@ func manualJobTriggersHTML(fullLogLink bool) (ret string) {
ret += fmt.Sprintf("-- job script %s not found --\n", cmdMap[jobName].script)
} else {
if isParameterizedBuildScript(cmdMap[jobName].script) {
ret += fmt.Sprintf("<a class='xhrlink' title='Play Job with Parameters' href='%s?param'>[&rtri;] %s [action %s] [last status: %d]</a>\n",
jobName, jobName, cmdMap[jobName].script, cmdMap[jobName].lastStatus)
ret += fmt.Sprintf("<a class='xhrlink' title='Play Job with Parameters' href='%s?param'>[&rtri;] %s [action %s] [last status: %d/%v]</a>\n",
jobName, jobName, cmdMap[jobName].script, cmdMap[jobName].lastStatus, cmdMap[jobName].lastRunDuration)
} else {
fn := strings.Replace(jobName, "-", "", -1)
ret += fmt.Sprintf(`<a class='xhrlink' onclick='%s(); return false;' title='Play Job' href='%s'>[&rtrif;] %s [action %s] [last status: %d]</a>`+"\n",
fn, jobName, jobName, cmdMap[jobName].script, cmdMap[jobName].lastStatus)
ret += fmt.Sprintf(`<a class='xhrlink' onclick='%s(); return false;' title='Play Job' href='%s'>[&rtrif;] %s [action %s] [last status: %d/%v]</a>`+"\n",
fn, jobName, jobName, cmdMap[jobName].script, cmdMap[jobName].lastStatus, cmdMap[jobName].lastRunDuration)
}
}
}
@ -920,6 +922,10 @@ func execJob(j jobCtx, hookData hookEvt) {
strings.Replace(workerOutputRelPath, jobHomeDir, "/"+jobHomeDir+"/fullconsole", 1)) //nolint:errcheck
fmt.Fprintf(c.Stdout, "%s\n", j.jobTag) //nolint:errcheck
tmp := cmdMap[j.jobTag]
tmp.lastRunStartTime = time.Now()
cmdMap[j.jobTag] = tmp
cerr := c.Start()
if cerr != nil {
log.Printf("[exec.Cmd: %+v]\n", c)
@ -951,6 +957,10 @@ func execJob(j jobCtx, hookData hookEvt) {
}
werr := c.Wait()
tmp = cmdMap[j.jobTag]
tmp.lastRunDuration = time.Since(tmp.lastRunStartTime).Truncate(100 * time.Millisecond)
cmdMap[j.jobTag] = tmp
var exitStatus int
if werr, ok := werr.(*exec.ExitError); ok {
// The program has exited with an exit code != 0
@ -990,26 +1000,32 @@ func execJob(j jobCtx, hookData hookEvt) {
}
if werr == nil {
log.Printf("<!--JOBID:%s:JOBID--><span style='background-color:%s'><a href='%s' title='Done'>[&check;]</a>%s[%s{%s}<a href='/artifacts/bacillus_%s_%s_%s/' title='Artifacts'>[&ccupssm;]</a> completed with status 0]%s</span><!--COMPLETION-->\n",
log.Printf("<!--JOBID:%s:JOBID--><span style='background-color:%s'><a href='%s' title='Done'>[&check;]</a>%s[%s{%s}<a href='/artifacts/bacillus_%s_%s_%s/' title='Artifacts'>[&ccupssm;]</a> completed with status 0/%v]%s</span><!--COMPLETION-->\n",
jobID, instColour,
workerOutputRelPath,
indentStr,
j.jobTag, jobID,
j.jobOpts, j.jobTag, jobID,
cmdMap[j.jobTag].lastRunDuration,
stageStr)
// Update job last completion status
cmdMap[j.jobTag] = cmdItem{script: cmdMap[j.jobTag].script, lastStatus: int(0)}
tmp := cmdMap[j.jobTag]
tmp.lastStatus = int(0)
cmdMap[j.jobTag] = tmp
} else {
log.Printf("<!--JOBID:%s:JOBID--><span style='background-color:%s'><span style='background-color:red'><a href='%s' title='Done With Errors'>[!]</a></span>%s[%s{%s}<a href='/artifacts/bacillus_%s_%s_%s/' title='Partial Artifacts'>[&ccups;]</a> completed with error %s]%s</span><!--COMPLETION-->\n",
log.Printf("<!--JOBID:%s:JOBID--><span style='background-color:%s'><span style='background-color:red'><a href='%s' title='Done With Errors'>[!]</a></span>%s[%s{%s}<a href='/artifacts/bacillus_%s_%s_%s/' title='Partial Artifacts'>[&ccups;]</a> completed with error %s/%v]%s</span><!--COMPLETION-->\n",
jobID, instColour,
workerOutputRelPath,
indentStr,
j.jobTag, jobID,
j.jobOpts, j.jobTag, jobID,
werr,
cmdMap[j.jobTag].lastRunDuration,
stageStr)
// Update job last completion status
cmdMap[j.jobTag] = cmdItem{script: cmdMap[j.jobTag].script, lastStatus: int(exitStatus)}
tmp := cmdMap[j.jobTag]
tmp.lastStatus = int(exitStatus)
cmdMap[j.jobTag] = tmp
}
delete(runningJobs, jobID)
}
@ -1023,7 +1039,7 @@ func jobLastStatusHandler(w http.ResponseWriter, r *http.Request) {
slugs := strings.Split(r.URL.String(), "/")
if len(slugs) >= 2 {
tag := slugs[len(slugs)-2]
writeStr(w, fmt.Sprintf("%s lastStatus:%d", tag, cmdMap[tag].lastStatus))
writeStr(w, fmt.Sprintf("%d", cmdMap[tag].lastStatus))
} else {
writeStr(w, "error")
}
@ -1050,6 +1066,20 @@ func jobLastStatusIconHandler(w http.ResponseWriter, r *http.Request) {
}
}
func jobLastRunDurationHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-type", "text/html;charset=UTF-8")
if !httpAuthSession(w, r) {
return
}
slugs := strings.Split(r.URL.String(), "/")
if len(slugs) >= 2 {
tag := slugs[len(slugs)-2]
writeStr(w, fmt.Sprintf("%v", cmdMap[tag].lastRunDuration))
} else {
writeStr(w, "error")
}
}
func jobCancelHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-type", "text/html;charset=UTF-8")
if !httpAuthSession(w, r) {
@ -1574,9 +1604,11 @@ func main() {
// it isn't read).
launchJobListener(mainCtx, cmd, tag, jobOpts, jobEnv, cmdMap)
// Endpoint to return lastStatus for the job
// Endpoint to return lastStatus value and success/fail badge for the job
http.HandleFunc("/"+tag+"/lastStatus", jobLastStatusHandler)
http.HandleFunc("/"+tag+"/lastStatusIcon", jobLastStatusIconHandler)
// Endpoint to return last run duration for the job
http.HandleFunc("/"+tag+"/lastRunDuration", jobLastRunDurationHandler)
}
}