Skip to content

Commit fbda374

Browse files
dgruberdgruber
authored
EH: Add qstat -f and fix qhost (#32)
* EH: Watch qacct file; improved types * BF: Fixing go.mod file for simulator * EH: Add helper function and example for flex accounting * Add qstat -f and fix qhost
1 parent 040035e commit fbda374

File tree

8 files changed

+331
-76
lines changed

8 files changed

+331
-76
lines changed

‎pkg/qhost/v9.0/parsers.go

-1
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,6 @@ func ParseHostFullMetrics(out string) ([]HostFullMetrics, error) {
148148
for i := 0; i < len(lines); i++ {
149149
line := strings.TrimSpace(lines[i])
150150
if line == "" || strings.HasPrefix(line, "HOSTNAME") ||
151-
strings.HasPrefix(line, "global") ||
152151
strings.HasPrefix(line, "----") {
153152
continue
154153
}

‎pkg/qhost/v9.0/parsers_test.go

+58-4
Original file line numberDiff line numberDiff line change
@@ -404,19 +404,73 @@ sim9 lx-amd64 4 1 4 4 0.60 15.6G 465.8M
404404
hl:np_load_long=0.110000
405405
hf:load_report_host=master`
406406

407+
qhostFOutput2 := `HOSTNAME ARCH NCPU NSOC NCOR NTHR LOAD MEMTOT MEMUSE SWAPTO SWAPUS
408+
----------------------------------------------------------------------------------------------
409+
global - - - - - - - - - -
410+
gc:testc=100000.000000
411+
master lx-amd64 14 1 14 14 1.50 7.7G 2.0G 1024.0M 12.0K
412+
gc:testc=100000.000000
413+
hl:load_avg=1.500000
414+
hl:load_short=1.670000
415+
hl:load_medium=1.500000
416+
hl:load_long=1.100000
417+
hl:arch=lx-amd64
418+
hl:num_proc=14.000000
419+
hl:mem_free=5.621G
420+
hl:swap_free=1023.984M
421+
hl:virtual_free=6.621G
422+
hl:mem_total=7.653G
423+
hl:swap_total=1023.996M
424+
hl:virtual_total=8.653G
425+
hl:mem_used=2.032G
426+
hl:swap_used=12.000K
427+
hl:virtual_used=2.032G
428+
hl:cpu=0.500000
429+
hl:m_topology=SCCCCCCCCCCCCCC
430+
hl:m_topology_inuse=SCCCCCCCCCCCCCC
431+
hl:m_socket=1.000000
432+
hl:m_core=14.000000
433+
hl:m_thread=14.000000
434+
hl:np_load_avg=0.107143
435+
hl:np_load_short=0.119286
436+
hl:np_load_medium=0.107143
437+
hl:np_load_long=0.078571
438+
`
439+
407440
It("should return error if output is invalid", func() {
408441
hosts, err := qhost.ParseHostFullMetrics(sample)
409442
Expect(err).To(BeNil())
410-
Expect(hosts).To(HaveLen(2))
443+
Expect(hosts).To(HaveLen(3))
411444
})
412445

413446
It("should parse host full metrics", func() {
414447
hosts, err := qhost.ParseHostFullMetrics(qhostFOutput1)
415448
Expect(err).To(BeNil())
416-
Expect(hosts).To(HaveLen(13))
417-
Expect(hosts[0].Name).To(Equal("master"))
418-
Expect(hosts[12].Name).To(Equal("sim9"))
449+
Expect(hosts).To(HaveLen(14))
450+
Expect(hosts[0].Name).To(Equal("global"))
451+
Expect(hosts[1].Name).To(Equal("master"))
452+
Expect(hosts[12].Name).To(Equal("sim8"))
453+
})
454+
455+
It("should parse host full metrics with global host values", func() {
456+
hosts, err := qhost.ParseHostFullMetrics(qhostFOutput2)
457+
Expect(err).To(BeNil())
458+
Expect(hosts).To(HaveLen(2))
459+
Expect(hosts[0].Name).To(Equal("global"))
460+
Expect(hosts[1].Name).To(Equal("master"))
461+
Expect(len(hosts[0].Resources)).To(Equal(1))
462+
Expect(hosts[0].Resources["testc"]).To(Equal(
463+
qhost.ResourceAvailability{
464+
Name: "testc",
465+
StringValue: "100000.000000",
466+
FloatValue: 100000.000000,
467+
ResourceAvailabilityLimitedBy: "g",
468+
Source: "c",
469+
FullString: "gc:testc=100000.000000",
470+
},
471+
))
419472
})
473+
420474
})
421475

422476
})

‎pkg/qstat/v9.0/parser.go

+193-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"strconv"
2727
"strings"
2828
"time"
29+
"unicode"
2930
)
3031

3132
const QstatDateFormat = "2006-01-02 03:04:05"
@@ -68,7 +69,9 @@ func parseFixedWidthJobs(input string) ([]ParallelJobTask, error) {
6869
return tasks, nil
6970
}
7071

71-
// Correct column positions based on your description
72+
// we have 9.0.0 to 9.0.2 format and 9.0.3 format with 3 more columns
73+
// for the job IDs
74+
7275
columnPositions := []struct {
7376
start int
7477
end int
@@ -942,3 +945,192 @@ func ParseJobArrayTask(out string) ([]JobArrayTask, error) {
942945
}
943946
return jobArrayTasks, nil
944947
}
948+
949+
/*
950+
qstat -f
951+
queuename qtype resv/used/tot. load_avg arch states
952+
---------------------------------------------------------------------------------
953+
all.q@master BIP 0/9/14 0.69 lx-amd64
954+
2 0.50500 sleep root r 2025-02-15 12:28:22 1
955+
3 0.50500 sleep root r 2025-02-15 12:28:23 1
956+
4 0.50500 sleep root r 2025-02-15 12:28:23 1
957+
5 0.50500 sleep root r 2025-02-15 12:28:24 1
958+
6 0.50500 sleep root r 2025-02-15 12:28:24 1
959+
7 0.50500 sleep root r 2025-02-15 12:28:25 1
960+
8 0.50500 sleep root r 2025-02-15 12:28:25 1
961+
12 0.60500 sleep root r 2025-02-15 12:29:31 2
962+
---------------------------------------------------------------------------------
963+
test.q@master BIP 0/6/10 0.69 lx-amd64
964+
9 0.50500 sleep root r 2025-02-15 12:28:34 1
965+
10 0.50500 sleep root r 2025-02-15 12:28:38 1
966+
11 0.50500 sleep root r 2025-02-15 12:29:03 1 1
967+
11 0.50500 sleep root r 2025-02-15 12:29:03 1 2
968+
13 0.60500 sleep root r 2025-02-15 12:29:35 2
969+
*/
970+
971+
// ParseQstatFullOutput parses the output of the "qstat -f" command and returns
972+
// a slice of FullQueueInfo containing queue details and associated job information.
973+
//
974+
// It expects an output with queue header lines (non-indented) followed by one or more
975+
// job lines (indented) until a separator (a line full of "-" characters) is encountered.
976+
func ParseQstatFullOutput(out string) ([]FullQueueInfo, error) {
977+
lines := strings.Split(out, "\n")
978+
var results []FullQueueInfo
979+
var currentQueue *FullQueueInfo
980+
981+
for _, line := range lines {
982+
trimmed := strings.TrimSpace(line)
983+
if trimmed == "" {
984+
continue
985+
}
986+
if strings.HasPrefix(trimmed, "####") {
987+
break
988+
}
989+
990+
// Skip any known header lines.
991+
lower := strings.ToLower(trimmed)
992+
if strings.HasPrefix(lower, "queuename") {
993+
continue
994+
}
995+
996+
// If this is a separator line, then finish the current block.
997+
if isSeparatorLine(trimmed) {
998+
if currentQueue != nil {
999+
results = append(results, *currentQueue)
1000+
currentQueue = nil
1001+
}
1002+
continue
1003+
}
1004+
1005+
// If the line does not start with whitespace, it is a queue header.
1006+
if !startsWithWhitespace(line) {
1007+
// If an active queue exists, push it into results before starting a new block.
1008+
if currentQueue != nil {
1009+
results = append(results, *currentQueue)
1010+
}
1011+
1012+
fields := strings.Fields(line)
1013+
if len(fields) < 5 {
1014+
return nil, fmt.Errorf("invalid queue header format: %q", line)
1015+
}
1016+
queueName := fields[0]
1017+
qtype := fields[1]
1018+
resvUsedTot := fields[2] // Expected format: "resv/used/tot"
1019+
loadAvgStr := fields[3]
1020+
arch := fields[4]
1021+
1022+
parts := strings.Split(resvUsedTot, "/")
1023+
if len(parts) != 3 {
1024+
return nil, fmt.Errorf("invalid resv/used/tot format in queue header: %q", line)
1025+
}
1026+
reserved, err := strconv.Atoi(parts[0])
1027+
if err != nil {
1028+
return nil, fmt.Errorf("invalid reserved value in queue header: %v", err)
1029+
}
1030+
used, err := strconv.Atoi(parts[1])
1031+
if err != nil {
1032+
return nil, fmt.Errorf("invalid used value in queue header: %v", err)
1033+
}
1034+
total, err := strconv.Atoi(parts[2])
1035+
if err != nil {
1036+
return nil, fmt.Errorf("invalid total value in queue header: %v", err)
1037+
}
1038+
loadAvg, err := strconv.ParseFloat(loadAvgStr, 64)
1039+
if err != nil {
1040+
return nil, fmt.Errorf("invalid load_avg value in queue header: %v", err)
1041+
}
1042+
currentQueue = &FullQueueInfo{
1043+
QueueName: queueName,
1044+
QueueType: qtype,
1045+
Reserved: reserved,
1046+
Used: used,
1047+
Total: total,
1048+
LoadAvg: loadAvg,
1049+
Arch: arch,
1050+
Jobs: []JobInfo{},
1051+
}
1052+
} else {
1053+
// This is a job line. It must belong to an already parsed queue header.
1054+
if currentQueue == nil {
1055+
return nil, fmt.Errorf("job info found without preceding queue header: %q", line)
1056+
}
1057+
fields := strings.Fields(line)
1058+
if len(fields) < 8 {
1059+
return nil, fmt.Errorf("invalid job line format: %q", line)
1060+
}
1061+
jobID, err := strconv.Atoi(fields[0])
1062+
if err != nil {
1063+
return nil, fmt.Errorf("invalid job id in job line %q: %v", line, err)
1064+
}
1065+
score, err := strconv.ParseFloat(fields[1], 64)
1066+
if err != nil {
1067+
return nil, fmt.Errorf("invalid score in job line %q: %v", line, err)
1068+
}
1069+
taskName := fields[2]
1070+
owner := fields[3]
1071+
state := fields[4]
1072+
datetimeStr := fields[5] + " " + fields[6]
1073+
startTime, err := time.Parse("2006-01-02 15:04:05", datetimeStr)
1074+
if err != nil {
1075+
return nil, fmt.Errorf(
1076+
"failed to parse datetime '%s' in job line %q: %v",
1077+
datetimeStr, line, err)
1078+
}
1079+
var submitTime time.Time
1080+
if strings.Contains(state, "q") {
1081+
submitTime = startTime
1082+
startTime = time.Time{}
1083+
}
1084+
slots, err := strconv.Atoi(fields[7])
1085+
if err != nil {
1086+
return nil, fmt.Errorf("invalid slots in job line %q: %v", line, err)
1087+
}
1088+
// optional tasks
1089+
var taskIDs []int64
1090+
if len(fields) > 8 {
1091+
taskID, err := strconv.Atoi(fields[8])
1092+
if err != nil {
1093+
return nil, fmt.Errorf("invalid task id in job line %q: %v", line, err)
1094+
}
1095+
taskIDs = []int64{int64(taskID)}
1096+
}
1097+
job := JobInfo{
1098+
JobID: jobID,
1099+
Priority: score,
1100+
Name: taskName,
1101+
User: owner,
1102+
State: state,
1103+
StartTime: startTime,
1104+
SubmitTime: submitTime,
1105+
Queue: currentQueue.QueueName,
1106+
Slots: slots,
1107+
JaTaskIDs: taskIDs,
1108+
}
1109+
currentQueue.Jobs = append(currentQueue.Jobs, job)
1110+
}
1111+
}
1112+
1113+
// Append the last queue block if it exists.
1114+
if currentQueue != nil {
1115+
results = append(results, *currentQueue)
1116+
}
1117+
return results, nil
1118+
}
1119+
1120+
// startsWithWhitespace returns true if the first rune of the string is a whitespace.
1121+
func startsWithWhitespace(s string) bool {
1122+
for _, r := range s {
1123+
return unicode.IsSpace(r)
1124+
}
1125+
return false
1126+
}
1127+
1128+
// isSeparatorLine checks if the provided line is made up entirely of '-' characters.
1129+
func isSeparatorLine(s string) bool {
1130+
for _, r := range s {
1131+
if r != '-' {
1132+
return false
1133+
}
1134+
}
1135+
return true
1136+
}

0 commit comments

Comments
 (0)