1 package reaper
2
3 import (
4 "math"
5 "os"
6 "path"
7 "strings"
8 "sync"
9 "time"
10
11 "github.com/cybertec-postgresql/pgwatch/v3/internal/metrics"
12 "github.com/shirou/gopsutil/v4/cpu"
13 "github.com/shirou/gopsutil/v4/disk"
14 "github.com/shirou/gopsutil/v4/load"
15 "github.com/shirou/gopsutil/v4/mem"
16 )
17
18
19 var prevCPULoadTimeStatsLock sync.RWMutex
20 var prevCPULoadTimeStats cpu.TimesStat
21 var prevCPULoadTimestamp time.Time
22
23 func goPsutilCalcCPUUtilization(probe0, probe1 cpu.TimesStat) float64 {
24 return 100 - (100.0 * (probe1.Idle - probe0.Idle + probe1.Iowait - probe0.Iowait + probe1.Steal - probe0.Steal) / (probe1.Total() - probe0.Total()))
25 }
26
27
28 func GetGoPsutilCPU(interval time.Duration) ([]map[string]any, error) {
29 prevCPULoadTimeStatsLock.RLock()
30 prevTime := prevCPULoadTimestamp
31 prevTimeStat := prevCPULoadTimeStats
32 prevCPULoadTimeStatsLock.RUnlock()
33
34 if prevTime.IsZero() || (time.Now().UnixNano()-prevTime.UnixNano()) < 1e9 {
35 probe0, err := cpu.Times(false)
36 if err != nil {
37 return nil, err
38 }
39 prevTimeStat = probe0[0]
40 time.Sleep(1e9)
41 }
42
43 curCallStats, err := cpu.Times(false)
44 if err != nil {
45 return nil, err
46 }
47 if prevTime.IsZero() || time.Now().UnixNano()-prevTime.UnixNano() < 1e9 || time.Now().Unix()-prevTime.Unix() >= int64(interval.Seconds()) {
48 prevCPULoadTimeStatsLock.Lock()
49 prevCPULoadTimeStats = curCallStats[0]
50 prevCPULoadTimestamp = time.Now()
51 prevCPULoadTimeStatsLock.Unlock()
52 }
53
54 la, err := load.Avg()
55 if err != nil {
56 return nil, err
57 }
58
59 cpus, err := cpu.Counts(true)
60 if err != nil {
61 return nil, err
62 }
63
64 retMap := metrics.NewMeasurement(time.Now().UnixNano())
65 retMap["cpu_utilization"] = math.Round(100*goPsutilCalcCPUUtilization(prevTimeStat, curCallStats[0])) / 100
66 retMap["load_1m_norm"] = math.Round(100*la.Load1/float64(cpus)) / 100
67 retMap["load_1m"] = math.Round(100*la.Load1) / 100
68 retMap["load_5m_norm"] = math.Round(100*la.Load5/float64(cpus)) / 100
69 retMap["load_5m"] = math.Round(100*la.Load5) / 100
70 retMap["user"] = math.Round(10000.0*(curCallStats[0].User-prevTimeStat.User)/(curCallStats[0].Total()-prevTimeStat.Total())) / 100
71 retMap["system"] = math.Round(10000.0*(curCallStats[0].System-prevTimeStat.System)/(curCallStats[0].Total()-prevTimeStat.Total())) / 100
72 retMap["idle"] = math.Round(10000.0*(curCallStats[0].Idle-prevTimeStat.Idle)/(curCallStats[0].Total()-prevTimeStat.Total())) / 100
73 retMap["iowait"] = math.Round(10000.0*(curCallStats[0].Iowait-prevTimeStat.Iowait)/(curCallStats[0].Total()-prevTimeStat.Total())) / 100
74 retMap["irqs"] = math.Round(10000.0*(curCallStats[0].Irq-prevTimeStat.Irq+curCallStats[0].Softirq-prevTimeStat.Softirq)/(curCallStats[0].Total()-prevTimeStat.Total())) / 100
75 retMap["other"] = math.Round(10000.0*(curCallStats[0].Steal-prevTimeStat.Steal+curCallStats[0].Guest-prevTimeStat.Guest+curCallStats[0].GuestNice-prevTimeStat.GuestNice)/(curCallStats[0].Total()-prevTimeStat.Total())) / 100
76
77 return []map[string]any{retMap}, nil
78 }
79
80 func GetGoPsutilMem() ([]map[string]any, error) {
81 vm, err := mem.VirtualMemory()
82 if err != nil {
83 return nil, err
84 }
85
86 retMap := metrics.NewMeasurement(time.Now().UnixNano())
87 retMap["total"] = int64(vm.Total)
88 retMap["used"] = int64(vm.Used)
89 retMap["free"] = int64(vm.Free)
90 retMap["buff_cache"] = int64(vm.Buffers)
91 retMap["available"] = int64(vm.Available)
92 retMap["percent"] = math.Round(100*vm.UsedPercent) / 100
93 retMap["swap_total"] = int64(vm.SwapTotal)
94 retMap["swap_used"] = int64(vm.SwapCached)
95 retMap["swap_free"] = int64(vm.SwapFree)
96 retMap["swap_percent"] = math.Round(100*float64(vm.SwapCached)/float64(vm.SwapTotal)) / 100
97
98 return []map[string]any{retMap}, nil
99 }
100
101 func GetGoPsutilDiskTotals() ([]map[string]any, error) {
102 d, err := disk.IOCounters()
103 if err != nil {
104 return nil, err
105 }
106
107 retMap := metrics.NewMeasurement(time.Now().UnixNano())
108 var readBytes, writeBytes, reads, writes float64
109
110 for _, v := range d {
111 readBytes += float64(v.ReadBytes)
112
113 writeBytes += float64(v.WriteBytes)
114 reads += float64(v.ReadCount)
115 writes += float64(v.WriteCount)
116 }
117 retMap["read_bytes"] = readBytes
118 retMap["write_bytes"] = writeBytes
119 retMap["read_count"] = reads
120 retMap["write_count"] = writes
121
122 return []map[string]any{retMap}, nil
123 }
124
125 func GetLoadAvgLocal() ([]map[string]any, error) {
126 la, err := load.Avg()
127 if err != nil {
128 return nil, err
129 }
130
131 row := metrics.NewMeasurement(time.Now().UnixNano())
132 row["load_1min"] = la.Load1
133 row["load_5min"] = la.Load5
134 row["load_15min"] = la.Load15
135
136 return []map[string]any{row}, nil
137 }
138
139 func CheckFolderExistsAndReadable(path string) bool {
140 _, err := os.ReadDir(path)
141 return err == nil
142 }
143
144 func GetGoPsutilDiskPG(DataDirs, TblspaceDirs []map[string]any) ([]map[string]any, error) {
145 var ddDevice, ldDevice, walDevice uint64
146
147 data := DataDirs
148
149 dataDirPath := data[0]["dd"].(string)
150 ddUsage, err := disk.Usage(dataDirPath)
151 if err != nil {
152 return nil, err
153 }
154
155 retRows := make([]map[string]any, 0)
156 epochNs := time.Now().UnixNano()
157 dd := metrics.NewMeasurement(epochNs)
158 dd["tag_dir_or_tablespace"] = "data_directory"
159 dd["tag_path"] = dataDirPath
160 dd["total"] = float64(ddUsage.Total)
161 dd["used"] = float64(ddUsage.Used)
162 dd["free"] = float64(ddUsage.Free)
163 dd["percent"] = math.Round(100*ddUsage.UsedPercent) / 100
164 retRows = append(retRows, dd)
165
166 ddDevice, err = GetPathUnderlyingDeviceID(dataDirPath)
167 if err != nil {
168 return nil, err
169 }
170
171 logDirPath := data[0]["ld"].(string)
172 if !strings.HasPrefix(logDirPath, "/") {
173 logDirPath = path.Join(dataDirPath, logDirPath)
174 }
175 if len(logDirPath) > 0 && CheckFolderExistsAndReadable(logDirPath) {
176 ldDevice, err = GetPathUnderlyingDeviceID(logDirPath)
177 if err != nil {
178 return nil, err
179 }
180 if ldDevice != ddDevice {
181 ld := metrics.NewMeasurement(epochNs)
182 ldUsage, err := disk.Usage(logDirPath)
183 if err != nil {
184 return nil, err
185 }
186
187 ld["tag_dir_or_tablespace"] = "log_directory"
188 ld["tag_path"] = logDirPath
189 ld["total"] = float64(ldUsage.Total)
190 ld["used"] = float64(ldUsage.Used)
191 ld["free"] = float64(ldUsage.Free)
192 ld["percent"] = math.Round(100*ldUsage.UsedPercent) / 100
193 retRows = append(retRows, ld)
194 }
195 }
196
197 var walDirPath string
198 if CheckFolderExistsAndReadable(path.Join(dataDirPath, "pg_wal")) {
199 walDirPath = path.Join(dataDirPath, "pg_wal")
200 } else if CheckFolderExistsAndReadable(path.Join(dataDirPath, "pg_xlog")) {
201 walDirPath = path.Join(dataDirPath, "pg_xlog")
202 }
203
204 if len(walDirPath) > 0 {
205 walDevice, err = GetPathUnderlyingDeviceID(walDirPath)
206 if err != nil {
207 return nil, err
208 }
209
210 if walDevice != ddDevice || walDevice != ldDevice {
211 walUsage, err := disk.Usage(walDirPath)
212 if err != nil {
213 return nil, err
214 }
215
216 wd := metrics.NewMeasurement(epochNs)
217 wd["tag_dir_or_tablespace"] = "pg_wal"
218 wd["tag_path"] = walDirPath
219 wd["total"] = float64(walUsage.Total)
220 wd["used"] = float64(walUsage.Used)
221 wd["free"] = float64(walUsage.Free)
222 wd["percent"] = math.Round(100*walUsage.UsedPercent) / 100
223 retRows = append(retRows, wd)
224 }
225 }
226
227 data = TblspaceDirs
228 if len(data) > 0 {
229 for _, row := range data {
230 tsPath := row["location"].(string)
231 tsName := row["name"].(string)
232
233 tsDevice, err := GetPathUnderlyingDeviceID(tsPath)
234 if err != nil {
235 return nil, err
236 }
237
238 if tsDevice == ddDevice || tsDevice == ldDevice || tsDevice == walDevice {
239 continue
240 }
241 tsUsage, err := disk.Usage(tsPath)
242 if err != nil {
243 return nil, err
244 }
245 ts := metrics.NewMeasurement(epochNs)
246 ts["tag_dir_or_tablespace"] = tsName
247 ts["tag_path"] = tsPath
248 ts["total"] = float64(tsUsage.Total)
249 ts["used"] = float64(tsUsage.Used)
250 ts["free"] = float64(tsUsage.Free)
251 ts["percent"] = math.Round(100*tsUsage.UsedPercent) / 100
252 retRows = append(retRows, ts)
253 }
254 }
255
256 return retRows, nil
257 }
258