1 package main
2
3 import (
4 "errors"
5 "io/fs"
6 "os"
7 "path"
8 "regexp"
9 "slices"
10 "strconv"
11 "strings"
12
13 "flag"
14 "fmt"
15
16 "gopkg.in/yaml.v3"
17 )
18
19 type (
20 ExtensionInfo struct {
21 ExtName string `yaml:"ext_name"`
22 ExtMinVersion string `yaml:"ext_min_version"`
23 }
24
25 ExtensionOverrides struct {
26 TargetMetric string `yaml:"target_metric"`
27 ExpectedExtensionVersions []ExtensionInfo `yaml:"expected_extension_versions"`
28 }
29
30 MetricAttrs struct {
31 IsInstanceLevel bool `yaml:"is_instance_level,omitempty"`
32 MetricStorageName string `yaml:"metric_storage_name,omitempty"`
33 ExtensionVersionOverrides []ExtensionOverrides `yaml:"extension_version_based_overrides,omitempty"`
34 IsPrivate bool `yaml:"is_private,omitempty"`
35 DisabledDays string `yaml:"disabled_days,omitempty"`
36 DisableTimes []string `yaml:"disabled_times,omitempty"`
37 StatementTimeoutSeconds int64 `yaml:"statement_timeout_seconds,omitempty"`
38 }
39
40 SQLs map[int]string
41
42 Metric struct {
43 SQLs SQLs
44 InitSQL string `yaml:"init_sql,omitempty"`
45 NodeStatus string `yaml:"node_status,omitempty"`
46 Gauges []string `yaml:",omitempty"`
47 MetricAttrs `yaml:",inline,omitempty"`
48 }
49
50 Metrics map[string]Metric
51 )
52
53 const (
54 FileBasedMetricHelpersDir = "00_helpers"
55 PresetConfigYAMLFile = "preset-configs.yaml"
56 )
57
58
59 func ReadMetricsFromFolder(folder string) (metricsMap Metrics, err error) {
60 metricFolders, err := os.ReadDir(folder)
61 if err != nil {
62 return
63 }
64
65 metricsMap = make(map[string]Metric)
66 metricNamePattern := `^[a-z0-9_\.]+$`
67 regexMetricNameFilter := regexp.MustCompile(metricNamePattern)
68 regexIsDigitOrPunctuation := regexp.MustCompile(`^[\d\.]+$`)
69
70 fmt.Printf("Searching for metrics from path %s ...\n", folder)
71
72 for _, metricFolder := range metricFolders {
73 if metricFolder.Name() == FileBasedMetricHelpersDir ||
74 !metricFolder.IsDir() ||
75 !regexMetricNameFilter.MatchString(metricFolder.Name()) {
76 continue
77 }
78
79 var versionFolders []fs.DirEntry
80 versionFolders, err = os.ReadDir(path.Join(folder, metricFolder.Name()))
81 if err != nil {
82 return
83 }
84
85 Metric := Metric{}
86 Metric.MetricAttrs, _ = ParseMetricAttrsFromYAML(path.Join(folder, metricFolder.Name(), "metric_attrs.yaml"))
87 _ = ParseMetricPrometheusAttrsFromYAML(path.Join(folder, metricFolder.Name(), "column_attrs.yaml"), &Metric)
88 Metric.SQLs = make(SQLs)
89
90 var version int
91 for _, versionFolder := range versionFolders {
92 if strings.HasSuffix(versionFolder.Name(), ".md") || versionFolder.Name() == "column_attrs.yaml" || versionFolder.Name() == "metric_attrs.yaml" {
93 continue
94 }
95 if !regexIsDigitOrPunctuation.MatchString(versionFolder.Name()) {
96 fmt.Printf("Invalid metric structure - version folder names should consist of only numerics/dots, found: %s", versionFolder.Name())
97 continue
98 }
99 if version, err = strconv.Atoi(versionFolder.Name()); err != nil {
100 version = 11
101 }
102
103 var metricDefs []fs.DirEntry
104 if metricDefs, err = os.ReadDir(path.Join(folder, metricFolder.Name(), versionFolder.Name())); err != nil {
105 return
106 }
107
108 for _, metricDef := range metricDefs {
109 if strings.HasPrefix(metricDef.Name(), "metric") && strings.HasSuffix(metricDef.Name(), ".sql") {
110 p := path.Join(folder, metricFolder.Name(), versionFolder.Name(), metricDef.Name())
111 metricSQL, err := os.ReadFile(p)
112 if err != nil {
113 continue
114 }
115 switch {
116 case strings.Contains(metricDef.Name(), "_master"):
117 Metric.NodeStatus = "primary"
118 case strings.Contains(metricDef.Name(), "_standby"):
119 Metric.NodeStatus = "standby"
120 }
121 Metric.SQLs[version] = strings.TrimRight(strings.TrimSpace(string(metricSQL)), ";")
122 }
123 }
124 }
125 metricsMap[metricFolder.Name()] = Metric
126 }
127 return
128 }
129
130 func ParseMetricPrometheusAttrsFromYAML(path string, m *Metric) (err error) {
131 type OldMetricPrometheusAttrs struct {
132 PrometheusGaugeColumns []string `yaml:"prometheus_gauge_columns,omitempty"`
133 PrometheusIgnoredColumns []string `yaml:"prometheus_ignored_columns,omitempty"`
134 PrometheusAllGaugeColumns bool `yaml:"prometheus_all_gauge_columns,omitempty"`
135 }
136
137 var val []byte
138 var oldPromAttrs OldMetricPrometheusAttrs
139 if val, err = os.ReadFile(path); err == nil {
140 if err = yaml.Unmarshal(val, &oldPromAttrs); err != nil {
141 return
142 }
143 if oldPromAttrs.PrometheusAllGaugeColumns {
144 m.Gauges = []string{"*"}
145 return
146 }
147 m.Gauges = slices.Clone(oldPromAttrs.PrometheusGaugeColumns)
148 }
149
150 return
151 }
152
153 func ParseMetricAttrsFromYAML(path string) (a MetricAttrs, err error) {
154 var val []byte
155 if val, err = os.ReadFile(path); err == nil {
156 err = yaml.Unmarshal(val, &a)
157 }
158 return
159 }
160
161 type Presets map[string]Preset
162
163 type Preset struct {
164 Name string `yaml:"name,omitempty"`
165 Description string
166 Metrics map[string]int
167 }
168
169
170 func ReadPresetsFromFolder(folder string) (presets Presets, err error) {
171 var presetMetrics []byte
172 fmt.Printf("Searching for presents from path %s ...\n", folder)
173 if presetMetrics, err = os.ReadFile(path.Join(folder, PresetConfigYAMLFile)); err != nil {
174 return
175 }
176 var oldPresets []Preset
177 if err = yaml.Unmarshal(presetMetrics, &oldPresets); err != nil {
178 return
179 }
180 presets = make(Presets, 0)
181 for _, p := range oldPresets {
182 pname := p.Name
183 p.Name = ""
184 presets[pname] = p
185 }
186 return
187 }
188
189 func WriteMetricsToFile(metricDefs any, filename string) error {
190 yamlData, err := yaml.Marshal(metricDefs)
191 if err != nil {
192 return err
193 }
194 return os.WriteFile(filename, yamlData, 0644)
195 }
196
197 func moveHelpersToMetrics(helpers, metrics Metrics) {
198 new_helper:
199 for helperName, h := range helpers {
200 for metricName, m := range metrics {
201 for _, sql := range m.SQLs {
202 if strings.Contains(sql, helperName) {
203 for _, v := range h.SQLs {
204 m.InitSQL = v
205 metrics[metricName] = m
206 continue new_helper
207 }
208 }
209 }
210 }
211 }
212 }
213
214 func getArgs(src *string, dst *string) (err error) {
215 flag.Parse()
216 if src == nil || *src == "" {
217 err = errors.New("-src option is required")
218 }
219 if dst == nil || *dst == "" {
220 err = errors.Join(err, errors.New("-dst option is required"))
221 }
222 if err != nil {
223 fmt.Println(err)
224 }
225 return err
226 }
227
228 func main() {
229 src := flag.String("src", "", "pgwatch v2 metric folder, e.g. `./metrics/sql`")
230 dst := flag.String("dst", "", "pgwatch v3 output metric file, e.g. `metrics.yaml`")
231 if err := getArgs(src, dst); err != nil {
232 return
233 }
234
235 helpers, err := ReadMetricsFromFolder(path.Join(*src, FileBasedMetricHelpersDir))
236 if err != nil {
237 panic(err)
238 }
239 metrics, err := ReadMetricsFromFolder(*src)
240 if err != nil {
241 panic(err)
242 }
243 moveHelpersToMetrics(helpers, metrics)
244 presets, err := ReadPresetsFromFolder(*src)
245 if err != nil {
246 panic(err)
247 }
248 err = WriteMetricsToFile(struct {
249 Metrics Metrics `yaml:"metrics,omitempty"`
250 Presets Presets `yaml:"presets,omitempty"`
251 }{
252 metrics,
253 presets,
254 }, *dst)
255 if err != nil {
256 panic(err)
257 }
258 }
259