convert_metrics.go

Documentation: github.com/cybertec-postgresql/pgwatch/v3/cmd/convert_metrics

     1  package main
     2  
     3  import (
     4  	"io/fs"
     5  	"math"
     6  	"os"
     7  	"path"
     8  	"regexp"
     9  	"slices"
    10  	"strconv"
    11  	"strings"
    12  
    13  	"flag"
    14  	"fmt"
    15  
    16  	"gopkg.in/yaml.v3"
    17  )
    18  
    19  type (
    20  	ExtensionInfo struct {
    21  		ExtName       string `yaml:"ext_name"`
    22  		ExtMinVersion string `yaml:"ext_min_version"`
    23  	}
    24  
    25  	ExtensionOverrides struct {
    26  		TargetMetric              string          `yaml:"target_metric"`
    27  		ExpectedExtensionVersions []ExtensionInfo `yaml:"expected_extension_versions"`
    28  	}
    29  
    30  	MetricAttrs struct {
    31  		IsInstanceLevel           bool                 `yaml:"is_instance_level,omitempty"`
    32  		MetricStorageName         string               `yaml:"metric_storage_name,omitempty"`
    33  		ExtensionVersionOverrides []ExtensionOverrides `yaml:"extension_version_based_overrides,omitempty"`
    34  		IsPrivate                 bool                 `yaml:"is_private,omitempty"`                // used only for extension overrides currently and ignored otherwise
    35  		DisabledDays              string               `yaml:"disabled_days,omitempty"`             // Cron style, 0 = Sunday. Ranges allowed: 0,2-4
    36  		DisableTimes              []string             `yaml:"disabled_times,omitempty"`            // "11:00-13:00"
    37  		StatementTimeoutSeconds   int64                `yaml:"statement_timeout_seconds,omitempty"` // overrides per monitored DB settings
    38  	}
    39  
    40  	SQLs map[int]string
    41  
    42  	Metric struct {
    43  		SQLs        SQLs
    44  		InitSQL     string   `yaml:"init_sql,omitempty"`
    45  		NodeStatus  string   `yaml:"node_status,omitempty"`
    46  		Gauges      []string `yaml:",omitempty"`
    47  		MetricAttrs `yaml:",inline,omitempty"`
    48  	}
    49  
    50  	Metrics map[string]Metric
    51  )
    52  
    53  const (
    54  	FileBasedMetricHelpersDir = "00_helpers"
    55  	PresetConfigYAMLFile      = "preset-configs.yaml"
    56  )
    57  
    58  // VersionToInt parses a given version and returns an integer  or
    59  // an error if unable to parse the version. Only parses valid semantic versions.
    60  // Performs checking that can find errors within the version.
    61  // Examples: v1.2 -> 01_02_00, v9.6.3 -> 09_06_03, v11 -> 11_00_00
    62  var regVer = regexp.MustCompile(`(\d+).?(\d*).?(\d*)`)
    63  
    64  func VersionToInt(version string) (v int) {
    65  	if matches := regVer.FindStringSubmatch(version); len(matches) > 1 {
    66  		for i, match := range matches[1:] {
    67  			v += func() (m int) { m, _ = strconv.Atoi(match); return }() * int(math.Pow10(4-i*2))
    68  		}
    69  	}
    70  	return
    71  }
    72  
    73  // expected is following structure: metric_name/pg_ver/metric(_master|standby).sql
    74  func ReadMetricsFromFolder(folder string) (metricsMap Metrics, err error) {
    75  	metricFolders, err := os.ReadDir(folder)
    76  	if err != nil {
    77  		return
    78  	}
    79  
    80  	metricsMap = make(map[string]Metric)
    81  	metricNamePattern := `^[a-z0-9_\.]+$`
    82  	regexMetricNameFilter := regexp.MustCompile(metricNamePattern)
    83  	regexIsDigitOrPunctuation := regexp.MustCompile(`^[\d\.]+$`)
    84  
    85  	fmt.Printf("Searching for metrics from path %s ...\n", folder)
    86  
    87  	for _, metricFolder := range metricFolders {
    88  		if metricFolder.Name() == FileBasedMetricHelpersDir ||
    89  			!metricFolder.IsDir() ||
    90  			!regexMetricNameFilter.MatchString(metricFolder.Name()) {
    91  			continue
    92  		}
    93  
    94  		var versionFolders []fs.DirEntry
    95  		versionFolders, err = os.ReadDir(path.Join(folder, metricFolder.Name()))
    96  		if err != nil {
    97  			return
    98  		}
    99  
   100  		Metric := Metric{}
   101  		Metric.MetricAttrs, _ = ParseMetricAttrsFromYAML(path.Join(folder, metricFolder.Name(), "metric_attrs.yaml"))
   102  		_ = ParseMetricPrometheusAttrsFromYAML(path.Join(folder, metricFolder.Name(), "column_attrs.yaml"), &Metric)
   103  		Metric.SQLs = make(SQLs)
   104  
   105  		var version int
   106  		for _, versionFolder := range versionFolders {
   107  			if strings.HasSuffix(versionFolder.Name(), ".md") || versionFolder.Name() == "column_attrs.yaml" || versionFolder.Name() == "metric_attrs.yaml" {
   108  				continue
   109  			}
   110  			if !regexIsDigitOrPunctuation.MatchString(versionFolder.Name()) {
   111  				fmt.Printf("Invalid metric structure - version folder names should consist of only numerics/dots, found: %s", versionFolder.Name())
   112  				continue
   113  			}
   114  			if version, err = strconv.Atoi(versionFolder.Name()); err != nil {
   115  				version = 11 // the oldest supported
   116  			}
   117  
   118  			var metricDefs []fs.DirEntry
   119  			if metricDefs, err = os.ReadDir(path.Join(folder, metricFolder.Name(), versionFolder.Name())); err != nil {
   120  				return
   121  			}
   122  
   123  			for _, metricDef := range metricDefs {
   124  				if strings.HasPrefix(metricDef.Name(), "metric") && strings.HasSuffix(metricDef.Name(), ".sql") {
   125  					p := path.Join(folder, metricFolder.Name(), versionFolder.Name(), metricDef.Name())
   126  					metricSQL, err := os.ReadFile(p)
   127  					if err != nil {
   128  						continue
   129  					}
   130  					switch {
   131  					case strings.Contains(metricDef.Name(), "_master"):
   132  						Metric.NodeStatus = "primary"
   133  					case strings.Contains(metricDef.Name(), "_standby"):
   134  						Metric.NodeStatus = "standby"
   135  					}
   136  					Metric.SQLs[version] = strings.TrimRight(strings.TrimSpace(string(metricSQL)), ";")
   137  				}
   138  			}
   139  		}
   140  		metricsMap[metricFolder.Name()] = Metric
   141  	}
   142  	return
   143  }
   144  
   145  func ParseMetricPrometheusAttrsFromYAML(path string, m *Metric) (err error) {
   146  	type OldMetricPrometheusAttrs struct {
   147  		PrometheusGaugeColumns    []string `yaml:"prometheus_gauge_columns,omitempty"`
   148  		PrometheusIgnoredColumns  []string `yaml:"prometheus_ignored_columns,omitempty"` // for cases where we don't want some columns to be exposed in Prom mode
   149  		PrometheusAllGaugeColumns bool     `yaml:"prometheus_all_gauge_columns,omitempty"`
   150  	}
   151  
   152  	var val []byte
   153  	var oldPromAttrs OldMetricPrometheusAttrs
   154  	if val, err = os.ReadFile(path); err == nil {
   155  		if err = yaml.Unmarshal(val, &oldPromAttrs); err != nil {
   156  			return
   157  		}
   158  		if oldPromAttrs.PrometheusAllGaugeColumns {
   159  			m.Gauges = []string{"*"}
   160  			return
   161  		}
   162  		m.Gauges = slices.Clone(oldPromAttrs.PrometheusGaugeColumns)
   163  	}
   164  
   165  	return
   166  }
   167  
   168  func ParseMetricAttrsFromYAML(path string) (a MetricAttrs, err error) {
   169  	var val []byte
   170  	if val, err = os.ReadFile(path); err == nil {
   171  		err = yaml.Unmarshal(val, &a)
   172  	}
   173  	return
   174  }
   175  
   176  type Presets map[string]Preset
   177  
   178  type Preset struct {
   179  	Name        string `yaml:"name,omitempty"`
   180  	Description string
   181  	Metrics     map[string]int
   182  }
   183  
   184  // Expects "preset metrics" definition file named preset-config.yaml to be present in provided --metrics folder
   185  func ReadPresetsFromFolder(folder string) (presets Presets, err error) {
   186  	var presetMetrics []byte
   187  	fmt.Printf("Searching for presents from path %s ...\n", folder)
   188  	if presetMetrics, err = os.ReadFile(path.Join(folder, PresetConfigYAMLFile)); err != nil {
   189  		return
   190  	}
   191  	var oldPresets []Preset
   192  	if err = yaml.Unmarshal(presetMetrics, &oldPresets); err != nil {
   193  		return
   194  	}
   195  	presets = make(Presets, 0)
   196  	for _, p := range oldPresets {
   197  		pname := p.Name
   198  		p.Name = ""
   199  		presets[pname] = p
   200  	}
   201  	return
   202  }
   203  
   204  func WriteMetricsToFile(metricDefs any, filename string) error {
   205  	yamlData, err := yaml.Marshal(metricDefs)
   206  	if err != nil {
   207  		return err
   208  	}
   209  	return os.WriteFile(filename, yamlData, 0644)
   210  }
   211  
   212  func moveHelpersToMetrics(helpers, metrics Metrics) {
   213  new_helper:
   214  	for helperName, h := range helpers {
   215  		for metricName, m := range metrics {
   216  			for _, sql := range m.SQLs {
   217  				if strings.Contains(sql, helperName) {
   218  					for _, v := range h.SQLs {
   219  						m.InitSQL = v
   220  						metrics[metricName] = m
   221  						continue new_helper
   222  					}
   223  				}
   224  			}
   225  		}
   226  	}
   227  }
   228  
   229  func main() {
   230  	// Define command-line flags
   231  	src := flag.String("src", "", "pgwatch v2 metric folder, e.g. `./metrics/sql`")
   232  	dst := flag.String("dst", "", "pgwatch v3 output metric file, e.g. `metrics.yaml`")
   233  
   234  	// Parse command-line flags
   235  	flag.Parse()
   236  
   237  	// Check if src flag is provided
   238  	if *src == "" {
   239  		fmt.Println("Error: src option is required")
   240  		return
   241  	}
   242  
   243  	// Check if dst flag is provided
   244  	if *dst == "" {
   245  		fmt.Println("Error: dst option is required")
   246  		return
   247  	}
   248  	helpers, err := ReadMetricsFromFolder(path.Join(*src, FileBasedMetricHelpersDir))
   249  	if err != nil {
   250  		panic(err)
   251  	}
   252  	metrics, err := ReadMetricsFromFolder(*src)
   253  	if err != nil {
   254  		panic(err)
   255  	}
   256  	moveHelpersToMetrics(helpers, metrics)
   257  	presets, err := ReadPresetsFromFolder(*src)
   258  	if err != nil {
   259  		panic(err)
   260  	}
   261  	err = WriteMetricsToFile(struct {
   262  		Metrics Metrics `yaml:"metrics,omitempty"`
   263  		Presets Presets `yaml:"presets,omitempty"`
   264  	}{
   265  		metrics,
   266  		presets,
   267  	}, *dst)
   268  	if err != nil {
   269  		panic(err)
   270  	}
   271  }
   272