metrics_yaml_test.go

Documentation: github.com/cybertec-postgresql/pgwatch/v6/internal/metrics

     1  package metrics
     2  
     3  import (
     4  	"regexp"
     5  	"strings"
     6  	"testing"
     7  
     8  	"github.com/stretchr/testify/assert"
     9  	"gopkg.in/yaml.v3"
    10  )
    11  
    12  // knownMetrics is the set of top-level metric names that the embedded
    13  // metrics.yaml is expected to ship. If a metric is renamed or removed
    14  // intentionally, update this list alongside the YAML change.
    15  var knownMetrics = []string{
    16  	"archiver",
    17  	"autovacuum_scores",
    18  	"backends",
    19  	"backup_age_pgbackrest",
    20  	"backup_age_walg",
    21  	"bgwriter",
    22  	"blocking_locks",
    23  	"buffercache_by_db",
    24  	"buffercache_by_type",
    25  	"change_events",
    26  	"checkpointer",
    27  	"configuration_hashes",
    28  	"cpu_load",
    29  	"database_conflicts",
    30  	"db_size",
    31  	"db_size_approx",
    32  	"db_stats",
    33  	"index_hashes",
    34  	"index_stats",
    35  	"instance_up",
    36  	"invalid_indexes",
    37  	"kpi",
    38  	"locks",
    39  	"locks_mode",
    40  	"logical_subscriptions",
    41  	"pgbouncer_stats",
    42  	"pgbouncer_clients",
    43  	"pgpool_processes",
    44  	"pgpool_stats",
    45  	"privilege_changes",
    46  	"psutil_cpu",
    47  	"psutil_disk",
    48  	"psutil_disk_io_total",
    49  	"psutil_mem",
    50  	"reco_add_index",
    51  	"reco_default_public_schema",
    52  	"reco_disabled_triggers",
    53  	"reco_drop_index",
    54  	"reco_nested_views",
    55  	"reco_partial_index_candidates",
    56  	"reco_sprocs_wo_search_path",
    57  	"reco_superusers",
    58  	"recovery",
    59  	"replication",
    60  	"replication_slot_stats",
    61  	"replication_slots",
    62  	"sequence_health",
    63  	"server_log_event_counts",
    64  	"settings",
    65  	"smart_health_per_disk",
    66  	"sproc_hashes",
    67  	"sproc_stats",
    68  	"stat_activity",
    69  	"stat_lock",
    70  	"stat_io",
    71  	"stat_ssl",
    72  	"stat_statements",
    73  	"stat_statements_calls",
    74  	"stat_statements_no_query_text",
    75  	"subscription_stats",
    76  	"table_bloat_approx_stattuple",
    77  	"table_bloat_approx_summary",
    78  	"table_bloat_approx_summary_sql",
    79  	"table_hashes",
    80  	"table_io_stats",
    81  	"table_stats",
    82  	"table_stats_approx",
    83  	"unused_indexes",
    84  	"vmstat",
    85  	"wait_events",
    86  	"wal",
    87  	"wal_receiver",
    88  	"wal_size",
    89  	"wal_stats",
    90  	"datfrozenxid",
    91  	"postgres_role",
    92  	"archiver_pending_count",
    93  }
    94  
    95  // metricsRoot parses the embedded metrics.yaml into the raw yaml.v3 node
    96  // tree. Going via the AST (rather than the typed Metrics struct) is what
    97  // lets the structural guards below catch orphan-scalar bleed: Go struct
    98  // unmarshalling silently absorbs misaligned continuation lines into the
    99  // surrounding block scalar, but the AST preserves indentation and
   100  // sequence structure.
   101  func metricsRoot(t *testing.T) *yaml.Node {
   102  	t.Helper()
   103  	var root yaml.Node
   104  	assert.NoError(t, yaml.Unmarshal(defaultMetricsYAML, &root),
   105  		"metrics.yaml must be valid YAML")
   106  	assert.Equal(t, yaml.DocumentNode, root.Kind, "metrics.yaml root must be a document")
   107  	assert.Len(t, root.Content, 1, "metrics.yaml document must contain exactly one node")
   108  	top := root.Content[0]
   109  	assert.Equal(t, yaml.MappingNode, top.Kind, "metrics.yaml top-level must be a mapping")
   110  	return &root
   111  }
   112  
   113  // metricsBlock returns the inner mapping node of the `metrics:` block.
   114  func metricsBlock(t *testing.T, root *yaml.Node) *yaml.Node {
   115  	t.Helper()
   116  	top := root.Content[0]
   117  	for i := 0; i < len(top.Content); i += 2 {
   118  		if top.Content[i].Value == "metrics" {
   119  			return top.Content[i+1]
   120  		}
   121  	}
   122  	t.Fatalf("metrics.yaml: top-level 'metrics:' key not found")
   123  	return nil
   124  }
   125  
   126  // SQL clause-classification regexes, anchored to start-of-line (after
   127  // optional whitespace). The "non-terminal" set (WHERE/GROUP BY/HAVING)
   128  // must appear strictly before every "terminal" set (LIMIT/ORDER BY).
   129  // A `|-` SQL scalar that has absorbed a sibling metric's body will show
   130  // WHERE/GROUP BY/HAVING clauses appearing AFTER a LIMIT/ORDER BY —
   131  // structurally impossible in any single valid SELECT.
   132  var (
   133  	sqlNonTerminal = regexp.MustCompile(`(?im)^\s*(where|group\s+by|having)\b`)
   134  	sqlTerminal    = regexp.MustCompile(`(?im)^\s*(limit|order\s+by)\b`)
   135  )
   136  
   137  // clauseOrderViolation returns the line number of the first non-terminal
   138  // clause (WHERE/GROUP BY/HAVING) that appears AFTER a terminal clause
   139  // (LIMIT/ORDER BY) at parenthesis depth 0 — i.e. in the outer SELECT body,
   140  // not inside a CTE definition or subquery. Returns 0 when the SQL is
   141  // well-ordered.
   142  func clauseOrderViolation(sql string) int {
   143  	lastTerminal := 0
   144  	depth := 0
   145  	for i, raw := range strings.Split(sql, "\n") {
   146  		line := raw
   147  		// Track paren depth using the line's raw text before classifying.
   148  		for _, r := range line {
   149  			switch r {
   150  			case '(':
   151  				depth++
   152  			case ')':
   153  				if depth > 0 {
   154  					depth--
   155  				}
   156  			}
   157  		}
   158  		if depth != 0 {
   159  			continue
   160  		}
   161  		if sqlNonTerminal.MatchString(line) {
   162  			if lastTerminal > 0 {
   163  				return i + 1
   164  			}
   165  		} else if sqlTerminal.MatchString(line) {
   166  			if i+1 > lastTerminal {
   167  				lastTerminal = i + 1
   168  			}
   169  		}
   170  	}
   171  	return 0
   172  }
   173  
   174  // TestAllKnownMetricsPresent guards against silent top-level key
   175  // deletions during future edits to metrics.yaml. The presence check runs
   176  // against the raw yaml.v3 AST, so a metric that disappears mid-edit (even
   177  // if its body is later re-pasted into a sibling block) is still caught.
   178  //
   179  // If a metric is renamed or removed intentionally, update knownMetrics in
   180  // this test.
   181  func TestAllKnownMetricsPresent(t *testing.T) {
   182  	a := assert.New(t)
   183  	block := metricsBlock(t, metricsRoot(t))
   184  	a.Equal(yaml.MappingNode, block.Kind, "'metrics:' block must be a mapping")
   185  
   186  	got := make(map[string]bool, len(block.Content)/2)
   187  	for i := 0; i < len(block.Content); i += 2 {
   188  		got[block.Content[i].Value] = true
   189  	}
   190  
   191  	for _, name := range knownMetrics {
   192  		a.True(got[name],
   193  			"metric %q missing from metrics.yaml top-level (was it silently removed by an edit?)", name)
   194  	}
   195  }
   196  
   197  // TestSQLScalarsAreStructurallyIntact guards against scalar-tail bleed
   198  // (orphan lines from a sibling metric absorbed into a `|-` block scalar).
   199  //
   200  // Go struct unmarshalling accepts continuation lines indented to look like
   201  // block-scalar content, folding them silently into the scalar value. This
   202  // guard detects the resulting structural impossibility: WHERE / GROUP BY /
   203  // HAVING must never appear after LIMIT / ORDER BY at parenthesis depth 0
   204  // in any single SELECT.
   205  //
   206  // The companion TestSQLScalarBleedRegression test feeds a corrupted
   207  // scalar through the same logic to prove the guard actually fires.
   208  func TestSQLScalarsAreStructurallyIntact(t *testing.T) {
   209  	a := assert.New(t)
   210  	block := metricsBlock(t, metricsRoot(t))
   211  
   212  	for i := 0; i < len(block.Content); i += 2 {
   213  		name := block.Content[i].Value
   214  		def := block.Content[i+1]
   215  		if def.Kind != yaml.MappingNode {
   216  			continue
   217  		}
   218  		var sqlsNode *yaml.Node
   219  		for j := 0; j < len(def.Content); j += 2 {
   220  			if def.Content[j].Value == "sqls" {
   221  				sqlsNode = def.Content[j+1]
   222  				break
   223  			}
   224  		}
   225  		if sqlsNode == nil || sqlsNode.Kind != yaml.MappingNode {
   226  			continue
   227  		}
   228  
   229  		for k := 0; k < len(sqlsNode.Content); k += 2 {
   230  			version := sqlsNode.Content[k].Value
   231  			scalar := sqlsNode.Content[k+1]
   232  			if scalar.Kind != yaml.ScalarNode {
   233  				a.Failf("expected literal block scalar",
   234  					"%s sqls[%s]: expected a literal block scalar, got kind=%d (%q)",
   235  					name, version, scalar.Kind, scalar.Tag)
   236  				continue
   237  			}
   238  			if line := clauseOrderViolation(scalar.Value); line > 0 {
   239  				a.Failf("scalar-tail bleed detected",
   240  					"%s sqls[%s]: terminal clause precedes non-terminal clause at line %d. Tail: %q",
   241  					name, version, line, tail(scalar.Value, 200))
   242  			}
   243  		}
   244  	}
   245  }
   246  
   247  // TestSQLScalarBleedRegression is a self-contained reproducer that proves
   248  // TestSQLScalarsAreStructurallyIntact catches the class of bug it is
   249  // written for: a `|-` SQL scalar that has absorbed orphan lines from a
   250  // neighbouring metric. It feeds a small synthetic YAML containing a
   251  // canonical bleed through the same parsing pipeline and asserts that the
   252  // guard fires on the corrupted scalar while passing on the clean one.
   253  func TestSQLScalarBleedRegression(t *testing.T) {
   254  	a := assert.New(t)
   255  	const fixture = `
   256  metrics:
   257      metric_a:
   258          sqls:
   259              14: |-
   260                  select /* pgwatch_generated */
   261                    1::int as a_value
   262                  from
   263                    some_table
   264                  LIMIT 300
   265              19: |-
   266                  select /* pgwatch_generated */
   267                    1::int as a_value
   268                  from
   269                    some_table
   270                  LIMIT 300
   271                  where s.datname = current_database()
   272                    and s.state = 'active'
   273                  group by s.query
   274      metric_b:
   275          sqls:
   276              14: |-
   277                  select /* pgwatch_generated */
   278                    1::int as count
   279                  from some_view s
   280                  where s.datname = current_database()
   281                    and s.state = 'active'
   282                  group by s.query
   283  `
   284  
   285  	var root yaml.Node
   286  	a.NoError(yaml.Unmarshal([]byte(fixture), &root),
   287  		"synthetic YAML must still parse (Go absorbs the bleed silently)")
   288  
   289  	block := metricsBlock(t, &root)
   290  	metricA := findMetricNode(t, block, "metric_a")
   291  	sqls := findChildMapping(t, metricA, "sqls")
   292  
   293  	v14 := sqlScalarValue(t, sqls, "14")
   294  	a.Zero(clauseOrderViolation(v14),
   295  		"clean v14 should pass the invariant but failed:\n%s", v14)
   296  
   297  	v19 := sqlScalarValue(t, sqls, "19")
   298  	a.Contains(v19, "where s.datname",
   299  		"fixture is wrong — bleed lines missing from v19 scalar:\n%s", v19)
   300  	a.NotZero(clauseOrderViolation(v19),
   301  		"guard did NOT catch the bleed — v19 scalar still looks OK:\n%s", v19)
   302  }
   303  
   304  func findMetricNode(t *testing.T, block *yaml.Node, name string) *yaml.Node {
   305  	t.Helper()
   306  	if block.Kind != yaml.MappingNode {
   307  		t.Fatalf("metrics block is not a mapping (kind=%d)", block.Kind)
   308  	}
   309  	for i := 0; i < len(block.Content); i += 2 {
   310  		if block.Content[i].Value == name {
   311  			return block.Content[i+1]
   312  		}
   313  	}
   314  	t.Fatalf("metric %q not found", name)
   315  	return nil
   316  }
   317  
   318  func findChildMapping(t *testing.T, parent *yaml.Node, key string) *yaml.Node {
   319  	t.Helper()
   320  	if parent.Kind != yaml.MappingNode {
   321  		t.Fatalf("parent is not a mapping (kind=%d)", parent.Kind)
   322  	}
   323  	for i := 0; i < len(parent.Content); i += 2 {
   324  		if parent.Content[i].Value == key {
   325  			if parent.Content[i+1].Kind != yaml.MappingNode {
   326  				t.Fatalf("%q child is not a mapping (kind=%d)", key, parent.Content[i+1].Kind)
   327  			}
   328  			return parent.Content[i+1]
   329  		}
   330  	}
   331  	t.Fatalf("child %q not found", key)
   332  	return nil
   333  }
   334  
   335  func sqlScalarValue(t *testing.T, sqls *yaml.Node, version string) string {
   336  	t.Helper()
   337  	for i := 0; i < len(sqls.Content); i += 2 {
   338  		if sqls.Content[i].Value == version {
   339  			return sqls.Content[i+1].Value
   340  		}
   341  	}
   342  	t.Fatalf("sqls[%s] not found", version)
   343  	return ""
   344  }
   345  
   346  func tail(s string, n int) string {
   347  	if len(s) <= n {
   348  		return s
   349  	}
   350  	return "..." + s[len(s)-n:]
   351  }
   352