...

Package metrics

import "github.com/cybertec-postgresql/pgwatch/v3/internal/metrics"
Overview
Index
Subdirectories

Overview ▾

Metrics

Code in this folder is responsible for reading and writing metric definitions. At the moment, metric definitions support two storages:

  • PostgreSQL database
  • YAML file

Content

  • `postgres*.go` files cover the functionality for the PostgreSQL database.
  • `yaml*.go` files cover the functionality for the YAML file.
  • `metrics.yaml` holds all default metrics and presets.
  • `default.go` provides access to default metrics.

Index ▾

Constants
Variables
func GetDefaultBuiltInMetrics() []string
func ParseLogs(ctx context.Context, conn db.PgxIface, mdb *sources.MonitoredDatabase, realDbname, metricName string, configMap map[string]float64, storeCh chan<- []MeasurementEnvelope)
func getFileWithLatestTimestamp(files []string) (string, error)
func getFileWithNextModTimestamp(logsGlobPath, currentFile string) (string, error)
func regexMatchesToMap(csvlogRegex *regexp.Regexp, matches []string) map[string]string
func severityToEnglish(serverLang, errorSeverity string) string
func tryDetermineLogFolder(ctx context.Context, conn db.PgxIface) (string, error)
func tryDetermineLogMessagesLanguage(ctx context.Context, conn db.PgxIface) (string, error)
func writeMetricsToPostgres(ctx context.Context, conn db.PgxIface, metricDefs *Metrics) error
func zeroEventCounts(eventCounts map[string]int64)
type CmdOpts
type ExtensionInfo
type ExtensionOverrides
type Measurement
type MeasurementEnvelope
    func eventCountsToMetricStoreMessages(eventCounts, eventCountsTotal map[string]int64, mdb *sources.MonitoredDatabase) []MeasurementEnvelope
type Measurements
type Metric
    func (m Metric) GetSQL(version int) string
    func (m Metric) PrimaryOnly() bool
    func (m Metric) StandbyOnly() bool
type MetricAttrs
type MetricDefs
type Metrics
    func GetDefaultMetrics() (metrics *Metrics)
type Migrator
type Preset
type PresetDefs
type Reader
type ReaderWriter
    func NewDefaultMetricReader(context.Context) (ReaderWriter, error)
    func NewPostgresMetricReaderWriter(ctx context.Context, connstr string) (ReaderWriter, error)
    func NewPostgresMetricReaderWriterConn(ctx context.Context, conn db.PgxPoolIface) (ReaderWriter, error)
    func NewYAMLMetricReaderWriter(ctx context.Context, path string) (ReaderWriter, error)
type SQLs
type Writer
type dbMetricReaderWriter
    func (dmrw *dbMetricReaderWriter) DeleteMetric(metricName string) error
    func (dmrw *dbMetricReaderWriter) DeletePreset(presetName string) error
    func (dmrw *dbMetricReaderWriter) GetMetrics() (metricDefMapNew *Metrics, err error)
    func (dmrw *dbMetricReaderWriter) Migrate() error
    func (dmrw *dbMetricReaderWriter) NeedsMigration() (bool, error)
    func (dmrw *dbMetricReaderWriter) UpdateMetric(metricName string, metric Metric) error
    func (dmrw *dbMetricReaderWriter) UpdatePreset(presetName string, preset Preset) error
    func (dmrw *dbMetricReaderWriter) WriteMetrics(metricDefs *Metrics) error
type defaultMetricReader
    func (dmrw *defaultMetricReader) DeleteMetric(string) error
    func (dmrw *defaultMetricReader) DeletePreset(string) error
    func (dmrw *defaultMetricReader) GetMetrics() (*Metrics, error)
    func (dmrw *defaultMetricReader) UpdateMetric(string, Metric) error
    func (dmrw *defaultMetricReader) UpdatePreset(string, Preset) error
    func (dmrw *defaultMetricReader) WriteMetrics(*Metrics) error
type fileMetricReader
    func (fmr *fileMetricReader) DeleteMetric(metricName string) error
    func (fmr *fileMetricReader) DeletePreset(presetName string) error
    func (fmr *fileMetricReader) GetMetrics() (metrics *Metrics, err error)
    func (fmr *fileMetricReader) UpdateMetric(metricName string, metric Metric) error
    func (fmr *fileMetricReader) UpdatePreset(presetName string, preset Preset) error
    func (fmr *fileMetricReader) WriteMetrics(metricDefs *Metrics) error

Package files

cmdopts.go default.go doc.go logparse.go postgres.go postgres_schema.go types.go yaml.go

Constants

const CSVLogDefaultGlobSuffix = "*.csv"
const CSVLogDefaultRegEx = `^^(?P<log_time>.*?),"?(?P<user_name>.*?)"?,"?(?P<database_name>.*?)"?,(?P<process_id>\d+),"?(?P<connection_from>.*?)"?,(?P<session_id>.*?),(?P<session_line_num>\d+),"?(?P<command_tag>.*?)"?,(?P<session_start_time>.*?),(?P<virtual_transaction_id>.*?),(?P<transaction_id>.*?),(?P<error_severity>\w+),`
const specialMetricServerLogEventCounts = "server_log_event_counts"

Variables

var PgSeverities = [...]string{"DEBUG", "INFO", "NOTICE", "WARNING", "ERROR", "LOG", "FATAL", "PANIC"}
var PgSeveritiesLocale = map[string]map[string]string{
    "C.": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTICE": "NOTICE", "WARNING": "WARNING", "ERROR": "ERROR", "FATAL": "FATAL", "PANIC": "PANIC"},
    "de": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "HINWEIS": "NOTICE", "WARNUNG": "WARNING", "FEHLER": "ERROR", "FATAL": "FATAL", "PANIK": "PANIC"},
    "fr": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTICE": "NOTICE", "ATTENTION": "WARNING", "ERREUR": "ERROR", "FATAL": "FATAL", "PANIK": "PANIC"},
    "it": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTIFICA": "NOTICE", "ATTENZIONE": "WARNING", "ERRORE": "ERROR", "FATALE": "FATAL", "PANICO": "PANIC"},
    "ko": {"디버그": "DEBUG", "로그": "LOG", "정보": "INFO", "알림": "NOTICE", "경고": "WARNING", "오류": "ERROR", "치명적오류": "FATAL", "손상": "PANIC"},
    "pl": {"DEBUG": "DEBUG", "DZIENNIK": "LOG", "INFORMACJA": "INFO", "UWAGA": "NOTICE", "OSTRZEŻENIE": "WARNING", "BŁĄD": "ERROR", "KATASTROFALNY": "FATAL", "PANIKA": "PANIC"},
    "ru": {"ОТЛАДКА": "DEBUG", "СООБЩЕНИЕ": "LOG", "ИНФОРМАЦИЯ": "INFO", "ЗАМЕЧАНИЕ": "NOTICE", "ПРЕДУПРЕЖДЕНИЕ": "WARNING", "ОШИБКА": "ERROR", "ВАЖНО": "FATAL", "ПАНИКА": "PANIC"},
    "sv": {"DEBUG": "DEBUG", "LOGG": "LOG", "INFO": "INFO", "NOTIS": "NOTICE", "VARNING": "WARNING", "FEL": "ERROR", "FATALT": "FATAL", "PANIK": "PANIC"},
    "tr": {"DEBUG": "DEBUG", "LOG": "LOG", "BİLGİ": "INFO", "NOT": "NOTICE", "UYARI": "WARNING", "HATA": "ERROR", "ÖLÜMCÜL (FATAL)": "FATAL", "KRİTİK": "PANIC"},
    "zh": {"调试": "DEBUG", "日志": "LOG", "信息": "INFO", "注意": "NOTICE", "警告": "WARNING", "错误": "ERROR", "致命错误": "FATAL", "比致命错误还过分的错误": "PANIC"},
}
var defaultMetricsYAML []byte
var initMigrator = func(dmrw *dbMetricReaderWriter) (*migrator.Migrator, error) {
    return migrator.New(
        migrator.TableName("pgwatch.migration"),
        migrator.SetNotice(func(s string) {
            log.GetLogger(dmrw.ctx).Info(s)
        }),
        migrations(),
    )
}
var initSchema = func(ctx context.Context, conn db.PgxIface) (err error) {
    var exists bool
    if exists, err = db.DoesSchemaExist(ctx, conn, "pgwatch"); err != nil || exists {
        return err
    }
    tx, err := conn.Begin(ctx)
    if err != nil {
        return err
    }
    defer func() { _ = tx.Rollback(ctx) }()
    if _, err := tx.Exec(ctx, sqlConfigSchema); err != nil {
        return err
    }
    if err := writeMetricsToPostgres(ctx, tx, GetDefaultMetrics()); err != nil {
        return err
    }
    if err := tx.Commit(ctx); err != nil {
        return err
    }
    return nil
}

migrations holds function returning all updgrade migrations needed

var migrations func() migrator.Option = func() migrator.Option {
    return migrator.Migrations(
        &migrator.Migration{
            Name: "00179 Apply metrics migrations for v3",
            Func: func(context.Context, pgx.Tx) error {

                return nil
            },
        },
    )
}
var sqlConfigSchema string

func GetDefaultBuiltInMetrics

func GetDefaultBuiltInMetrics() []string

func ParseLogs

func ParseLogs(ctx context.Context, conn db.PgxIface, mdb *sources.MonitoredDatabase, realDbname, metricName string, configMap map[string]float64, storeCh chan<- []MeasurementEnvelope)

func getFileWithLatestTimestamp

func getFileWithLatestTimestamp(files []string) (string, error)

func getFileWithNextModTimestamp

func getFileWithNextModTimestamp(logsGlobPath, currentFile string) (string, error)

func regexMatchesToMap

func regexMatchesToMap(csvlogRegex *regexp.Regexp, matches []string) map[string]string

func severityToEnglish

func severityToEnglish(serverLang, errorSeverity string) string

func tryDetermineLogFolder

func tryDetermineLogFolder(ctx context.Context, conn db.PgxIface) (string, error)

func tryDetermineLogMessagesLanguage

func tryDetermineLogMessagesLanguage(ctx context.Context, conn db.PgxIface) (string, error)

func writeMetricsToPostgres

func writeMetricsToPostgres(ctx context.Context, conn db.PgxIface, metricDefs *Metrics) error

writeMetricsToPostgres writes the metrics and presets definitions to the pgwatch.metric and pgwatch.preset tables in the ConfigDB.

func zeroEventCounts

func zeroEventCounts(eventCounts map[string]int64)

type CmdOpts

CmdOpts specifies metric command-line options

type CmdOpts struct {
    Metrics                      string `short:"m" long:"metrics" mapstructure:"metrics" description:"File or folder of YAML files with metrics definitions" env:"PW_METRICS"`
    CreateHelpers                bool   `long:"create-helpers" mapstructure:"create-helpers" description:"Create helper database objects from metric definitions" env:"PW_CREATE_HELPERS"`
    DirectOSStats                bool   `long:"direct-os-stats" mapstructure:"direct-os-stats" description:"Extract OS related psutil statistics not via PL/Python wrappers but directly on host" env:"PW_DIRECT_OS_STATS"`
    InstanceLevelCacheMaxSeconds int64  `long:"instance-level-cache-max-seconds" mapstructure:"instance-level-cache-max-seconds" description:"Max allowed staleness for instance level metric data shared between DBs of an instance. Affects 'continuous' host types only. Set to 0 to disable" env:"PW_INSTANCE_LEVEL_CACHE_MAX_SECONDS" default:"30"`
    EmergencyPauseTriggerfile    string `long:"emergency-pause-triggerfile" mapstructure:"emergency-pause-triggerfile" description:"When the file exists no metrics will be temporarily fetched / scraped" env:"PW_EMERGENCY_PAUSE_TRIGGERFILE" default:"/tmp/pgwatch-emergency-pause"`
}

type ExtensionInfo

type ExtensionInfo struct {
    ExtName       string `yaml:"ext_name"`
    ExtMinVersion string `yaml:"ext_min_version"`
}

type ExtensionOverrides

type ExtensionOverrides struct {
    TargetMetric              string          `yaml:"target_metric"`
    ExpectedExtensionVersions []ExtensionInfo `yaml:"expected_extension_versions"`
}

type Measurement

type Measurement map[string]any

type MeasurementEnvelope

type MeasurementEnvelope struct {
    DBName           string
    SourceType       string
    MetricName       string
    CustomTags       map[string]string
    Data             Measurements
    MetricDef        Metric
    RealDbname       string
    SystemIdentifier string
}

func eventCountsToMetricStoreMessages

func eventCountsToMetricStoreMessages(eventCounts, eventCountsTotal map[string]int64, mdb *sources.MonitoredDatabase) []MeasurementEnvelope

1. add zero counts for severity levels that didn't have any occurrences in the log

type Measurements

type Measurements []map[string]any

type Metric

type Metric struct {
    SQLs            SQLs
    InitSQL         string   `yaml:"init_sql,omitempty"`
    NodeStatus      string   `yaml:"node_status,omitempty"`
    Gauges          []string `yaml:",omitempty"`
    IsInstanceLevel bool     `yaml:"is_instance_level,omitempty"`
    StorageName     string   `yaml:"storage_name,omitempty"`
    Description     string   `yaml:"description,omitempty"`
}

func (Metric) GetSQL

func (m Metric) GetSQL(version int) string

func (Metric) PrimaryOnly

func (m Metric) PrimaryOnly() bool

func (Metric) StandbyOnly

func (m Metric) StandbyOnly() bool

type MetricAttrs

type MetricAttrs struct {
    ExtensionVersionOverrides []ExtensionOverrides `yaml:"extension_version_based_overrides,omitempty"`
    IsPrivate                 bool                 `yaml:"is_private,omitempty"`                // used only for extension overrides currently and ignored otherwise
    DisabledDays              string               `yaml:"disabled_days,omitempty"`             // Cron style, 0 = Sunday. Ranges allowed: 0,2-4
    DisableTimes              []string             `yaml:"disabled_times,omitempty"`            // "11:00-13:00"
    StatementTimeoutSeconds   int64                `yaml:"statement_timeout_seconds,omitempty"` // overrides per monitored DB settings
}

type MetricDefs

type MetricDefs map[string]Metric

type Metrics

type Metrics struct {
    MetricDefs MetricDefs `yaml:"metrics"`
    PresetDefs PresetDefs `yaml:"presets"`
}

func GetDefaultMetrics

func GetDefaultMetrics() (metrics *Metrics)

type Migrator

type Migrator interface {
    Migrate() error
    NeedsMigration() (bool, error)
}

make sure *dbMetricReaderWriter implements the Migrator interface

var _ Migrator = (*dbMetricReaderWriter)(nil)

type Preset

type Preset struct {
    Description string
    Metrics     map[string]float64
}

type PresetDefs

type PresetDefs map[string]Preset

type Reader

type Reader interface {
    GetMetrics() (*Metrics, error)
}

type ReaderWriter

type ReaderWriter interface {
    Reader
    Writer
}

func NewDefaultMetricReader

func NewDefaultMetricReader(context.Context) (ReaderWriter, error)

NewDefaultMetricReader creates a new default metric reader with an empty path.

func NewPostgresMetricReaderWriter

func NewPostgresMetricReaderWriter(ctx context.Context, connstr string) (ReaderWriter, error)

func NewPostgresMetricReaderWriterConn

func NewPostgresMetricReaderWriterConn(ctx context.Context, conn db.PgxPoolIface) (ReaderWriter, error)

func NewYAMLMetricReaderWriter

func NewYAMLMetricReaderWriter(ctx context.Context, path string) (ReaderWriter, error)

type SQLs

type SQLs map[int]string

type Writer

type Writer interface {
    WriteMetrics(metricDefs *Metrics) error
    DeleteMetric(metricName string) error
    DeletePreset(presetName string) error
    UpdateMetric(metricName string, metric Metric) error
    UpdatePreset(presetName string, preset Preset) error
}

type dbMetricReaderWriter

type dbMetricReaderWriter struct {
    ctx      context.Context
    configDb db.PgxIface
}

func (*dbMetricReaderWriter) DeleteMetric

func (dmrw *dbMetricReaderWriter) DeleteMetric(metricName string) error

func (*dbMetricReaderWriter) DeletePreset

func (dmrw *dbMetricReaderWriter) DeletePreset(presetName string) error

func (*dbMetricReaderWriter) GetMetrics

func (dmrw *dbMetricReaderWriter) GetMetrics() (metricDefMapNew *Metrics, err error)

ReadMetricsFromPostgres reads the metrics and presets definitions from the pgwatch.metric and pgwatch.preset tables from the ConfigDB.

func (*dbMetricReaderWriter) Migrate

func (dmrw *dbMetricReaderWriter) Migrate() error

MigrateDb upgrades database with all migrations

func (*dbMetricReaderWriter) NeedsMigration

func (dmrw *dbMetricReaderWriter) NeedsMigration() (bool, error)

NeedsMigration checks if database needs migration

func (*dbMetricReaderWriter) UpdateMetric

func (dmrw *dbMetricReaderWriter) UpdateMetric(metricName string, metric Metric) error

func (*dbMetricReaderWriter) UpdatePreset

func (dmrw *dbMetricReaderWriter) UpdatePreset(presetName string, preset Preset) error

func (*dbMetricReaderWriter) WriteMetrics

func (dmrw *dbMetricReaderWriter) WriteMetrics(metricDefs *Metrics) error

type defaultMetricReader

type defaultMetricReader struct{}

func (*defaultMetricReader) DeleteMetric

func (dmrw *defaultMetricReader) DeleteMetric(string) error

func (*defaultMetricReader) DeletePreset

func (dmrw *defaultMetricReader) DeletePreset(string) error

func (*defaultMetricReader) GetMetrics

func (dmrw *defaultMetricReader) GetMetrics() (*Metrics, error)

func (*defaultMetricReader) UpdateMetric

func (dmrw *defaultMetricReader) UpdateMetric(string, Metric) error

func (*defaultMetricReader) UpdatePreset

func (dmrw *defaultMetricReader) UpdatePreset(string, Preset) error

func (*defaultMetricReader) WriteMetrics

func (dmrw *defaultMetricReader) WriteMetrics(*Metrics) error

type fileMetricReader

type fileMetricReader struct {
    ctx  context.Context
    path string
}

func (*fileMetricReader) DeleteMetric

func (fmr *fileMetricReader) DeleteMetric(metricName string) error

func (*fileMetricReader) DeletePreset

func (fmr *fileMetricReader) DeletePreset(presetName string) error

func (*fileMetricReader) GetMetrics

func (fmr *fileMetricReader) GetMetrics() (metrics *Metrics, err error)

func (*fileMetricReader) UpdateMetric

func (fmr *fileMetricReader) UpdateMetric(metricName string, metric Metric) error

func (*fileMetricReader) UpdatePreset

func (fmr *fileMetricReader) UpdatePreset(presetName string, preset Preset) error

func (*fileMetricReader) WriteMetrics

func (fmr *fileMetricReader) WriteMetrics(metricDefs *Metrics) error

Subdirectories

Name Synopsis
..
psutil