const CSVLogDefaultGlobSuffix = "*.csv"
const CSVLogDefaultRegEx = `^^(?P<log_time>.*?),"?(?P<user_name>.*?)"?,"?(?P<database_name>.*?)"?,(?P<process_id>\d+),"?(?P<connection_from>.*?)"?,(?P<session_id>.*?),(?P<session_line_num>\d+),"?(?P<command_tag>.*?)"?,(?P<session_start_time>.*?),(?P<virtual_transaction_id>.*?),(?P<transaction_id>.*?),(?P<error_severity>\w+),`
const specialMetricServerLogEventCounts = "server_log_event_counts"
var PgSeverities = [...]string{"DEBUG", "INFO", "NOTICE", "WARNING", "ERROR", "LOG", "FATAL", "PANIC"}
var PgSeveritiesLocale = map[string]map[string]string{ "C.": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTICE": "NOTICE", "WARNING": "WARNING", "ERROR": "ERROR", "FATAL": "FATAL", "PANIC": "PANIC"}, "de": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "HINWEIS": "NOTICE", "WARNUNG": "WARNING", "FEHLER": "ERROR", "FATAL": "FATAL", "PANIK": "PANIC"}, "fr": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTICE": "NOTICE", "ATTENTION": "WARNING", "ERREUR": "ERROR", "FATAL": "FATAL", "PANIK": "PANIC"}, "it": {"DEBUG": "DEBUG", "LOG": "LOG", "INFO": "INFO", "NOTIFICA": "NOTICE", "ATTENZIONE": "WARNING", "ERRORE": "ERROR", "FATALE": "FATAL", "PANICO": "PANIC"}, "ko": {"디버그": "DEBUG", "로그": "LOG", "정보": "INFO", "알림": "NOTICE", "경고": "WARNING", "오류": "ERROR", "치명적오류": "FATAL", "손상": "PANIC"}, "pl": {"DEBUG": "DEBUG", "DZIENNIK": "LOG", "INFORMACJA": "INFO", "UWAGA": "NOTICE", "OSTRZEŻENIE": "WARNING", "BŁĄD": "ERROR", "KATASTROFALNY": "FATAL", "PANIKA": "PANIC"}, "ru": {"ОТЛАДКА": "DEBUG", "СООБЩЕНИЕ": "LOG", "ИНФОРМАЦИЯ": "INFO", "ЗАМЕЧАНИЕ": "NOTICE", "ПРЕДУПРЕЖДЕНИЕ": "WARNING", "ОШИБКА": "ERROR", "ВАЖНО": "FATAL", "ПАНИКА": "PANIC"}, "sv": {"DEBUG": "DEBUG", "LOGG": "LOG", "INFO": "INFO", "NOTIS": "NOTICE", "VARNING": "WARNING", "FEL": "ERROR", "FATALT": "FATAL", "PANIK": "PANIC"}, "tr": {"DEBUG": "DEBUG", "LOG": "LOG", "BİLGİ": "INFO", "NOT": "NOTICE", "UYARI": "WARNING", "HATA": "ERROR", "ÖLÜMCÜL (FATAL)": "FATAL", "KRİTİK": "PANIC"}, "zh": {"调试": "DEBUG", "日志": "LOG", "信息": "INFO", "注意": "NOTICE", "警告": "WARNING", "错误": "ERROR", "致命错误": "FATAL", "比致命错误还过分的错误": "PANIC"}, }
var defaultMetricsYAML []byte
var initMigrator = func(dmrw *dbMetricReaderWriter) (*migrator.Migrator, error) { return migrator.New( migrator.TableName("pgwatch.migration"), migrator.SetNotice(func(s string) { log.GetLogger(dmrw.ctx).Info(s) }), migrations(), ) }
var initSchema = func(ctx context.Context, conn db.PgxIface) (err error) { var exists bool if exists, err = db.DoesSchemaExist(ctx, conn, "pgwatch"); err != nil || exists { return err } tx, err := conn.Begin(ctx) if err != nil { return err } defer func() { _ = tx.Rollback(ctx) }() if _, err := tx.Exec(ctx, sqlConfigSchema); err != nil { return err } if err := writeMetricsToPostgres(ctx, tx, GetDefaultMetrics()); err != nil { return err } if err := tx.Commit(ctx); err != nil { return err } return nil }
migrations holds function returning all updgrade migrations needed
var migrations func() migrator.Option = func() migrator.Option { return migrator.Migrations( &migrator.Migration{ Name: "00179 Apply metrics migrations for v3", Func: func(context.Context, pgx.Tx) error { return nil }, }, ) }
var sqlConfigSchema string
func GetDefaultBuiltInMetrics() []string
func ParseLogs(ctx context.Context, conn db.PgxIface, mdb *sources.MonitoredDatabase, realDbname, metricName string, configMap map[string]float64, storeCh chan<- []MeasurementEnvelope)
func getFileWithLatestTimestamp(files []string) (string, error)
func getFileWithNextModTimestamp(logsGlobPath, currentFile string) (string, error)
func regexMatchesToMap(csvlogRegex *regexp.Regexp, matches []string) map[string]string
func severityToEnglish(serverLang, errorSeverity string) string
func tryDetermineLogFolder(ctx context.Context, conn db.PgxIface) (string, error)
func tryDetermineLogMessagesLanguage(ctx context.Context, conn db.PgxIface) (string, error)
func writeMetricsToPostgres(ctx context.Context, conn db.PgxIface, metricDefs *Metrics) error
writeMetricsToPostgres writes the metrics and presets definitions to the pgwatch.metric and pgwatch.preset tables in the ConfigDB.
func zeroEventCounts(eventCounts map[string]int64)
CmdOpts specifies metric command-line options
type CmdOpts struct { Metrics string `short:"m" long:"metrics" mapstructure:"metrics" description:"File or folder of YAML files with metrics definitions" env:"PW_METRICS"` CreateHelpers bool `long:"create-helpers" mapstructure:"create-helpers" description:"Create helper database objects from metric definitions" env:"PW_CREATE_HELPERS"` DirectOSStats bool `long:"direct-os-stats" mapstructure:"direct-os-stats" description:"Extract OS related psutil statistics not via PL/Python wrappers but directly on host" env:"PW_DIRECT_OS_STATS"` InstanceLevelCacheMaxSeconds int64 `long:"instance-level-cache-max-seconds" mapstructure:"instance-level-cache-max-seconds" description:"Max allowed staleness for instance level metric data shared between DBs of an instance. Affects 'continuous' host types only. Set to 0 to disable" env:"PW_INSTANCE_LEVEL_CACHE_MAX_SECONDS" default:"30"` EmergencyPauseTriggerfile string `long:"emergency-pause-triggerfile" mapstructure:"emergency-pause-triggerfile" description:"When the file exists no metrics will be temporarily fetched / scraped" env:"PW_EMERGENCY_PAUSE_TRIGGERFILE" default:"/tmp/pgwatch-emergency-pause"` }
type ExtensionInfo struct { ExtName string `yaml:"ext_name"` ExtMinVersion string `yaml:"ext_min_version"` }
type ExtensionOverrides struct { TargetMetric string `yaml:"target_metric"` ExpectedExtensionVersions []ExtensionInfo `yaml:"expected_extension_versions"` }
type Measurement map[string]any
type MeasurementEnvelope struct { DBName string SourceType string MetricName string CustomTags map[string]string Data Measurements MetricDef Metric RealDbname string SystemIdentifier string }
func eventCountsToMetricStoreMessages(eventCounts, eventCountsTotal map[string]int64, mdb *sources.MonitoredDatabase) []MeasurementEnvelope
1. add zero counts for severity levels that didn't have any occurrences in the log
type Measurements []map[string]any
type Metric struct { SQLs SQLs InitSQL string `yaml:"init_sql,omitempty"` NodeStatus string `yaml:"node_status,omitempty"` Gauges []string `yaml:",omitempty"` IsInstanceLevel bool `yaml:"is_instance_level,omitempty"` StorageName string `yaml:"storage_name,omitempty"` Description string `yaml:"description,omitempty"` }
func (m Metric) GetSQL(version int) string
func (m Metric) PrimaryOnly() bool
func (m Metric) StandbyOnly() bool
type MetricAttrs struct { ExtensionVersionOverrides []ExtensionOverrides `yaml:"extension_version_based_overrides,omitempty"` IsPrivate bool `yaml:"is_private,omitempty"` // used only for extension overrides currently and ignored otherwise DisabledDays string `yaml:"disabled_days,omitempty"` // Cron style, 0 = Sunday. Ranges allowed: 0,2-4 DisableTimes []string `yaml:"disabled_times,omitempty"` // "11:00-13:00" StatementTimeoutSeconds int64 `yaml:"statement_timeout_seconds,omitempty"` // overrides per monitored DB settings }
type MetricDefs map[string]Metric
type Metrics struct { MetricDefs MetricDefs `yaml:"metrics"` PresetDefs PresetDefs `yaml:"presets"` }
func GetDefaultMetrics() (metrics *Metrics)
type Migrator interface { Migrate() error NeedsMigration() (bool, error) }
make sure *dbMetricReaderWriter implements the Migrator interface
var _ Migrator = (*dbMetricReaderWriter)(nil)
type Preset struct { Description string Metrics map[string]float64 }
type PresetDefs map[string]Preset
type Reader interface { GetMetrics() (*Metrics, error) }
type ReaderWriter interface { Reader Writer }
func NewDefaultMetricReader(context.Context) (ReaderWriter, error)
NewDefaultMetricReader creates a new default metric reader with an empty path.
func NewPostgresMetricReaderWriter(ctx context.Context, connstr string) (ReaderWriter, error)
func NewPostgresMetricReaderWriterConn(ctx context.Context, conn db.PgxPoolIface) (ReaderWriter, error)
func NewYAMLMetricReaderWriter(ctx context.Context, path string) (ReaderWriter, error)
type SQLs map[int]string
type Writer interface { WriteMetrics(metricDefs *Metrics) error DeleteMetric(metricName string) error DeletePreset(presetName string) error UpdateMetric(metricName string, metric Metric) error UpdatePreset(presetName string, preset Preset) error }
type dbMetricReaderWriter struct { ctx context.Context configDb db.PgxIface }
func (dmrw *dbMetricReaderWriter) DeleteMetric(metricName string) error
func (dmrw *dbMetricReaderWriter) DeletePreset(presetName string) error
func (dmrw *dbMetricReaderWriter) GetMetrics() (metricDefMapNew *Metrics, err error)
ReadMetricsFromPostgres reads the metrics and presets definitions from the pgwatch.metric and pgwatch.preset tables from the ConfigDB.
func (dmrw *dbMetricReaderWriter) Migrate() error
MigrateDb upgrades database with all migrations
func (dmrw *dbMetricReaderWriter) NeedsMigration() (bool, error)
NeedsMigration checks if database needs migration
func (dmrw *dbMetricReaderWriter) UpdateMetric(metricName string, metric Metric) error
func (dmrw *dbMetricReaderWriter) UpdatePreset(presetName string, preset Preset) error
func (dmrw *dbMetricReaderWriter) WriteMetrics(metricDefs *Metrics) error
type defaultMetricReader struct{}
func (dmrw *defaultMetricReader) DeleteMetric(string) error
func (dmrw *defaultMetricReader) DeletePreset(string) error
func (dmrw *defaultMetricReader) GetMetrics() (*Metrics, error)
func (dmrw *defaultMetricReader) UpdateMetric(string, Metric) error
func (dmrw *defaultMetricReader) UpdatePreset(string, Preset) error
func (dmrw *defaultMetricReader) WriteMetrics(*Metrics) error
type fileMetricReader struct { ctx context.Context path string }
func (fmr *fileMetricReader) DeleteMetric(metricName string) error
func (fmr *fileMetricReader) DeletePreset(presetName string) error
func (fmr *fileMetricReader) GetMetrics() (metrics *Metrics, err error)
func (fmr *fileMetricReader) UpdateMetric(metricName string, metric Metric) error
func (fmr *fileMetricReader) UpdatePreset(presetName string, preset Preset) error
func (fmr *fileMetricReader) WriteMetrics(metricDefs *Metrics) error