resolver.go

Documentation: github.com/cybertec-postgresql/pgwatch/v6/internal/sources

     1  package sources
     2  
     3  // This file contains the implemendation of Patroni and PostgrSQL resolvers for continuous monitoring.
     4  // Patroni resolver will return the list of databases from the Patroni cluster.
     5  // Postgres resolver will return the list of databases from the given Postgres instance.
     6  
     7  import (
     8  	"cmp"
     9  	"context"
    10  	"crypto/tls"
    11  	"crypto/x509"
    12  	"errors"
    13  	"fmt"
    14  	"net/url"
    15  	"os"
    16  	"strings"
    17  	"sync"
    18  	"time"
    19  
    20  	jsoniter "github.com/json-iterator/go"
    21  
    22  	"github.com/cybertec-postgresql/pgwatch/v6/internal/db"
    23  	"github.com/cybertec-postgresql/pgwatch/v6/internal/log"
    24  	pgx "github.com/jackc/pgx/v5"
    25  	client "go.etcd.io/etcd/client/v3"
    26  	"go.uber.org/zap"
    27  )
    28  
    29  // Resolver discovers the monitored databases behind continuous-monitoring
    30  // sources (Patroni, Postgres discovery). It owns the last-known-good fallback
    31  // caches so that a transient DCS/DB outage does not tear down monitoring of
    32  // already-known databases.
    33  //
    34  // A Resolver is safe for concurrent use. Create independent Resolvers via
    35  // NewResolver when isolated cache state is desired (e.g. in tests or when
    36  // running several unrelated resolution pipelines in one process).
    37  type Resolver struct {
    38  	mu sync.Mutex // guards the fallback caches below
    39  	// lastFoundClusterMembers is needed for cases where DCS is temporarily down;
    40  	// we don't want to immediately remove monitoring of DBs. Keyed by source name.
    41  	lastFoundClusterMembers map[string][]PatroniClusterMember
    42  	// lastFoundDatabases is keyed by the source's identity (name + conn string +
    43  	// include/exclude patterns) so a reconfigured source never inherits the
    44  	// previous target's database list.
    45  	lastFoundDatabases map[postgresDiscoveryKey]SourceConns
    46  }
    47  
    48  // NewResolver returns a Resolver with freshly initialized, empty caches.
    49  func NewResolver() *Resolver {
    50  	return &Resolver{
    51  		lastFoundClusterMembers: make(map[string][]PatroniClusterMember),
    52  		lastFoundDatabases:      make(map[postgresDiscoveryKey]SourceConns),
    53  	}
    54  }
    55  
    56  // defaultResolver backs the convenience methods on Source and Sources so that
    57  // existing callers keep a single process-wide fallback cache without having to
    58  // thread a Resolver through.
    59  var defaultResolver = NewResolver()
    60  
    61  // ResolveDatabases() updates list of monitored objects from continuous monitoring sources, e.g. patroni.
    62  // Each source is resolved concurrently so that a slow or unreachable source does not block the others.
    63  // It delegates to the package-wide defaultResolver.
    64  func (srcs Sources) ResolveDatabases(onError func(string)) (SourceConns, error) {
    65  	return defaultResolver.ResolveDatabases(srcs, onError)
    66  }
    67  
    68  // ResolveDatabases() return a slice of found databases for continuous monitoring sources, e.g. patroni.
    69  // It delegates to the package-wide defaultResolver.
    70  func (s Source) ResolveDatabases() (SourceConns, error) {
    71  	return defaultResolver.ResolveDatabase(s)
    72  }
    73  
    74  // ResolveDatabases updates the list of monitored objects from continuous monitoring
    75  // sources, e.g. patroni. Each source is resolved concurrently so that a slow or
    76  // unreachable source does not block the others.
    77  func (r *Resolver) ResolveDatabases(srcs Sources, onError func(string)) (_ SourceConns, err error) {
    78  	type result struct {
    79  		dbs SourceConns
    80  		err error
    81  	}
    82  	results := make([]result, len(srcs))
    83  	var wg sync.WaitGroup
    84  	for i, s := range srcs {
    85  		wg.Go(func() {
    86  			dbs, e := r.ResolveDatabase(s)
    87  			results[i] = result{dbs, e}
    88  		})
    89  	}
    90  	wg.Wait()
    91  	resolvedDbs := make(SourceConns, 0, len(srcs))
    92  	for i, res := range results {
    93  		if res.err != nil {
    94  			if onError != nil {
    95  				onError(srcs[i].Name)
    96  			}
    97  			logger.WithField("source", srcs[i].Name).WithError(res.err).Error("could not resolve databases from source")
    98  			err = errors.Join(err, res.err)
    99  		}
   100  		resolvedDbs = append(resolvedDbs, res.dbs...)
   101  	}
   102  	return resolvedDbs, err
   103  }
   104  
   105  // ResolveDatabase returns a slice of found databases for a single continuous
   106  // monitoring source, e.g. patroni.
   107  func (r *Resolver) ResolveDatabase(s Source) (SourceConns, error) {
   108  	switch s.Kind {
   109  	case SourcePatroniDiscovery:
   110  		return r.ResolveDatabasesFromPatroni(s)
   111  	case SourcePostgresDiscovery:
   112  		return r.ResolveDatabasesFromPostgres(s)
   113  	case SourcePrometheus:
   114  		return SourceConns{NewPromConn(s)}, nil
   115  	}
   116  	return SourceConns{NewDbConn(s)}, nil
   117  }
   118  
   119  type PatroniClusterMember struct {
   120  	Scope   string
   121  	Name    string
   122  	ConnURL string `yaml:"conn_url"`
   123  	Role    string
   124  }
   125  
   126  func (pcm PatroniClusterMember) IsPrimary() bool {
   127  	return pcm.Role == "primary" || pcm.Role == "master"
   128  }
   129  
   130  var logger log.Logger = log.FallbackLogger
   131  
   132  // postgresDiscoveryKey uniquely identifies a Postgres discovery source for caching purposes.
   133  // It MUST incorporate every field that would change the set of discovered databases; otherwise a
   134  // reconfigured source could be served the previous target's last-known-good list.
   135  type postgresDiscoveryKey struct {
   136  	name           string
   137  	connStr        string
   138  	includePattern string
   139  	excludePattern string
   140  }
   141  
   142  func postgresDiscoveryKeyOf(s Source) postgresDiscoveryKey {
   143  	return postgresDiscoveryKey{
   144  		name:           s.Name,
   145  		connStr:        s.ConnStr,
   146  		includePattern: s.IncludePattern,
   147  		excludePattern: s.ExcludePattern,
   148  	}
   149  }
   150  
   151  func getConsulClusterMembers(Source) ([]PatroniClusterMember, error) {
   152  	return nil, errors.ErrUnsupported
   153  }
   154  
   155  func getZookeeperClusterMembers(Source, HostConfig) ([]PatroniClusterMember, error) {
   156  	return nil, errors.ErrUnsupported
   157  }
   158  
   159  func jsonTextToStringMap(jsonText string) (map[string]string, error) {
   160  	retmap := make(map[string]string)
   161  	if jsonText == "" {
   162  		return retmap, nil
   163  	}
   164  	var iMap map[string]any
   165  	if err := jsoniter.ConfigFastest.Unmarshal([]byte(jsonText), &iMap); err != nil {
   166  		return nil, err
   167  	}
   168  	for k, v := range iMap {
   169  		retmap[k] = fmt.Sprintf("%v", v)
   170  	}
   171  	return retmap, nil
   172  }
   173  
   174  func getTransport(conf HostConfig) (*tls.Config, error) {
   175  	var caCertPool *x509.CertPool
   176  
   177  	// create valid CertPool only if the ca certificate file exists
   178  	if conf.CAFile != "" {
   179  		caCert, err := os.ReadFile(conf.CAFile)
   180  		if err != nil {
   181  			return nil, fmt.Errorf("cannot load CA file: %s", err)
   182  		}
   183  
   184  		caCertPool = x509.NewCertPool()
   185  		caCertPool.AppendCertsFromPEM(caCert)
   186  	}
   187  
   188  	var certificates []tls.Certificate
   189  
   190  	// create valid []Certificate only if the client cert and key files exists
   191  	if conf.CertFile != "" && conf.KeyFile != "" {
   192  		cert, err := tls.LoadX509KeyPair(conf.CertFile, conf.KeyFile)
   193  		if err != nil {
   194  			return nil, fmt.Errorf("cannot load client cert or key file: %s", err)
   195  		}
   196  
   197  		certificates = []tls.Certificate{cert}
   198  	}
   199  
   200  	tlsClientConfig := new(tls.Config)
   201  
   202  	if caCertPool != nil {
   203  		tlsClientConfig.RootCAs = caCertPool
   204  		if certificates != nil {
   205  			tlsClientConfig.Certificates = certificates
   206  		}
   207  	}
   208  
   209  	return tlsClientConfig, nil
   210  }
   211  
   212  func (r *Resolver) getEtcdClusterMembers(s Source, hc HostConfig) ([]PatroniClusterMember, error) {
   213  	var ret = make([]PatroniClusterMember, 0)
   214  	var cfg client.Config
   215  
   216  	if len(hc.DcsEndpoints) == 0 {
   217  		return ret, errors.New("missing ETCD connect info, make sure host config has a 'dcs_endpoints' key")
   218  	}
   219  
   220  	tlsConfig, err := getTransport(hc)
   221  	if err != nil {
   222  		return nil, err
   223  	}
   224  	cfg = client.Config{
   225  		Endpoints:            hc.DcsEndpoints,
   226  		TLS:                  tlsConfig,
   227  		DialKeepAliveTimeout: time.Second,
   228  		Username:             hc.Username,
   229  		Password:             hc.Password,
   230  		DialTimeout:          5 * time.Second,
   231  		Logger:               zap.NewNop(),
   232  	}
   233  
   234  	c, err := client.New(cfg)
   235  	if err != nil {
   236  		return ret, err
   237  	}
   238  	defer c.Close()
   239  
   240  	ctx, cancel := context.WithTimeoutCause(context.Background(), 5*time.Second, errors.New("etcd client timeout"))
   241  	defer cancel()
   242  
   243  	// etcd3 does not have a dir node.
   244  	// Key="/namespace/scope/leader", e.g. "/service/batman/leader"
   245  	// Key="/namespace/scope/members/node", e.g. "/service/batman/members/pg1"
   246  
   247  	resp, err := c.Get(ctx, hc.Path, client.WithPrefix())
   248  	if err != nil {
   249  		return ret, cmp.Or(context.Cause(ctx), err)
   250  	}
   251  
   252  	for _, node := range resp.Kvs {
   253  		// remove leading slash and split by "/"
   254  		parts := strings.Split(strings.TrimPrefix(string(node.Key), "/"), "/")
   255  		if len(parts) < 4 || parts[2] != "members" {
   256  			continue // skip non-member keys
   257  		}
   258  		nodeData, err := jsonTextToStringMap(string(node.Value))
   259  		if err != nil {
   260  			logger.Errorf("Could not parse ETCD node data for node \"%s\": %s", node.Key, err)
   261  			continue
   262  		}
   263  		role := nodeData["role"]
   264  		connURL := nodeData["conn_url"]
   265  		scope := parts[1]
   266  		name := parts[3]
   267  		ret = append(ret, PatroniClusterMember{Scope: scope, ConnURL: connURL, Role: role, Name: name})
   268  	}
   269  
   270  	r.mu.Lock()
   271  	r.lastFoundClusterMembers[s.Name] = ret
   272  	r.mu.Unlock()
   273  	return ret, nil
   274  }
   275  
   276  const (
   277  	dcsTypeEtcd      = "etcd"
   278  	dcsTypeZookeeper = "zookeeper"
   279  	dcsTypeConsul    = "consul"
   280  )
   281  
   282  type HostConfig struct {
   283  	DcsType      string   `yaml:"dcs_type"`
   284  	DcsEndpoints []string `yaml:"dcs_endpoints"`
   285  	Path         string
   286  	Username     string
   287  	Password     string
   288  	CAFile       string `yaml:"ca_file"`
   289  	CertFile     string `yaml:"cert_file"`
   290  	KeyFile      string `yaml:"key_file"`
   291  }
   292  
   293  func (hc HostConfig) IsScopeSpecified() bool {
   294  	// Path is usually "/namespace/scope"
   295  	// so we check if it has at least 2 slashes
   296  	return strings.Count(hc.Path, "/") >= 2
   297  }
   298  
   299  func NewHostConfig(URI string) (hc HostConfig, err error) {
   300  	// Extract scheme
   301  	before, after, ok := strings.Cut(URI, "://")
   302  	if !ok {
   303  		return hc, fmt.Errorf("invalid URI: missing scheme")
   304  	}
   305  	scheme := before
   306  	remainder := after // skip "://"
   307  
   308  	// Find where the host portion ends (at first '/' or '?' or end of string)
   309  	hostEnd := strings.IndexAny(remainder, "/?")
   310  	var hostPart, pathAndQuery string
   311  	if hostEnd == -1 {
   312  		hostPart = remainder
   313  		pathAndQuery = ""
   314  	} else {
   315  		hostPart = remainder[:hostEnd]
   316  		pathAndQuery = remainder[hostEnd:]
   317  	}
   318  
   319  	// Check for user info (username:password@)
   320  	var userInfo string
   321  	if atIdx := strings.LastIndex(hostPart, "@"); atIdx != -1 {
   322  		userInfo = hostPart[:atIdx]
   323  		hostPart = hostPart[atIdx+1:]
   324  	}
   325  
   326  	// Split hosts by comma for multiple endpoints
   327  	hosts := strings.Split(hostPart, ",")
   328  
   329  	// Parse a clean URL with just the first host to extract other components
   330  	cleanURI := scheme + "://"
   331  	if userInfo != "" {
   332  		cleanURI += userInfo + "@"
   333  	}
   334  	cleanURI += hosts[0] + pathAndQuery
   335  
   336  	var url *url.URL
   337  	url, err = url.Parse(cleanURI)
   338  	if err != nil {
   339  		return
   340  	}
   341  
   342  	switch url.Scheme {
   343  	case dcsTypeEtcd:
   344  		hc.DcsType = dcsTypeEtcd
   345  		for _, h := range hosts {
   346  			hc.DcsEndpoints = append(hc.DcsEndpoints, "http://"+h)
   347  		}
   348  	case dcsTypeZookeeper:
   349  		hc.DcsType = dcsTypeZookeeper
   350  		hc.DcsEndpoints = hosts // Use the split hosts directly
   351  	case dcsTypeConsul:
   352  		hc.DcsType = dcsTypeConsul
   353  		hc.DcsEndpoints = hosts // Use the split hosts directly
   354  	default:
   355  		return hc, fmt.Errorf("unsupported DCS type: %s", url.Scheme)
   356  	}
   357  
   358  	hc.Path = url.Path
   359  	hc.Username = url.User.Username()
   360  	hc.Password, _ = url.User.Password() // password is optional, so we ignore the error
   361  	hc.CAFile = url.Query().Get("ca_file")
   362  	hc.CertFile = url.Query().Get("cert_file")
   363  	hc.KeyFile = url.Query().Get("key_file")
   364  
   365  	return hc, nil
   366  }
   367  
   368  func (r *Resolver) ResolveDatabasesFromPatroni(source Source) (SourceConns, error) {
   369  	var mds SourceConns
   370  	var clusterMembers []PatroniClusterMember
   371  	var err error
   372  	var ok bool
   373  
   374  	hostConfig, err := NewHostConfig(source.ConnStr)
   375  	if err != nil {
   376  		return nil, err
   377  	}
   378  
   379  	switch hostConfig.DcsType {
   380  	case dcsTypeEtcd:
   381  		clusterMembers, err = r.getEtcdClusterMembers(source, hostConfig)
   382  	case dcsTypeZookeeper:
   383  		clusterMembers, err = getZookeeperClusterMembers(source, hostConfig)
   384  	case dcsTypeConsul:
   385  		clusterMembers, err = getConsulClusterMembers(source)
   386  	default:
   387  		return nil, errors.New("unknown DCS")
   388  	}
   389  	logger := logger.WithField("source", source.Name)
   390  	if err != nil {
   391  		if errors.Is(err, errors.ErrUnsupported) {
   392  			return nil, err
   393  		}
   394  		logger.Debug("failed to get info from DCS, using previous member info if any")
   395  		r.mu.Lock()
   396  		clusterMembers, ok = r.lastFoundClusterMembers[source.Name] // mask error from main loop not to remove monitored DBs due to "jitter"
   397  		r.mu.Unlock()
   398  		if ok {
   399  			err = nil
   400  		}
   401  	} else {
   402  		r.mu.Lock()
   403  		r.lastFoundClusterMembers[source.Name] = clusterMembers
   404  		r.mu.Unlock()
   405  	}
   406  	if len(clusterMembers) == 0 {
   407  		return mds, err
   408  	}
   409  
   410  	for _, patroniMember := range clusterMembers {
   411  		logger.Info("processing Patroni cluster member: ", patroniMember.Name)
   412  		if source.OnlyIfMaster && !patroniMember.IsPrimary() {
   413  			continue
   414  		}
   415  		src := *source.Clone()
   416  		src.ConnStr = patroniMember.ConnURL
   417  		if !hostConfig.IsScopeSpecified() {
   418  			src.Name += "_" + patroniMember.Scope
   419  		}
   420  		src.Name += "_" + patroniMember.Name
   421  		if dbs, err := r.ResolveDatabasesFromPostgres(src); err == nil {
   422  			mds = append(mds, dbs...)
   423  		} else {
   424  			logger.WithError(err).Error("failed to resolve databases for Patroni member: ", patroniMember.Name)
   425  		}
   426  	}
   427  	return mds, err
   428  }
   429  
   430  // ResolveDatabasesFromPostgres reads all the databases from the given cluster,
   431  // additionally matching/not matching specified regex patterns.
   432  //
   433  // On any helper error (pool create or discovery query) and the presence of a
   434  // previously-cached successful result for the same source identity, the cached
   435  // list is returned with a nil error: a transient discovery failure (DNS hiccup,
   436  // connect timeout, discovery-SQL permission error) must not tear down monitoring
   437  // of already-known databases. The accepted trade-off is that a database dropped
   438  // server-side while discovery keeps failing stays monitored from cache (surfaced
   439  // as per-cycle connect errors reported as down) until the next successful
   440  // resolution replaces the entry.
   441  func (r *Resolver) ResolveDatabasesFromPostgres(s Source) (resolvedDbs SourceConns, err error) {
   442  	resolvedDbs, err = resolveDatabasesFromPostgres(s)
   443  	key := postgresDiscoveryKeyOf(s)
   444  	r.mu.Lock()
   445  	defer r.mu.Unlock()
   446  	if err == nil {
   447  		r.lastFoundDatabases[key] = resolvedDbs
   448  		return resolvedDbs, nil
   449  	}
   450  	cached, ok := r.lastFoundDatabases[key]
   451  	if ok && len(cached) > 0 {
   452  		logger.WithField("source", s.Name).WithError(err).Warning("postgres discovery failed; serving last-known-good database list from cache")
   453  		return cached, nil
   454  	}
   455  	return nil, err
   456  }
   457  
   458  // resolveDatabasesFromPostgres is the inner helper that actually runs the discovery query.
   459  func resolveDatabasesFromPostgres(s Source) (resolvedDbs SourceConns, err error) {
   460  	var (
   461  		c      db.PgxPoolIface
   462  		dbname string
   463  		rows   pgx.Rows
   464  	)
   465  	ctx, cancel := db.WithOpTimeout(context.Background(), "resolve "+s.Name, db.ResolverTimeout)
   466  	defer cancel()
   467  
   468  	c, err = NewConn(ctx, s.ConnStr)
   469  	if err != nil {
   470  		return nil, cmp.Or(context.Cause(ctx), err)
   471  	}
   472  	defer c.Close()
   473  
   474  	sql := `select /* pgwatch_generated */
   475  	datname
   476  	from pg_database
   477  	where not datistemplate
   478  	and datallowconn
   479  	and has_database_privilege (datname, 'CONNECT')
   480  	and case when length(trim($1)) > 0 then datname ~ $1 else true end
   481  	and case when length(trim($2)) > 0 then not datname ~ $2 else true end`
   482  
   483  	if rows, err = c.Query(ctx, sql, s.IncludePattern, s.ExcludePattern); err != nil {
   484  		return nil, cmp.Or(context.Cause(ctx), err)
   485  	}
   486  	for rows.Next() {
   487  		if err = rows.Scan(&dbname); err != nil {
   488  			return nil, err
   489  		}
   490  		rdb := NewDbConn(*s.Clone())
   491  		rdb.Name += "_" + dbname
   492  		rdb.SetDatabaseName(dbname)
   493  		resolvedDbs = append(resolvedDbs, rdb)
   494  	}
   495  
   496  	if err := rows.Err(); err != nil {
   497  		return nil, err
   498  	}
   499  	return
   500  }
   501