#!/opt/gitlab/embedded/bin/ruby require '/opt/gitlab/embedded/cookbooks/consul/libraries/failover_helper' require 'open3' require 'logger' @log = Logger.new('<%= @log_directory %>/failover_pgbouncer.log') @log.level = Logger::INFO def run_command(command) exit_status = 0 Open3.popen3(command) do |stdin, stdout, stderr, wait_thr| @log.info("Running: #{command}") while line = stdout.gets @log.info("STDOUT: #{line}") end while line = stderr.gets @log.error("STDERR: #{line}") end exit_status = wait_thr.value end exit_status.to_i end service_data = FailoverHelper::ServiceData.new service_data.service_name = "<%= @watcher_service_name %>" service_data.check_field = "Status" service_data.leader_value = "passing" begin failover = FailoverHelper::LeaderFinder.new($stdin.gets, service_data) new_primary = failover.primary_node_address @log.info("Found primary: #{new_primary}") Kernel.exit run_command("gitlab-ctl pgb-notify --pg-database <%= @database_name %> --newhost #{new_primary} --user pgbouncer --hostuser gitlab-consul") rescue JSON::ParserError => jparser_error @log.error(jparser_error.message) Kernel.exit 2 rescue FailoverHelper::SplitBrain => splitbrain_error @log.error(splitbrain_error.message) splitbrain_error.primary_nodes.each do |primary| @log.error(" Node: #{primary.name}") end @log.error('Stopping pgbouncer to prevent issues. Once the error is cleared, consul will reload pgbouncer') results = run_command('gitlab-ctl pgb-kill --pg-database <%= @database_name %> --user pgbouncer --hostuser gitlab-consul') Kernel.exit 3 + results rescue FailoverHelper::PrimaryMissing => no_primary_error @log.error(no_primary_error.message) Kernel.exit 4 end