lib/gitlab/qa/component/gitlab.rb



# frozen_string_literal: true

# rubocop:disable Metrics/AbcSize

require 'securerandom'
require 'net/http'
require 'uri'
require 'forwardable'
require 'openssl'
require 'tempfile'
require 'json'

module Gitlab
  module QA
    module Component
      class Gitlab < Base
        extend Forwardable
        using Rainbow

        attr_reader :release,
          :omnibus_configuration,
          :omnibus_gitlab_rails_env,
          :authority_volume,
          :ssl_volume

        attr_accessor :tls,
          :skip_availability_check,
          :runner_network,
          :seed_admin_token,
          :seed_db,
          :skip_server_hooks,
          :gitaly_tls,
          :secrets

        attr_writer :name, :relative_path

        def_delegators :release, :tag, :image, :edition

        DATA_SEED_PATH = File.expand_path('../../../../support/data', __dir__)

        TRUSTED_PATH = '/etc/gitlab/trusted-certs'
        SSL_PATH = '/etc/gitlab/ssl'
        DATA_PATH = '/tmp/data-seeds'

        def initialize
          super

          @skip_availability_check = false
          @omnibus_gitlab_rails_env = {}
          @omnibus_configuration = Runtime::OmnibusConfiguration.new(Runtime::Scenario.omnibus_configuration)
          @cert_volumes = { "authority" => TRUSTED_PATH, "gitlab-ssl" => SSL_PATH }
          @seed_admin_token = Runtime::Scenario.seed_admin_token
          @seed_db = Runtime::Scenario.seed_db
          @skip_server_hooks = Runtime::Scenario.skip_server_hooks
          @secrets = []

          self.release = 'CE'
        end

        def set_formless_login_token
          return if Runtime::Env.gitlab_qa_formless_login_token.to_s.strip.empty?

          @omnibus_gitlab_rails_env['GITLAB_QA_FORMLESS_LOGIN_TOKEN'] = Runtime::Env.gitlab_qa_formless_login_token
        end

        def set_license_mode
          return unless Runtime::Env.gitlab_license_mode == 'test'

          @omnibus_gitlab_rails_env['GITLAB_LICENSE_MODE'] = 'test'
          @omnibus_gitlab_rails_env['CUSTOMER_PORTAL_URL'] = Runtime::Env.customer_portal_url
        end

        # Sets GITLAB_QA_USER_AGENT as a Rail environment variable so that it can be used by GitLab to bypass features
        # that can't be automated.
        def set_qa_user_agent
          return if Runtime::Env.gitlab_qa_user_agent.to_s.strip.empty?

          @omnibus_gitlab_rails_env['GITLAB_QA_USER_AGENT'] = Runtime::Env.gitlab_qa_user_agent
          secrets << Runtime::Env.gitlab_qa_user_agent
        end

        def elastic_url=(url)
          @environment['ELASTIC_URL'] = url
        end

        def set_ee_activation_code
          raise 'QA_EE_ACTIVATION_CODE is required!' if Runtime::Env.ee_activation_code.to_s.strip.empty?

          @omnibus_gitlab_rails_env['QA_EE_ACTIVATION_CODE'] = Runtime::Env.ee_activation_code
          secrets << Runtime::Env.ee_activation_code
        end

        def release=(release)
          @release = QA::Release.new(release)
        end

        def name
          @name ||= "gitlab-#{edition}-#{SecureRandom.hex(4)}"
        end

        def address
          "#{scheme}://#{hostname}#{relative_path}"
        end

        def scheme
          tls ? 'https' : 'http'
        end

        def gitlab_port
          tls ? ["443:443"] : ["80"]
        end

        def relative_path
          @relative_path ||= ''
        end

        def set_accept_insecure_certs
          Runtime::Env.accept_insecure_certs = 'true'
        end

        def prepare
          prepare_gitlab_omnibus_config
          copy_certificates

          super
        end

        def pull
          docker.login(**release.login_params) if release.login_params

          super
        end

        def exist?(image, tag)
          docker.manifest_exists?("#{image}:#{tag}")
        end

        def prepare_gitlab_omnibus_config
          @omnibus_configuration.expand_config_template(self)
          set_formless_login_token
          set_license_mode
          set_qa_user_agent
          env = @omnibus_gitlab_rails_env.merge(
            {
              'GITLAB_ALLOW_SEPARATE_CI_DATABASE' => Runtime::Env.allow_separate_ci_database.to_s
            }
          )

          @omnibus_configuration << "gitlab_rails['env'] = #{env}"
        end

        def start # rubocop:disable Metrics/AbcSize
          ensure_configured!

          docker.run(image: image, tag: tag) do |command|
            command << "-d"
            command << "--name #{name}"
            command << "--net #{network}"
            command << "--hostname #{hostname}"

            [*@ports, *gitlab_port].each do |mapping|
              command.port(mapping)
            end

            @volumes.to_h.merge(cert_volumes).each do |to, from|
              command.volume(to, from, 'Z')
            end

            command.volume(File.join(Runtime::Env.host_artifacts_dir, name, 'logs'), '/var/log/gitlab', 'Z')

            @environment.to_h.each do |key, value|
              command.env(key, value)
            end

            @network_aliases.to_a.each do |network_alias|
              command << "--network-alias #{network_alias}"
            end

            @additional_hosts.each do |host|
              command << "--add-host=#{host}"
            end
          end

          return unless runner_network

          Docker::Command.execute(
            "network connect --alias #{name}.#{network} --alias #{name}.#{runner_network} #{runner_network} #{name}"
          )
        end

        # Path of the log file to write to
        # @note
        #   if an error occurs during #reconfigure,
        #   "retry-{n}" is prefixed to the file name where {n} is retry number
        # @return [String] the path to the log file
        def log_file
          retries = 0
          log_file = "#{Runtime::Env.host_artifacts_dir}/#{name}-reconfigure.log"
          while File.exist?(log_file)
            break unless File.exist?(log_file)

            retries += 1
            log_file = "#{Runtime::Env.host_artifacts_dir}/#{name}-retry-#{retries}-reconfigure.log"
          end

          log_file
        end

        def get_reconfigure_log_file_from_artefact
          all_reconfigure_log_file = Dir["#{Runtime::Env.host_artifacts_dir}/*reconfigure.log"].sort_by { |f| File.mtime(f) }
          all_reconfigure_log_file.last
        end

        private :log_file

        def reconfigure
          setup_omnibus
          log_file_path = log_file
          config_file = File.open(log_file_path, "w")
          @docker.attach(name) do |line, _wait|
            config_file.write(line)

            if line.include?('There was an error running gitlab-ctl reconfigure')
              Runtime::Logger.error(
                "Failure while running gitlab-ctl reconfigure command. Please check the #{log_file_path} in the artefact for more info"
              )
            end

            # TODO, workaround which allows to detach from the container
            break if line.include?('gitlab Reconfigured!')
          end
        end

        def wait_until_ready
          return if skip_availability_check

          availability = Availability.new(
            name,
            relative_path: relative_path,
            scheme: scheme,
            protocol_port: gitlab_port.first.to_i
          )

          Runtime::Logger.info("Waiting for GitLab to become healthy ...")

          if availability.check(Runtime::Env.gitlab_availability_timeout)
            Runtime::Logger.info("-> GitLab is available at `#{availability.uri}`!".bright)
          else
            abort '-> GitLab unavailable!'.red
          end
        end

        def process_exec_commands
          Support::ConfigScripts.add_git_server_hooks(docker, name) unless skip_server_hooks

          @docker.copy(name, DATA_SEED_PATH, DATA_PATH) if seed_admin_token || seed_db
          exec_commands << seed_admin_token_command if seed_admin_token
          exec_commands << seed_test_data_command if seed_db
          exec_commands << Runtime::Scenario.omnibus_exec_commands

          commands = exec_commands.flatten.uniq
          return if commands.empty?

          Runtime::Logger.info("Running exec_commands...")
          commands.each { |command| @docker.exec(name, command, mask_secrets: secrets) }
        end

        def rails_version
          manifest = JSON.parse(read_package_manifest)
          {
            sha: manifest['software']['gitlab-rails']['locked_version'],
            source: manifest['software']['gitlab-rails']['locked_source']['git']
          }
        end

        def package_version
          manifest = JSON.parse(read_package_manifest)
          manifest['software']['package-scripts']['locked_version']
        end

        def create_key_file(env_key)
          directory = ENV['CI_PROJECT_DIR'] || Dir.tmpdir
          unique_filename = "#{env_key.downcase}_#{Time.now.to_i}_#{rand(100)}"
          key_file_path = File.join(directory, unique_filename)

          File.open(key_file_path, 'w') do |file|
            file.write(ENV.fetch(env_key))
            file.fsync
          end

          File.chmod(0o744, key_file_path)
          @volumes[key_file_path] = key_file_path

          key_file_path
        end

        def delete_key_file(path)
          FileUtils.rm_f(path)
        end

        def teardown!
          log_pg_stats

          super
        end

        private

        attr_reader :cert_volumes

        def read_package_manifest
          @docker.read_file(@release.image, @release.tag, '/opt/gitlab/version-manifest.json')
        end

        # Create cert files in separate volumes
        #
        # tls_certificates folder can't be mounted directly when remote docker context is used
        # due to not having access to local dir
        #
        # @return [void]
        def copy_certificates
          Alpine.perform do |alpine|
            alpine.volumes = cert_volumes

            alpine.start_instance
            docker.copy(alpine.name, "#{CERTIFICATES_PATH}/authority/.", TRUSTED_PATH)
            docker.copy(alpine.name, "#{CERTIFICATES_PATH}/#{gitaly_tls ? 'gitaly' : 'gitlab'}/.", SSL_PATH)
          ensure
            alpine.teardown! # always remove container, even when global `--no-tests` flag was provided
          end
        end

        def ensure_configured!
          raise 'Please configure an instance first!' unless [name, release, network].all?
        end

        def setup_omnibus
          @docker.write_files(name, mask_secrets: secrets) do |f|
            f.write('/etc/gitlab/gitlab.rb', @omnibus_configuration.to_s)
          end
        end

        def seed_test_data_command
          cmd = []

          Runtime::Scenario.seed_db.each do |file_patterns|
            Dir["#{DATA_SEED_PATH}/#{file_patterns}"].map { |f| File.basename f }.each do |file|
              cmd << "gitlab-rails runner #{DATA_PATH}/#{file}"
            end
          end

          cmd.uniq
        end

        def seed_admin_token_command
          ["gitlab-rails runner #{DATA_PATH}/admin_access_token_seed.rb"]
        end

        def log_pg_stats
          Runtime::Logger.debug('Fetching pg statistics')
          File.open("#{Runtime::Env.host_artifacts_dir}/pg_stats.log", 'a') do |file|
            file << "#{Time.now.strftime('%Y-%m-%d %H:%M:%S')} -- #{name} -- Postgres statistics after test run:\n"
            file << "Live and dead row counts:\n"
            file << @docker.exec(name, %(gitlab-psql -c 'select n_live_tup, n_dead_tup, relname from pg_stat_all_tables order by n_live_tup DESC, n_dead_tup DESC;'))
            file << "Cumulative user table statistics:\n"
            file << @docker.exec(name, %(gitlab-psql -c 'select * from pg_stat_user_tables;'))
          end
        rescue StandardError => e
          Runtime::Logger.error("Error getting pg statistics: #{e}")
        end

        class Availability
          def initialize(name, relative_path: '', scheme: 'http', protocol_port: 80)
            @docker = Docker::Engine.new

            @name = name
            @scheme = scheme
            @relative_path = relative_path
            @protocol_port = protocol_port
          end

          def check(retries)
            retries.times do
              return true if service_available?

              sleep 1
            end

            false
          end

          def uri
            @uri ||= begin
              port = docker.port(name, protocol_port).split(':').last

              URI.join("#{scheme}://#{docker.hostname}:#{port}", relative_path)
            end
          end

          private

          attr_reader :docker, :name, :relative_path, :scheme, :protocol_port

          def service_available?
            output = docker.inspect(name) { |command| command << "--format='{{json .State.Health.Status}}'" }

            output == '"healthy"'
          rescue Support::ShellCommand::StatusError
            false
          end
        end
      end
    end
  end
end
# rubocop:enable Metrics/AbcSize