class Bundler::Fetcher - bundler-1.9.9 Documentation

# class Bundler::Fetcher

Handles all the fetching with the rubygems server

### Constants

AUTH_ERRORS

  Exceptions classes that should bypass retry attempts. If your password didn't work the first time, it's not going to the third time.

HTTP_ERRORS

### Attributes

api_timeout[RW]
disable_endpoint[RW]
max_retries[RW]
redirect_limit[RW]

### Public Class Methods

download_gem_from_uri(spec, uri) click to toggle source

    # File lib/bundler/fetcher.rb, line 56
    def download_gem_from_uri(spec, uri)
      spec.fetch_platform

      download_path = Bundler.requires_sudo? ? Bundler.tmp(spec.full_name) : Bundler.rubygems.gem_dir
      gem_path = "#{Bundler.rubygems.gem_dir}/cache/#{spec.full_name}.gem"

      FileUtils.mkdir_p("#{download_path}/cache")
      Bundler.rubygems.download_gem(spec, uri, download_path)

      if Bundler.requires_sudo?
        Bundler.mkdir_p "#{Bundler.rubygems.gem_dir}/cache"
        Bundler.sudo "mv #{download_path}/cache/#{spec.full_name}.gem #{gem_path}"
      end

      gem_path
    ensure
      Bundler.rm_rf(download_path) if Bundler.requires_sudo?
    end

new(remote_uri) click to toggle source

    # File lib/bundler/fetcher.rb, line 106
    def initialize(remote_uri)
      @redirect_limit = 5  # How many redirects to allow in one request
      @api_timeout    = 10 # How long to wait for each API call
      @max_retries    = 3  # How many retries for the API call

      @anonymizable_uri = configured_uri_for(remote_uri)

      Socket.do_not_reverse_lookup = true
      connection # create persistent connection
    end

user_agent() click to toggle source

    # File lib/bundler/fetcher.rb, line 75
    def user_agent
      @user_agent ||= begin
        ruby = Bundler.ruby_version

        agent = "bundler/#{Bundler::VERSION}"
        agent << " rubygems/#{Gem::VERSION}"
        agent << " ruby/#{ruby.version}"
        agent << " (#{ruby.host})"
        agent << " command/#{ARGV.first}"

        if ruby.engine != "ruby"
          # engine_version raises on unknown engines
          engine_version = ruby.engine_version rescue "???"
          agent << " #{ruby.engine}/#{engine_version}"
        end

        agent << " options/#{Bundler.settings.all.join(",")}"

        # add a random ID so we can consolidate runs server-side
        agent << " " << SecureRandom.hex(8)

        # add any user agent strings set in the config
        extra_ua = Bundler.settings[:user_agent]
        agent << " " << extra_ua if extra_ua

        agent
      end
    end

### Public Instance Methods

connection() click to toggle source

    # File lib/bundler/fetcher.rb, line 117
    def connection
      @connection ||= begin
        needs_ssl = remote_uri.scheme == "https" ||
          Bundler.settings[:ssl_verify_mode] ||
          Bundler.settings[:ssl_client_cert]
        raise SSLError if needs_ssl && !defined?(OpenSSL::SSL)

        con = Net::HTTP::Persistent.new 'bundler', :ENV

        if remote_uri.scheme == "https"
          con.verify_mode = (Bundler.settings[:ssl_verify_mode] ||
            OpenSSL::SSL::VERIFY_PEER)
          con.cert_store = bundler_cert_store
        end

        if Bundler.settings[:ssl_client_cert]
          pem = File.read(Bundler.settings[:ssl_client_cert])
          con.cert = OpenSSL::X509::Certificate.new(pem)
          con.key  = OpenSSL::PKey::RSA.new(pem)
        end

        con.read_timeout = @api_timeout
        con.override_headers["User-Agent"] = self.class.user_agent
        con
      end
    end

fetch_remote_specs(gem_names, full_dependency_list = [], last_spec_list = []) click to toggle source

fetch index

    # File lib/bundler/fetcher.rb, line 213
    def fetch_remote_specs(gem_names, full_dependency_list = [], last_spec_list = [])
      query_list = gem_names - full_dependency_list

      # only display the message on the first run
      if Bundler.ui.debug?
        Bundler.ui.debug "Query List: #{query_list.inspect}"
      else
        Bundler.ui.info ".", false
      end

      return {remote_uri => last_spec_list} if query_list.empty?

      remote_specs = Bundler::Retry.new("dependency api", AUTH_ERRORS).attempts do
        fetch_dependency_remote_specs(query_list)
      end

      spec_list, deps_list = remote_specs
      returned_gems = spec_list.map {|spec| spec.first }.uniq
      fetch_remote_specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list)
    rescue HTTPError, MarshalError, GemspecError
      Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
      Bundler.ui.debug "could not fetch from the dependency API, trying the full index"
      @use_api = false
      return nil
    end

fetch_spec(spec) click to toggle source

fetch a gem specification

    # File lib/bundler/fetcher.rb, line 149
    def fetch_spec(spec)
      spec = spec - [nil, 'ruby', '']
      spec_file_name = "#{spec.join '-'}.gemspec"

      uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
      if uri.scheme == 'file'
        Bundler.load_marshal Gem.inflate(Gem.read_binary(uri.path))
      elsif cached_spec_path = gemspec_cached_path(spec_file_name)
        Bundler.load_gemspec(cached_spec_path)
      else
        Bundler.load_marshal Gem.inflate(fetch(uri))
      end
    rescue MarshalError
      raise HTTPError, "Gemspec #{spec} contained invalid data.\n"          "Your network or your gem server is probably having issues right now."
    end

gemspec_cached_path(spec_file_name) click to toggle source

cached gem specification path, if one exists

    # File lib/bundler/fetcher.rb, line 167
    def gemspec_cached_path spec_file_name
      paths = Bundler.rubygems.spec_cache_dirs.map { |dir| File.join(dir, spec_file_name) }
      paths = paths.select {|path| File.file? path }
      paths.first
    end

inspect() click to toggle source

    # File lib/bundler/fetcher.rb, line 256
    def inspect
      "#<#{self.class}:0x#{object_id} uri=#{uri}>"
    end

specs(gem_names, source) click to toggle source

return the specs in the bundler format as an index

    # File lib/bundler/fetcher.rb, line 174
    def specs(gem_names, source)
      old = Bundler.rubygems.sources
      index = Index.new

      if gem_names && use_api
        specs = fetch_remote_specs(gem_names)
      end

      if specs.nil?
        # API errors mean we should treat this as a non-API source
        @use_api = false

        specs = Bundler::Retry.new("source fetch", AUTH_ERRORS).attempts do
          fetch_all_remote_specs
        end
      end

      specs[remote_uri].each do |name, version, platform, dependencies|
        next if name == 'bundler'
        spec = nil
        if dependencies
          spec = EndpointSpecification.new(name, version, platform, dependencies)
        else
          spec = RemoteSpecification.new(name, version, platform, self)
        end
        spec.source = source
        spec.source_uri = @anonymizable_uri
        index << spec
      end

      index
    rescue CertificateFailureError => e
      Bundler.ui.info "" if gem_names && use_api # newline after dots
      raise e
    ensure
      Bundler.rubygems.sources = old
    end

uri() click to toggle source

    # File lib/bundler/fetcher.rb, line 144
    def uri
      @anonymizable_uri.without_credentials
    end

use_api() click to toggle source

    # File lib/bundler/fetcher.rb, line 239
    def use_api
      return @use_api if defined?(@use_api)

      if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint
        @use_api = false
      elsif fetch(dependency_api_uri)
        @use_api = true
      end
    rescue NetworkDownError => e
      raise HTTPError, e.message
    rescue AuthenticationRequiredError
      # We got a 401 from the server. Don't fall back to the full index, just fail.
      raise
    rescue HTTPError
      @use_api = false
    end

### Private Instance Methods

bundler_cert_store() click to toggle source

    # File lib/bundler/fetcher.rb, line 383
    def bundler_cert_store
      store = OpenSSL::X509::Store.new
      if Bundler.settings[:ssl_ca_cert]
        if File.directory? Bundler.settings[:ssl_ca_cert]
          store.add_path Bundler.settings[:ssl_ca_cert]
        else
          store.add_file Bundler.settings[:ssl_ca_cert]
        end
      else
        store.set_default_paths
        certs = File.expand_path("../ssl_certs/*.pem", __FILE__)
        Dir.glob(certs).each { |c| store.add_file c }
      end
      store
    end

configured_uri_for(uri) click to toggle source

    # File lib/bundler/fetcher.rb, line 401
    def configured_uri_for(uri)
      uri = Bundler::Source.mirror_for(uri)
      config_auth = Bundler.settings[uri.to_s] || Bundler.settings[uri.host]
      AnonymizableURI.new(uri, config_auth)
    end

dependency_api_uri(gem_names = []) click to toggle source

    # File lib/bundler/fetcher.rb, line 316
    def dependency_api_uri(gem_names = [])
      uri = fetch_uri + "api/v1/dependencies"
      uri.query = "gems=#{URI.encode(gem_names.join(","))}" if gem_names.any?
      uri
    end

fetch(uri, counter = 0) click to toggle source

    # File lib/bundler/fetcher.rb, line 269
    def fetch(uri, counter = 0)
      raise HTTPError, "Too many redirects" if counter >= @redirect_limit

      response = request(uri)
      Bundler.ui.debug("HTTP #{response.code} #{response.message}")

      case response
      when Net::HTTPRedirection
        new_uri = URI.parse(response["location"])
        if new_uri.host == uri.host
          new_uri.user = uri.user
          new_uri.password = uri.password
        end
        fetch(new_uri, counter + 1)
      when Net::HTTPSuccess
        response.body
      when Net::HTTPRequestEntityTooLarge
        raise FallbackError, response.body
      when Net::HTTPUnauthorized
        raise AuthenticationRequiredError, remote_uri.host
      else
        raise HTTPError, "#{response.class}: #{response.body}"
      end
    end

fetch_all_remote_specs() click to toggle source

fetch from modern index: specs.4.8.gz

    # File lib/bundler/fetcher.rb, line 347
    def fetch_all_remote_specs
      old_sources = Bundler.rubygems.sources
      Bundler.rubygems.sources = [remote_uri.to_s]
      Bundler.rubygems.fetch_all_remote_specs
    rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError => e
      case e.message
      when /certificate verify failed/
        raise CertificateFailureError.new(uri)
      when /401/
        raise AuthenticationRequiredError, remote_uri
      when /403/
        if remote_uri.userinfo
          raise BadAuthenticationError, remote_uri
        else
          raise AuthenticationRequiredError, remote_uri
        end
      else
        Bundler.ui.trace e
        raise HTTPError, "Could not fetch specs from #{uri}"
      end
    ensure
      Bundler.rubygems.sources = old_sources
    end

fetch_dependency_remote_specs(gem_names) click to toggle source

fetch from Gemcutter Dependency Endpoint API

    # File lib/bundler/fetcher.rb, line 323
    def fetch_dependency_remote_specs(gem_names)
      Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(',')}"
      gem_list = []
      deps_list = []

      gem_names.each_slice(Source::Rubygems::API_REQUEST_SIZE) do |names|
        marshalled_deps = fetch dependency_api_uri(names)
        gem_list += Bundler.load_marshal(marshalled_deps)
      end

      spec_list = gem_list.map do |s|
        dependencies = s[:dependencies].map do |name, requirement|
          dep = well_formed_dependency(name, requirement.split(", "))
          deps_list << dep.name
          dep
        end

        [s[:name], Gem::Version.new(s[:number]), s[:platform], dependencies]
      end

      [spec_list, deps_list.uniq]
    end

fetch_uri() click to toggle source

    # File lib/bundler/fetcher.rb, line 407
    def fetch_uri
      @fetch_uri ||= begin
        if remote_uri.host == "rubygems.org"
          uri = remote_uri.dup
          uri.host = "bundler.rubygems.org"
          uri
        else
          remote_uri
        end
      end
    end

remote_uri() click to toggle source

    # File lib/bundler/fetcher.rb, line 419
    def remote_uri
      @anonymizable_uri.original_uri
    end

request(uri) click to toggle source

    # File lib/bundler/fetcher.rb, line 294
    def request(uri)
      Bundler.ui.debug "HTTP GET #{uri}"
      req = Net::HTTP::Get.new uri.request_uri
      if uri.user
        user = CGI.unescape(uri.user)
        password = uri.password ? CGI.unescape(uri.password) : nil
        req.basic_auth(user, password)
      end
      connection.request(uri, req)
    rescue OpenSSL::SSL::SSLError
      raise CertificateFailureError.new(uri)
    rescue *HTTP_ERRORS => e
      Bundler.ui.trace e
      case e.message
      when /host down:/, /getaddrinfo: nodename nor servname provided/
        raise NetworkDownError, "Could not reach host #{uri.host}. Check your network "          "connection and try again."
      else
        raise HTTPError, "Network error while fetching #{uri}"
      end
    end

well_formed_dependency(name, *requirements) click to toggle source

    # File lib/bundler/fetcher.rb, line 371
    def well_formed_dependency(name, *requirements)
      Gem::Dependency.new(name, *requirements)
    rescue ArgumentError => e
      illformed = 'Ill-formed requirement ["#<YAML::Syck::DefaultKey'
      raise e unless e.message.include?(illformed)
      puts # we shouldn't print the error message on the "fetching info" status line
      raise GemspecError,
        "Unfortunately, the gem #{s[:name]} (#{s[:number]}) has an invalid "          "gemspec. \nPlease ask the gem author to yank the bad version to fix "          "this issue. For more information, see http://bit.ly/syck-defaultkey."
    end