Skip to content

Commit

Permalink
refactoring the server object to not require curb, but use it if its …
Browse files Browse the repository at this point in the history
…installed... should help with non-MRI-based interpreters
  • Loading branch information
HamptonMakes committed Oct 25, 2009
1 parent 14b1f34 commit 2bba31f
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 38 deletions.
82 changes: 46 additions & 36 deletions app/models/server.rb
Original file line number Diff line number Diff line change
Expand Up @@ -36,47 +36,47 @@ def file(title)
# You can pass in multiple paths, and it will try each one until it gets a 200
def fetch(*paths)
paths.each do |path|
begin
Merb.logger.debug("loading... " + base_url + path)
result = nil

result = fetch_from_web(path)
Merb.logger.debug("loading... " + base_url + path)

compressed_size = result.downloaded_content_length
result = fetch_from_web(path)


Merb.logger.debug("loaded #{result.downloaded_content_length} compressed characters")
compressed_size = result.downloaded_content_length

if result.response_code == 200

if result.header_str.include?("Cache-Lookup: HIT")
Merb.logger.debug("Spider HIT")
else
Merb.logger.debug("Spider MISS")
end
Merb.logger.debug("loaded #{result.downloaded_content_length} compressed characters")

body = nil

if result.response_code == 200

if result.header_str.include?("Cache-Lookup: HIT")
Merb.logger.debug("Spider HIT")
else
Merb.logger.debug("Spider MISS")
end

body = nil


time_to "decompress" do

time_to "decompress" do

begin
gz = Zlib::GzipReader.new( StringIO.new( result.body_str ) )
body = gz.read
rescue Zlib::GzipFile::Error
# If its not looking gzipped, just display it
body = result.body_str
end

body = body.force_encoding("UTF-8")
begin
gz = Zlib::GzipReader.new( StringIO.new( result.body_str ) )
body = gz.read
rescue Zlib::GzipFile::Error
# If its not looking gzipped, just display it
body = result.body_str
end

Merb.logger.debug("Decompressed to #{body.size} characters")

return {:url => result.last_effective_url, :body => body}
body = body.force_encoding("UTF-8")
end

rescue Curl::Err::HostResolutionError, Curl::Err::GotNothingError, Curl::Err::ConnectionFailedError, Curl::Err::PartialFileError
Merb.logger.error("Could not connect to " + base_url + path)

Merb.logger.debug("Decompressed to #{body.size} characters")

return {:url => result.last_effective_url, :body => body}
end


end

return {}
Expand All @@ -85,11 +85,21 @@ def fetch(*paths)
# :api: private
def fetch_from_web(path)
time_to "fetch" do
Curl::Easy.perform(base_url + path) do |curl|
# This configures Curl::Easy to follow redirects
curl.follow_location = true
curl.headers = {"Accept-Encoding" => "compress, gzip"}
end
if defined?(Curl)
begin
Curl::Easy.perform(base_url + path) do |curl|
# This configures Curl::Easy to follow redirects
curl.follow_location = true
curl.headers = {"Accept-Encoding" => "compress, gzip"}
end
rescue Curl::Err::HostResolutionError, Curl::Err::GotNothingError, Curl::Err::ConnectionFailedError, Curl::Err::PartialFileError
Merb.logger.error("Could not connect to " + base_url + path)
end
else
# This is for if we are using a non-curl supported version of Ruby
require 'open-uri'
open(base_url + path).read
end
end
end
end
7 changes: 6 additions & 1 deletion config/init.rb
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,16 @@ def is19?

require 'sass'
require 'cgi'
require 'curb'
require 'merb-haml'
require 'nokogiri'
require 'mime/types'

begin
require 'curb'
rescue
puts "no curb installed.. using open-uri"
end

if is19?
Encoding.default_internal = Encoding.default_external = "UTF-8"
require Merb.root / 'merb' / 'monkey' / 'haml_fix'
Expand Down
7 changes: 6 additions & 1 deletion spec/models/article_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,12 @@

before(:all) do
# Stub out the device
@device = Device.new(nil)
request = mock("request")
request.stub!("preferred_format").and_return nil
request.stub!("accept").and_return "xhtml"


@device = Device.new(request)
@device.instance_eval("@user_agent = 'Webkit'")

# Stubs out networking
Expand Down

0 comments on commit 2bba31f

Please sign in to comment.