diff --git a/Gemfile b/Gemfile index 258d5855..534120cb 100644 --- a/Gemfile +++ b/Gemfile @@ -9,6 +9,7 @@ gem "RubyInline" gem "png" gem "rest-client" gem "ruby-openid" +gem "couchrest" group :development do gem 'ruby-debug', :require => 'ruby-debug', :platform => :mri_18 diff --git a/Gemfile.lock b/Gemfile.lock index 9e7ae471..14477ff2 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,6 +16,10 @@ GEM ffi (~> 1.0.6) columnize (0.3.4) configuration (1.2.0) + couchrest (1.1.2) + mime-types (~> 1.15) + multi_json (~> 1.0.0) + rest-client (~> 1.6.1) daemons (1.1.4) diff-lcs (1.1.2) eventmachine (0.12.10) @@ -93,6 +97,7 @@ PLATFORMS DEPENDENCIES RubyInline capybara + couchrest haml json launchy diff --git a/server/sinatra/ReadMe.md b/server/sinatra/ReadMe.md index 4f85e17c..8c568d55 100644 --- a/server/sinatra/ReadMe.md +++ b/server/sinatra/ReadMe.md @@ -52,3 +52,16 @@ The server will create subdirectories with farm for each virtual host name and l The thin web server cannot handle recursive web requests that can happen with federated sites hosted in the same farm. Use webrick instead. Launch it with this command: bundle exec rackup -s webrick -p 1111 + +CouchDB +======= + +By default, all pages, favicons, and server claims are stored in the server's local filesystem. +If you'd prefer to use CouchDB for storage, you need to set two environment variables: + + STORE_TYPE=CouchStore + COUCHDB_URL=https://username:password@some-couchdb-host.com + +If you want to run a farm with CouchDB, you should also set this environment variable: + + FARM_MODE=true diff --git a/server/sinatra/favicon.rb b/server/sinatra/favicon.rb index 8603d262..cbc00042 100644 --- a/server/sinatra/favicon.rb +++ b/server/sinatra/favicon.rb @@ -3,7 +3,7 @@ class Favicon class << self - def create(path) + def create_blob canvas = PNG::Canvas.new 32, 32 light = PNG::Color.from_hsv(256*rand,200,255).rgb() dark = PNG::Color.from_hsv(256*rand,200,125).rgb() @@ -20,8 +20,7 @@ def create(path) light[2]*p + dark[2]*(1-p)) end end - png = PNG.new canvas - png.save path + PNG.new(canvas).to_blob end end end diff --git a/server/sinatra/page.rb b/server/sinatra/page.rb index 8d026f23..c5b5b5fa 100644 --- a/server/sinatra/page.rb +++ b/server/sinatra/page.rb @@ -1,12 +1,13 @@ require 'json' require File.expand_path("../random_id", __FILE__) +require File.expand_path("../stores/all", __FILE__) class PageError < StandardError; end; # Page Class # Handles writing and reading JSON data to and from files. class Page - # class << self + # Directory where pages are to be stored. attr_accessor :directory # Directory where default (pre-existing) pages are stored. @@ -17,27 +18,21 @@ class Page # @param [String] name - The name of the file to retrieve, relative to Page.directory. # @return [Hash] The contents of the retrieved page (parsed JSON). def get(name) - assert_directories_set - + assert_attributes_set path = File.join(directory, name) - - if File.exist? path - load_and_parse path + default_path = File.join(default_directory, name) + page = Store.get_page(path) + if page + page + elsif File.exist?(default_path) + put name, FileStore.get_page(default_path) else - default_path = File.join(default_directory, name) - - if File.exist?(default_path) - FileUtils.mkdir_p File.dirname(path) - FileUtils.cp default_path, path - load_and_parse path - else - put name, {'title'=>name,'story'=>[{'type'=>'factory', 'id'=>RandomId.generate}]} unless File.file? path - end + put name, {'title'=>name,'story'=>[{'type'=>'factory', 'id'=>RandomId.generate}]} end end def exists?(name) - File.exists?(File.join(directory, name)) or File.exist?(File.join(default_directory, name)) + Store.exists?(File.join(directory, name)) or File.exist?(File.join(default_directory, name)) end # Create or update a page @@ -46,20 +41,15 @@ def exists?(name) # @param [Hash] page - The page data to be written to the file (it will be converted to JSON). # @return [Hash] The contents of the retrieved page (parsed JSON). def put(name, page) - assert_directories_set - File.open(File.join(directory, name), 'w') { |file| file.write(JSON.pretty_generate(page)) } - page + assert_attributes_set + path = File.join directory, name + Store.put_page(path, page, :name => name, :directory => directory) end private - def load_and_parse(path) - JSON.parse(File.read(path)) - end - - def assert_directories_set + def assert_attributes_set raise PageError.new('Page.directory must be set') unless directory raise PageError.new('Page.default_directory must be set') unless default_directory end - # end end diff --git a/server/sinatra/server.rb b/server/sinatra/server.rb index 4095c8d2..5237d06c 100644 --- a/server/sinatra/server.rb +++ b/server/sinatra/server.rb @@ -9,6 +9,7 @@ Encoding.default_external = Encoding::UTF_8 +require 'stores/all' require 'random_id' require 'page' require 'favicon' @@ -24,6 +25,8 @@ class Controller < Sinatra::Base set :versions, `git log -10 --oneline` || "no git log" enable :sessions + Store.set ENV['STORE_TYPE'], APP_ROOT + class << self # overridden in test def data_root File.join APP_ROOT, "data" @@ -31,30 +34,28 @@ def data_root end def farm_page - data = File.exists?(File.join(self.class.data_root, "farm")) ? File.join(self.class.data_root, "farm", request.host) : self.class.data_root page = Page.new - page.directory = File.join(data, "pages") + page.directory = File.join data_dir, "pages" page.default_directory = File.join APP_ROOT, "default-data", "pages" - FileUtils.mkdir_p page.directory + Store.mkdir page.directory page end def farm_status - data = File.exists?(File.join(self.class.data_root, "farm")) ? File.join(self.class.data_root, "farm", request.host) : self.class.data_root - status = File.join(data, "status") - FileUtils.mkdir_p status + status = File.join data_dir, "status" + Store.mkdir status status end + def data_dir + Store.farm?(self.class.data_root) ? File.join(self.class.data_root, "farm", request.host) : self.class.data_root + end + def identity default_path = File.join APP_ROOT, "default-data", "status", "local-identity" real_path = File.join farm_status, "local-identity" - unless File.exist? real_path - FileUtils.mkdir_p File.dirname(real_path) - FileUtils.cp default_path, real_path - end - - JSON.parse(File.read(real_path)) + id_data = Store.get_hash real_path + id_data ||= Store.put_hash(real_path, FileStore.get_hash(default_path)) end helpers do @@ -84,7 +85,7 @@ def authenticated? end def claimed? - File.exists? "#{farm_status}/open_id.identity" + Store.exists? "#{farm_status}/open_id.identity" end def authenticate! @@ -123,8 +124,8 @@ def oops status, message when OpenID::Consumer::SUCCESS id = params['openid.identity'] id_file = File.join farm_status, "open_id.identity" - if File.exist?(id_file) - stored_id = File.read(id_file) + stored_id = Store.get_text(id_file) + if stored_id if stored_id == id # login successful authenticate! @@ -132,7 +133,7 @@ def oops status, message oops 403, "This is not your wiki" end else - File.open(id_file, "w") {|f| f << id } + Store.put_text id_file, id # claim successful authenticate! end @@ -154,8 +155,7 @@ def oops status, message content_type 'image/png' cross_origin local = File.join farm_status, 'favicon.png' - Favicon.create local unless File.exists? local - File.read local + Store.get_blob(local) || Store.put_blob(local, Favicon.create_blob) end get '/random.png' do @@ -165,9 +165,8 @@ def oops status, message end content_type 'image/png' - local = File.join farm_status, 'favicon.png' - Favicon.create local - File.read local + path = File.join farm_status, 'favicon.png' + Store.put_blob path, Favicon.create_blob end get '/' do @@ -206,21 +205,21 @@ def oops status, message content_type 'application/json' cross_origin bins = Hash.new {|hash, key| hash[key] = Array.new} - Dir.chdir(farm_page.directory) do - Dir.glob("*").collect do |slug| - dt = Time.now - File.new(slug).mtime - bins[(dt/=60)<1?'Minute':(dt/=60)<1?'Hour':(dt/=24)<1?'Day':(dt/=7)<1?'Week':(dt/=4)<1?'Month':(dt/=3)<1?'Season':(dt/=4)<1?'Year':'Forever']<0 story << {'type' => 'paragraph', 'text' => "

Within a #{key}

", 'id' => RandomId.generate} - bins[key].each do |slug| - page = farm_page.get(slug) - next if page['story'].length == 0 + bins[key].each do |page| + next if page['story'].empty? site = "#{request.host}#{request.port==80 ? '' : ':'+request.port.to_s}" - story << {'type' => 'federatedWiki', 'site' => site, 'slug' => slug, 'title' => page['title'], 'text' => "", 'id' => RandomId.generate} + story << {'type' => 'federatedWiki', 'site' => site, 'slug' => page['name'], 'title' => page['title'], 'text' => "", 'id' => RandomId.generate} end end page = {'title' => 'Recent Changes', 'story' => story} @@ -266,7 +265,7 @@ def oops status, message get %r{^/([a-z0-9-]+)\.json$} do |name| content_type 'application/json' cross_origin - halt 404 unless File.exists? "#{farm_page.directory}/#{name}" or File.exists? "#{farm_page.default_directory}/#{name}" + halt 404 unless Store.exists?("#{farm_page.directory}/#{name}") || Store.exists?("#{farm_page.default_directory}/#{name}") JSON.pretty_generate(farm_page.get(name)) end diff --git a/server/sinatra/stores/all.rb b/server/sinatra/stores/all.rb new file mode 100644 index 00000000..99a0bf27 --- /dev/null +++ b/server/sinatra/stores/all.rb @@ -0,0 +1,3 @@ +require File.expand_path('store', File.dirname(__FILE__)) +require File.expand_path('file', File.dirname(__FILE__)) +require File.expand_path('couch', File.dirname(__FILE__)) diff --git a/server/sinatra/stores/couch.rb b/server/sinatra/stores/couch.rb new file mode 100644 index 00000000..b835276a --- /dev/null +++ b/server/sinatra/stores/couch.rb @@ -0,0 +1,120 @@ +require 'time' # for Time#iso8601 + +class CouchStore < Store + class << self + + attr_writer :db # used by specs + + def db + unless @db + couchdb_server = ENV['COUCHDB_URL'] || raise('please set ENV["COUCHDB_URL"]') + @db = CouchRest.database!("#{couchdb_server}/sfw") + begin + @db.save_doc "_id" => "_design/recent-changes", :views => {} + rescue RestClient::Conflict + # design document already exists, do nothing + end + end + @db + end + + ### GET + + def get_text(path) + path = relative_path(path) + begin + db.get(path)['data'] + rescue RestClient::ResourceNotFound + nil + end + end + + def get_blob(path) + blob = get_text path + Base64.decode64 blob if blob + end + + ### PUT + + def put_text(path, text, metadata={}) + path = relative_path(path) + metadata = metadata.each{ |k,v| metadata[k] = relative_path(v) } + attrs = { + 'data' => text, + 'updated_at' => Time.now.utc.iso8601 + }.merge! metadata + + begin + db.save_doc attrs.merge('_id' => path) + rescue RestClient::Conflict + doc = db.get path + doc.merge! attrs + doc.save + end + text + end + + def put_blob(path, blob) + put_text path, Base64.strict_encode64(blob) + blob + end + + ### COLLECTIONS + + def recently_changed_pages(pages_dir) + pages_dir = relative_path pages_dir + pages_dir_safe = CGI.escape pages_dir + changes = begin + db.view("recent-changes/#{pages_dir_safe}")['rows'] + rescue RestClient::ResourceNotFound + create_view 'recent-changes', pages_dir + db.view("recent-changes/#{pages_dir_safe}")['rows'] + end + + pages = changes.map do |change| + page = JSON.parse change['value']['data'] + page.merge! 'updated_at' => Time.parse(change['value']['updated_at']) + page.merge! 'name' => change['value']['name'] + page + end + + pages + end + + ### UTILITY + + def create_view(design_name, view_name) + design = db.get "_design/#{design_name}" + design['views'][view_name] = { + :map => " + function(doc) { + if (doc.directory == '#{view_name}') + emit(doc._id, doc) + } + " + } + design.save + end + + def farm?(_) + ENV['FARM_MODE'] && !ENV['FARM_MODE'].empty? + end + + def mkdir(_) + # do nothing + end + + def exists?(path) + !(get_text path).nil? + end + + def relative_path(path) + raise "Please set @app_root" unless @app_root + path.match(%r[^#{Regexp.escape @app_root}/?(.+?)$]) ? $1 : path + end + + end + +end + + diff --git a/server/sinatra/stores/file.rb b/server/sinatra/stores/file.rb new file mode 100644 index 00000000..99f366d6 --- /dev/null +++ b/server/sinatra/stores/file.rb @@ -0,0 +1,53 @@ +class FileStore < Store + class << self + + ### GET + + def get_text(path) + File.read path if File.exist? path + end + + alias_method :get_blob, :get_text + + ### PUT + + def put_text(path, text, metadata=nil) + # Note: metadata is ignored for filesystem storage + File.open(path, 'w'){ |file| file.write text } + text + end + + def put_blob(path, blob) + File.open(path, 'wb'){ |file| file.write blob } + blob + end + + ### COLLECTIONS + + def recently_changed_pages(pages_dir) + Dir.chdir(pages_dir) do + Dir.glob("*").collect do |name| + page = get_page(File.join pages_dir, name) + page.merge!({ + 'name' => name, + 'updated_at' => File.new(name).mtime + }) + end + end + end + + ### UTILITY + + def farm?(data_root) + File.exists?(File.join data_root, "farm") + end + + def mkdir(directory) + FileUtils.mkdir_p directory + end + + def exists?(path) + File.exists?(path) + end + end +end diff --git a/server/sinatra/stores/store.rb b/server/sinatra/stores/store.rb new file mode 100644 index 00000000..147fd6ec --- /dev/null +++ b/server/sinatra/stores/store.rb @@ -0,0 +1,36 @@ +class Store + class << self + + attr_writer :app_root + + def set(store_classname, app_root) + @store_class = store_classname ? Kernel.const_get(store_classname) : FileStore + @store_class.app_root = app_root + @store_class + end + + def method_missing(*args) + @store_class.send(*args) + end + + ### GET + + def get_hash(path) + json = get_text path + JSON.parse json if json + end + + alias_method :get_page, :get_hash + + ### PUT + + def put_hash(path, ruby_data, metadata={}) + json = JSON.pretty_generate(ruby_data) + put_text path, json, metadata + ruby_data + end + + alias_method :put_page, :put_hash + + end +end diff --git a/spec/favicon_spec.rb b/spec/favicon_spec.rb index c30860c4..4ff348eb 100644 --- a/spec/favicon_spec.rb +++ b/spec/favicon_spec.rb @@ -15,9 +15,9 @@ describe "create" do it "creates a favicon.png image" do + favicon = Favicon.create_blob favicon_path = File.join(@test_data_dir, 'favicon-test.png') - Favicon.create favicon_path - File.exist?(favicon_path).should be_true + File.open(favicon_path, 'wb') { |file| file.write(favicon) } file = PNG.load_file(favicon_path) file.should be_a(PNG::Canvas) file.width.should == 32 diff --git a/spec/page_spec.rb b/spec/page_spec.rb index f37af9eb..0e8b46c6 100644 --- a/spec/page_spec.rb +++ b/spec/page_spec.rb @@ -2,6 +2,7 @@ describe "Page" do before(:all) do + Store.set 'FileStore', nil @page = Page.new @page.directory = nil @page.default_directory = nil diff --git a/spec/server_spec.rb b/spec/server_spec.rb index 343e0ee4..32b48bf3 100644 --- a/spec/server_spec.rb +++ b/spec/server_spec.rb @@ -71,15 +71,9 @@ end end -describe "GET /welcome-visitors.json" do - before(:all) do - get "/welcome-visitors.json" - @response = last_response - @body = last_response.body - end - +shared_examples_for "GET to JSON resource" do it "returns 200" do - last_response.status.should == 200 + @response.status.should == 200 end it "returns Content-Type application/json" do @@ -91,6 +85,16 @@ JSON.parse(@body) }.should_not raise_error end +end + +describe "GET /welcome-visitors.json" do + before(:all) do + get "/welcome-visitors.json" + @response = last_response + @body = last_response.body + end + + it_behaves_like "GET to JSON resource" context "JSON from GET /welcome-visitors.json" do before(:all) do @@ -101,7 +105,7 @@ @json['title'].class.should == String end - it "has a story arry" do + it "has a story array" do @json['story'].class.should == Array end @@ -115,6 +119,48 @@ end end +describe "GET /recent-changes.json" do + def create_sample_page + page = { "title" => "A Page", "story" => [ { "type" => "paragraph", "text" => "Hello test" } ] } + pages_path = File.join TestDirs::TEST_DATA_DIR, 'pages' + FileUtils.rm_f pages_path + FileUtils.mkdir_p pages_path + page_path = File.join pages_path, 'a-page' + File.open(page_path, 'w'){|file| file.write(page.to_json)} + end + + before(:all) do + create_sample_page + get "/recent-changes.json" + @response = last_response + @body = last_response.body + @json = JSON.parse(@body) + end + + it_behaves_like "GET to JSON resource" + + context "the JSON" do + it "has a title string" do + @json['title'].class.should == String + end + + it "has a story array" do + @json['story'].class.should == Array + end + + it "has the heading 'Within a Minute'" do + @json['story'].first['text'].should == "

Within a Minute

" + @json['story'].first['type'].should == 'paragraph' + end + + it "has a listing of the single recent change" do + @json['story'][1]['slug'].should == "a-page" + @json['story'][1]['title'].should == "A Page" + @json['story'][1]['type'].should == 'federatedWiki' + end + end +end + describe "GET /non-existent-test-page" do before(:all) do @non_existent_page = "#{TestDirs::TEST_DATA_DIR}/pages/non-existent-test-page" diff --git a/spec/stores/couch_spec.rb b/spec/stores/couch_spec.rb new file mode 100644 index 00000000..b6eb3aa5 --- /dev/null +++ b/spec/stores/couch_spec.rb @@ -0,0 +1,74 @@ +require File.dirname(__FILE__) + '/../spec_helper' +require File.dirname(__FILE__) + '/../../server/sinatra/stores/all' + +describe CouchStore do + before :each do + CouchStore.app_root = '' + end + + before :each do + @db = CouchStore.db = double() + @couch_doc = double(:save => nil, :merge! => nil, :[]= => nil) + end + + describe 'put_text' do + it 'should store a string to Couch' do + @db.should_receive(:save_doc) do |hash| + hash['_id'].should == 'some/path/segments' + hash['data'].should == 'value -- any sting data' + end + + CouchStore.put_text('some/path/segments', 'value -- any sting data') + end + + it 'should convert full paths to relative paths' do + CouchStore.app_root = '/home/joe/sfw/' + @db.should_receive(:save_doc) do |hash| + hash['_id'].should == 'data/pages/joes-place' + hash['data'].should == '

Joe\'s Place

' + hash['directory'].should == 'data/pages/' + hash['any_param'].should == '/home/jennifer/sfw/is/not/affected' + end + + CouchStore.put_text '/home/joe/sfw/data/pages/joes-place', '

Joe\'s Place

', { + 'directory' => '/home/joe/sfw/data/pages/', + 'any_param' => '/home/jennifer/sfw/is/not/affected', + } + + end + + it 'should not blow up even when Couch initially raises a "conflict" exception' do + @db.should_receive(:save_doc).and_raise(RestClient::Conflict) + @db.should_receive(:get).and_return(@couch_doc) # .with('same/key/a/second/time') + + CouchStore.put_text('same/key/a/second/time', 'value') + end + + it 'should return the data' do + CouchStore.db = double(:save_doc => nil) + CouchStore.put_text('key', 'value').should == 'value' + end + end + + describe 'get_text' do + it 'retrieve a string from Couch' do + @db.should_receive(:get).with('some/path/segments').and_return('data' => 'some string value') + + CouchStore.get_text('some/path/segments').should == 'some string value' + end + + it 'should not blow up even when Couch raises a "not found" exception' do + @db.should_receive(:get).and_raise(RestClient::ResourceNotFound) + + CouchStore.get_text('not/found/key').should be_nil + end + + it 'should return the data' do + CouchStore.db = double(:get => {'data' => 'value'}) + + CouchStore.get_text('key').should == 'value' + end + end + +end +