From cc2e3b9358ef1f30e004bbf6e32759a8fb4cf918 Mon Sep 17 00:00:00 2001 From: Devin Howard Date: Fri, 23 Sep 2016 14:29:28 +0800 Subject: [PATCH 1/5] hack to get the tag when importing a url, without CORS issues --- app/controllers/hacks_controller.rb | 35 +++++++++++++++++++++++++++++ app/policies/hack_policy.rb | 5 +++++ config/routes.rb | 4 ++++ frontend/src/Metamaps/PasteInput.js | 12 ++++++++++ 4 files changed, 56 insertions(+) create mode 100644 app/controllers/hacks_controller.rb create mode 100644 app/policies/hack_policy.rb diff --git a/app/controllers/hacks_controller.rb b/app/controllers/hacks_controller.rb new file mode 100644 index 00000000..42bafd6f --- /dev/null +++ b/app/controllers/hacks_controller.rb @@ -0,0 +1,35 @@ +# bad code that should be seriously checked over before entering one of the +# other prim and proper files in the nice section of this repo +class HacksController < ApplicationController + include ActionView::Helpers::TextHelper # string truncate method + + def load_url_title + authorize :Hack + url = params[:url] # TODO verify!?!?!?! + response, url = get_with_redirects(url) + title = get_encoded_title(response) + render json: { success: true, title: title, url: url } + rescue StandardError => e + render json: { success: false, error_type: e.class.name, error_message: e.message } + end + + private + + def get_with_redirects(url) + uri = URI.parse(url) + response = Net::HTTP.get_response(uri) + while response.code == '301' + uri = URI.parse(response['location']) + response = Net::HTTP.get_response(uri) + end + [response, uri.to_s] + end + + def get_encoded_title(http_response) + title = http_response.body.sub(/.*<title>(.*)<\/title>.*/m, '\1') + charset = http_response['content-type'].sub(/.*charset=(.*);?.*/, '\1') + charset = nil if charset == 'text/html' + title = title.force_encoding(charset) if charset + truncate(title, length: 140) + end +end diff --git a/app/policies/hack_policy.rb b/app/policies/hack_policy.rb new file mode 100644 index 00000000..b6fbf6ce --- /dev/null +++ b/app/policies/hack_policy.rb @@ -0,0 +1,5 @@ +class HackPolicy < ApplicationPolicy + def load_url_title? + true + end +end diff --git a/config/routes.rb b/config/routes.rb index fe48b6ba..84112d23 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -80,4 +80,8 @@ Metamaps::Application.routes.draw do get 'users/:id/details', to: 'users#details', as: :details post 'user/updatemetacodes', to: 'users#updatemetacodes', as: :updatemetacodes resources :users, except: [:index, :destroy] + + namespace :hacks do + get 'load_url_title' + end end diff --git a/frontend/src/Metamaps/PasteInput.js b/frontend/src/Metamaps/PasteInput.js index d14a0cf4..13258857 100644 --- a/frontend/src/Metamaps/PasteInput.js +++ b/frontend/src/Metamaps/PasteInput.js @@ -88,6 +88,18 @@ const PasteInput = { import_id, { success: function(topic) { + $.get('/hacks/load_url_title', { + url: text + }, function success(data, textStatus) { + var selector = '#showcard #topic_' + topic.get('id') + ' .best_in_place' + if ($(selector).find('form').length > 0) { + $(selector).find('textarea, input').val(data.title) + } else { + $(selector).html(data.title) + } + topic.set('name', data.title) + topic.save() + }) TopicCard.showCard(topic.get('node'), function() { $('#showcard #titleActivator').click() .find('textarea, input').focus() From ceb26997606d9ca9f464b2f2647aaf86efc78945 Mon Sep 17 00:00:00 2001 From: Devin Howard <devin@callysto.com> Date: Sun, 25 Sep 2016 22:54:40 +0800 Subject: [PATCH 2/5] install rack-attack --- Gemfile | 1 + Gemfile.lock | 3 +++ 2 files changed, 4 insertions(+) diff --git a/Gemfile b/Gemfile index b4a0967b..d5b42d83 100644 --- a/Gemfile +++ b/Gemfile @@ -23,6 +23,7 @@ gem 'pg' gem 'pundit' gem 'pundit_extra' gem 'rack-cors' +gem 'rack-attack' gem 'redis' gem 'slack-notifier' gem 'snorlax' diff --git a/Gemfile.lock b/Gemfile.lock index 7e2590c1..23d4c827 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -177,6 +177,8 @@ GEM activesupport (>= 3.0.0) pundit_extra (0.3.0) rack (2.0.1) + rack-attack (5.0.1) + rack rack-cors (0.4.0) rack-test (0.6.3) rack (>= 1.0) @@ -316,6 +318,7 @@ DEPENDENCIES pry-rails pundit pundit_extra + rack-attack rack-cors rails (~> 5.0.0) rails3-jquery-autocomplete From 7f8110b6be5c486b71084fe6683594f1db2bbb23 Mon Sep 17 00:00:00 2001 From: Devin Howard <devin@callysto.com> Date: Sun, 25 Sep 2016 23:00:07 +0800 Subject: [PATCH 3/5] configure rack attack to allow 5r/s for the load_url_title route --- config/application.rb | 2 ++ config/initializers/rack-attack.rb | 15 +++++++++++++++ 2 files changed, 17 insertions(+) create mode 100644 config/initializers/rack-attack.rb diff --git a/config/application.rb b/config/application.rb index b629682a..96505b32 100644 --- a/config/application.rb +++ b/config/application.rb @@ -26,6 +26,8 @@ module Metamaps Doorkeeper::ApplicationController.helper ApplicationHelper end + config.middleware.use Rack::Attack + # Configure sensitive parameters which will be filtered from the log file. config.filter_parameters += [:password] diff --git a/config/initializers/rack-attack.rb b/config/initializers/rack-attack.rb new file mode 100644 index 00000000..6c23e151 --- /dev/null +++ b/config/initializers/rack-attack.rb @@ -0,0 +1,15 @@ +class Rack::Attack +end + +Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new + +# Throttle requests to 5 requests per second per ip +Rack::Attack.throttle('load_url_title/req/ip', :limit => 5, :period => 1.second) do |req| + # If the return value is truthy, the cache key for the return value + # is incremented and compared with the limit. In this case: + # "rack::attack:#{Time.now.to_i/1.second}:load_url_title/req/ip:#{req.ip}" + # + # If falsy, the cache key is neither incremented nor checked. + + req.ip if req.path === 'hacks/load_url_title' +end From 959aa693f357888aa77ff17378ab03ac0a082e00 Mon Sep 17 00:00:00 2001 From: Devin Howard <devin@callysto.com> Date: Sun, 25 Sep 2016 23:06:09 +0800 Subject: [PATCH 4/5] ok, i guess this is ready --- app/controllers/hacks_controller.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/controllers/hacks_controller.rb b/app/controllers/hacks_controller.rb index 42bafd6f..1abe3e60 100644 --- a/app/controllers/hacks_controller.rb +++ b/app/controllers/hacks_controller.rb @@ -1,16 +1,18 @@ -# bad code that should be seriously checked over before entering one of the -# other prim and proper files in the nice section of this repo +# bad code that should be checked over before entering one of the +# nice files from the right side of this repo class HacksController < ApplicationController include ActionView::Helpers::TextHelper # string truncate method + # rate limited by rack-attack - currently 5r/s + # TODO: what else can we do to make get_with_redirects safer? def load_url_title authorize :Hack - url = params[:url] # TODO verify!?!?!?! + url = params[:url] response, url = get_with_redirects(url) title = get_encoded_title(response) render json: { success: true, title: title, url: url } rescue StandardError => e - render json: { success: false, error_type: e.class.name, error_message: e.message } + render json: { success: false } end private From eed5ff76efc021fad81b363874ca86f4c615fa56 Mon Sep 17 00:00:00 2001 From: Devin Howard <devin@callysto.com> Date: Sun, 25 Sep 2016 23:21:51 +0800 Subject: [PATCH 5/5] add rate limiting headers --- config/initializers/rack-attack.rb | 64 +++++++++++++++++++++++++----- 1 file changed, 54 insertions(+), 10 deletions(-) diff --git a/config/initializers/rack-attack.rb b/config/initializers/rack-attack.rb index 6c23e151..9dfe3746 100644 --- a/config/initializers/rack-attack.rb +++ b/config/initializers/rack-attack.rb @@ -1,15 +1,59 @@ class Rack::Attack -end + Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new -Rack::Attack.cache.store = ActiveSupport::Cache::MemoryStore.new - -# Throttle requests to 5 requests per second per ip -Rack::Attack.throttle('load_url_title/req/ip', :limit => 5, :period => 1.second) do |req| - # If the return value is truthy, the cache key for the return value - # is incremented and compared with the limit. In this case: - # "rack::attack:#{Time.now.to_i/1.second}:load_url_title/req/ip:#{req.ip}" + # Throttle all requests by IP (60rpm) # - # If falsy, the cache key is neither incremented nor checked. + # Key: "rack::attack:#{Time.now.to_i/:period}:req/ip:#{req.ip}" + throttle('req/ip', :limit => 300, :period => 5.minutes) do |req| + req.ip # unless req.path.start_with?('/assets') + end - req.ip if req.path === 'hacks/load_url_title' + # Throttle POST requests to /login by IP address + # + # Key: "rack::attack:#{Time.now.to_i/:period}:logins/ip:#{req.ip}" + throttle('logins/ip', :limit => 5, :period => 20.seconds) do |req| + if req.path == '/login' && req.post? + req.ip + end + end + + # Throttle POST requests to /login by email param + # + # Key: "rack::attack:#{Time.now.to_i/:period}:logins/email:#{req.email}" + # + # Note: This creates a problem where a malicious user could intentionally + # throttle logins for another user and force their login requests to be + # denied, but that's not very common and shouldn't happen to you. (Knock + # on wood!) + throttle("logins/email", :limit => 5, :period => 20.seconds) do |req| + if req.path == '/login' && req.post? + # return the email if present, nil otherwise + req.params['email'].presence + end + end + + throttle('load_url_title/req/ip', :limit => 5, :period => 1.second) do |req| + # If the return value is truthy, the cache key for the return value + # is incremented and compared with the limit. In this case: + # "rack::attack:#{Time.now.to_i/1.second}:load_url_title/req/ip:#{req.ip}" + # + # If falsy, the cache key is neither incremented nor checked. + + req.ip if req.path == 'hacks/load_url_title' + end + + self.throttled_response = lambda do |env| + now = Time.now + match_data = env['rack.attack.match_data'] + period = match_data[:period] + limit = match_data[:limit] + + headers = { + 'X-RateLimit-Limit' => limit.to_s, + 'X-RateLimit-Remaining' => '0', + 'X-RateLimit-Reset' => (now + (period - now.to_i % period)).to_s + } + + [429, headers, ['']] + end end