Fix #463 - Fetch and display previews of URLs using OpenGraph tags

This commit is contained in:
Eugen Rochko 2017-01-20 01:00:14 +01:00
parent 8d0284f8d9
commit f0de621e76
26 changed files with 302 additions and 7 deletions

View File

@ -48,6 +48,7 @@ gem 'rails-settings-cached'
gem 'pg_search' gem 'pg_search'
gem 'simple-navigation' gem 'simple-navigation'
gem 'statsd-instrument' gem 'statsd-instrument'
gem 'ruby-oembed', require: 'oembed'
gem 'react-rails' gem 'react-rails'
gem 'browserify-rails' gem 'browserify-rails'

View File

@ -334,6 +334,7 @@ GEM
rainbow (>= 1.99.1, < 3.0) rainbow (>= 1.99.1, < 3.0)
ruby-progressbar (~> 1.7) ruby-progressbar (~> 1.7)
unicode-display_width (~> 1.0, >= 1.0.1) unicode-display_width (~> 1.0, >= 1.0.1)
ruby-oembed (0.10.1)
ruby-progressbar (1.8.1) ruby-progressbar (1.8.1)
safe_yaml (1.0.4) safe_yaml (1.0.4)
sass (3.4.22) sass (3.4.22)
@ -457,6 +458,7 @@ DEPENDENCIES
rspec-rails rspec-rails
rspec-sidekiq rspec-sidekiq
rubocop rubocop
ruby-oembed
sass-rails (~> 5.0) sass-rails (~> 5.0)
sdoc (~> 0.4.0) sdoc (~> 0.4.0)
sidekiq sidekiq

View File

@ -0,0 +1,40 @@
import api from '../api';
export const STATUS_CARD_FETCH_REQUEST = 'STATUS_CARD_FETCH_REQUEST';
export const STATUS_CARD_FETCH_SUCCESS = 'STATUS_CARD_FETCH_SUCCESS';
export const STATUS_CARD_FETCH_FAIL = 'STATUS_CARD_FETCH_FAIL';
export function fetchStatusCard(id) {
return (dispatch, getState) => {
dispatch(fetchStatusCardRequest(id));
api(getState).get(`/api/v1/statuses/${id}/card`).then(response => {
dispatch(fetchStatusCardSuccess(id, response.data));
}).catch(error => {
dispatch(fetchStatusCardFail(id, error));
});
};
};
export function fetchStatusCardRequest(id) {
return {
type: STATUS_CARD_FETCH_REQUEST,
id
};
};
export function fetchStatusCardSuccess(id, card) {
return {
type: STATUS_CARD_FETCH_SUCCESS,
id,
card
};
};
export function fetchStatusCardFail(id, error) {
return {
type: STATUS_CARD_FETCH_FAIL,
id,
error
};
};

View File

@ -1,6 +1,7 @@
import api from '../api'; import api from '../api';
import { deleteFromTimelines } from './timelines'; import { deleteFromTimelines } from './timelines';
import { fetchStatusCard } from './cards';
export const STATUS_FETCH_REQUEST = 'STATUS_FETCH_REQUEST'; export const STATUS_FETCH_REQUEST = 'STATUS_FETCH_REQUEST';
export const STATUS_FETCH_SUCCESS = 'STATUS_FETCH_SUCCESS'; export const STATUS_FETCH_SUCCESS = 'STATUS_FETCH_SUCCESS';
@ -31,6 +32,7 @@ export function fetchStatus(id) {
api(getState).get(`/api/v1/statuses/${id}`).then(response => { api(getState).get(`/api/v1/statuses/${id}`).then(response => {
dispatch(fetchStatusSuccess(response.data, skipLoading)); dispatch(fetchStatusSuccess(response.data, skipLoading));
dispatch(fetchContext(id)); dispatch(fetchContext(id));
dispatch(fetchStatusCard(id));
}).catch(error => { }).catch(error => {
dispatch(fetchStatusFail(id, error, skipLoading)); dispatch(fetchStatusFail(id, error, skipLoading));
}); });

View File

@ -0,0 +1,96 @@
import PureRenderMixin from 'react-addons-pure-render-mixin';
import ImmutablePropTypes from 'react-immutable-proptypes';
const outerStyle = {
display: 'flex',
cursor: 'pointer',
fontSize: '14px',
border: '1px solid #363c4b',
borderRadius: '4px',
color: '#616b86',
marginTop: '14px',
textDecoration: 'none',
overflow: 'hidden'
};
const contentStyle = {
flex: '2',
padding: '8px',
paddingLeft: '14px'
};
const titleStyle = {
display: 'block',
fontWeight: '500',
marginBottom: '5px',
color: '#d9e1e8'
};
const descriptionStyle = {
color: '#d9e1e8'
};
const imageOuterStyle = {
flex: '1',
background: '#373b4a'
};
const imageStyle = {
display: 'block',
width: '100%',
height: 'auto',
margin: '0',
borderRadius: '4px 0 0 4px'
};
const hostStyle = {
display: 'block',
marginTop: '5px',
fontSize: '13px'
};
const getHostname = url => {
const parser = document.createElement('a');
parser.href = url;
return parser.hostname;
};
const Card = React.createClass({
propTypes: {
card: ImmutablePropTypes.map
},
mixins: [PureRenderMixin],
render () {
const { card } = this.props;
if (card === null) {
return null;
}
let image = '';
if (card.get('image')) {
image = (
<div style={imageOuterStyle}>
<img src={card.get('image')} alt={card.get('title')} style={imageStyle} />
</div>
);
}
return (
<a style={outerStyle} href={card.get('url')} className='status-card'>
{image}
<div style={contentStyle}>
<strong style={titleStyle}>{card.get('title')}</strong>
<p style={descriptionStyle}>{card.get('description')}</p>
<span style={hostStyle}>{getHostname(card.get('url'))}</span>
</div>
</a>
);
}
});
export default Card;

View File

@ -7,6 +7,7 @@ import MediaGallery from '../../../components/media_gallery';
import VideoPlayer from '../../../components/video_player'; import VideoPlayer from '../../../components/video_player';
import { Link } from 'react-router'; import { Link } from 'react-router';
import { FormattedDate, FormattedNumber } from 'react-intl'; import { FormattedDate, FormattedNumber } from 'react-intl';
import CardContainer from '../containers/card_container';
const DetailedStatus = React.createClass({ const DetailedStatus = React.createClass({
@ -42,6 +43,8 @@ const DetailedStatus = React.createClass({
} else { } else {
media = <MediaGallery sensitive={status.get('sensitive')} media={status.get('media_attachments')} height={300} onOpenMedia={this.props.onOpenMedia} />; media = <MediaGallery sensitive={status.get('sensitive')} media={status.get('media_attachments')} height={300} onOpenMedia={this.props.onOpenMedia} />;
} }
} else {
media = <CardContainer statusId={status.get('id')} />;
} }
if (status.get('application')) { if (status.get('application')) {

View File

@ -0,0 +1,8 @@
import { connect } from 'react-redux';
import Card from '../components/card';
const mapStateToProps = (state, { statusId }) => ({
card: state.getIn(['cards', statusId], null)
});
export default connect(mapStateToProps)(Card);

View File

@ -0,0 +1,14 @@
import { STATUS_CARD_FETCH_SUCCESS } from '../actions/cards';
import Immutable from 'immutable';
const initialState = Immutable.Map();
export default function cards(state = initialState, action) {
switch(action.type) {
case STATUS_CARD_FETCH_SUCCESS:
return state.set(action.id, Immutable.fromJS(action.card));
default:
return state;
}
};

View File

@ -13,6 +13,7 @@ import search from './search';
import notifications from './notifications'; import notifications from './notifications';
import settings from './settings'; import settings from './settings';
import status_lists from './status_lists'; import status_lists from './status_lists';
import cards from './cards';
export default combineReducers({ export default combineReducers({
timelines, timelines,
@ -28,5 +29,6 @@ export default combineReducers({
relationships, relationships,
search, search,
notifications, notifications,
settings settings,
cards
}); });

View File

@ -680,3 +680,9 @@ button.active i.fa-retweet {
transition-duration: 0.9s; transition-duration: 0.9s;
background-position: 0 -209px; background-position: 0 -209px;
} }
.status-card {
&:hover {
background: #363c4b;
}
}

View File

@ -3,8 +3,8 @@
class Api::V1::StatusesController < ApiController class Api::V1::StatusesController < ApiController
before_action -> { doorkeeper_authorize! :read }, except: [:create, :destroy, :reblog, :unreblog, :favourite, :unfavourite] before_action -> { doorkeeper_authorize! :read }, except: [:create, :destroy, :reblog, :unreblog, :favourite, :unfavourite]
before_action -> { doorkeeper_authorize! :write }, only: [:create, :destroy, :reblog, :unreblog, :favourite, :unfavourite] before_action -> { doorkeeper_authorize! :write }, only: [:create, :destroy, :reblog, :unreblog, :favourite, :unfavourite]
before_action :require_user!, except: [:show, :context, :reblogged_by, :favourited_by] before_action :require_user!, except: [:show, :context, :card, :reblogged_by, :favourited_by]
before_action :set_status, only: [:show, :context, :reblogged_by, :favourited_by] before_action :set_status, only: [:show, :context, :card, :reblogged_by, :favourited_by]
respond_to :json respond_to :json
@ -21,6 +21,10 @@ class Api::V1::StatusesController < ApiController
set_counters_maps(statuses) set_counters_maps(statuses)
end end
def card
@card = PreviewCard.find_by!(status: @status)
end
def reblogged_by def reblogged_by
results = @status.reblogs.paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id]) results = @status.reblogs.paginate_by_max_id(DEFAULT_ACCOUNTS_LIMIT, params[:max_id], params[:since_id])
accounts = Account.where(id: results.map(&:account_id)).map { |a| [a.id, a] }.to_h accounts = Account.where(id: results.map(&:account_id)).map { |a| [a.id, a] }.to_h

View File

@ -0,0 +1,20 @@
# frozen_string_literal: true
class PreviewCard < ApplicationRecord
IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze
belongs_to :status
has_attached_file :image, styles: { original: '120x120#' }, convert_options: { all: '-quality 80 -strip' }
validates :url, presence: true
validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES
validates_attachment_size :image, less_than: 1.megabytes
def save_with_optional_image!
save!
rescue ActiveRecord::RecordInvalid
self.image = nil
save!
end
end

View File

@ -23,6 +23,7 @@ class Status < ApplicationRecord
has_and_belongs_to_many :tags has_and_belongs_to_many :tags
has_one :notification, as: :activity, dependent: :destroy has_one :notification, as: :activity, dependent: :destroy
has_one :preview_card, dependent: :destroy
validates :account, presence: true validates :account, presence: true
validates :uri, uniqueness: true, unless: 'local?' validates :uri, uniqueness: true, unless: 'local?'

View File

@ -0,0 +1,33 @@
# frozen_string_literal: true
class FetchLinkCardService < BaseService
def call(status)
# Get first URL
url = URI.extract(status.text).reject { |uri| (uri =~ /\Ahttps?:\/\//).nil? }.first
return if url.nil?
response = http_client.get(url)
return if response.code != 200
page = Nokogiri::HTML(response.to_s)
card = PreviewCard.where(status: status).first_or_initialize(status: status, url: url)
card.title = meta_property(page, 'og:title') || page.at_xpath('//title')&.content
card.description = meta_property(page, 'og:description') || meta_property(page, 'description')
card.image = URI.parse(meta_property(page, 'og:image')) if meta_property(page, 'og:image')
card.save_with_optional_image!
end
private
def http_client
HTTP.timeout(:per_operation, write: 10, connect: 10, read: 10).follow
end
def meta_property(html, property)
html.at_xpath("//meta[@property=\"#{property}\"]")&.attribute('content')&.value || html.at_xpath("//meta[@name=\"#{property}\"]")&.attribute('content')&.value
end
end

View File

@ -22,6 +22,7 @@ class PostStatusService < BaseService
process_mentions_service.call(status) process_mentions_service.call(status)
process_hashtags_service.call(status) process_hashtags_service.call(status)
LinkCrawlWorker.perform_async(status.id)
DistributionWorker.perform_async(status.id) DistributionWorker.perform_async(status.id)
Pubsubhubbub::DistributionWorker.perform_async(status.stream_entry.id) Pubsubhubbub::DistributionWorker.perform_async(status.stream_entry.id)

View File

@ -0,0 +1,5 @@
object @card
attributes :url, :title, :description
node(:image) { |card| card.image? ? full_asset_url(card.image.url(:original)) : nil }

View File

@ -0,0 +1,13 @@
# frozen_string_literal: true
class LinkCrawlWorker
include Sidekiq::Worker
sidekiq_options retry: false
def perform(status_id)
FetchLinkCardService.new.call(Status.find(status_id))
rescue ActiveRecord::RecordNotFound
true
end
end

View File

@ -3,6 +3,7 @@ require_relative 'boot'
require 'rails/all' require 'rails/all'
require_relative '../app/lib/exceptions' require_relative '../app/lib/exceptions'
require_relative '../lib/statsd_monitor'
# Require the gems listed in Gemfile, including any gems # Require the gems listed in Gemfile, including any gems
# you've limited to :test, :development, or :production. # you've limited to :test, :development, or :production.
@ -30,7 +31,7 @@ module Mastodon
config.active_job.queue_adapter = :sidekiq config.active_job.queue_adapter = :sidekiq
config.middleware.insert(0, 'StatsDMonitor') config.middleware.insert(0, ::StatsDMonitor)
config.middleware.insert_before 0, Rack::Cors do config.middleware.insert_before 0, Rack::Cors do
allow do allow do

View File

@ -12,4 +12,5 @@
ActiveSupport::Inflector.inflections(:en) do |inflect| ActiveSupport::Inflector.inflections(:en) do |inflect|
inflect.acronym 'StatsD' inflect.acronym 'StatsD'
inflect.acronym 'OEmbed'
end end

View File

@ -86,6 +86,7 @@ Rails.application.routes.draw do
resources :statuses, only: [:create, :show, :destroy] do resources :statuses, only: [:create, :show, :destroy] do
member do member do
get :context get :context
get :card
get :reblogged_by get :reblogged_by
get :favourited_by get :favourited_by
@ -146,7 +147,7 @@ Rails.application.routes.draw do
get '/about', to: 'about#index' get '/about', to: 'about#index'
get '/about/more', to: 'about#more' get '/about/more', to: 'about#more'
get '/terms', to: 'about#terms' get '/terms', to: 'about#terms'
root 'home#index' root 'home#index'
match '*unmatched_route', via: :all, to: 'application#raise_not_found' match '*unmatched_route', via: :all, to: 'application#raise_not_found'

View File

@ -0,0 +1,17 @@
class CreatePreviewCards < ActiveRecord::Migration[5.0]
def change
create_table :preview_cards do |t|
t.integer :status_id
t.string :url, null: false, default: ''
# OpenGraph
t.string :title, null: true
t.string :description, null: true
t.attachment :image
t.timestamps
end
add_index :preview_cards, :status_id, unique: true
end
end

View File

@ -10,7 +10,7 @@
# #
# It's strongly recommended that you check this file into your version control system. # It's strongly recommended that you check this file into your version control system.
ActiveRecord::Schema.define(version: 20170114203041) do ActiveRecord::Schema.define(version: 20170119214911) do
# These are extensions that must be enabled in order to support this database # These are extensions that must be enabled in order to support this database
enable_extension "plpgsql" enable_extension "plpgsql"
@ -157,6 +157,20 @@ ActiveRecord::Schema.define(version: 20170114203041) do
t.index ["uid"], name: "index_oauth_applications_on_uid", unique: true, using: :btree t.index ["uid"], name: "index_oauth_applications_on_uid", unique: true, using: :btree
end end
create_table "preview_cards", force: :cascade do |t|
t.integer "status_id"
t.string "url", default: "", null: false
t.string "title"
t.string "description"
t.string "image_file_name"
t.string "image_content_type"
t.integer "image_file_size"
t.datetime "image_updated_at"
t.datetime "created_at", null: false
t.datetime "updated_at", null: false
t.index ["status_id"], name: "index_preview_cards_on_status_id", unique: true, using: :btree
end
create_table "pubsubhubbub_subscriptions", force: :cascade do |t| create_table "pubsubhubbub_subscriptions", force: :cascade do |t|
t.string "topic", default: "", null: false t.string "topic", default: "", null: false
t.string "callback", default: "", null: false t.string "callback", default: "", null: false

View File

@ -0,0 +1,5 @@
Fabricator(:preview_card) do
status_id 1
url "MyString"
html "MyText"
end

View File

@ -0,0 +1,5 @@
require 'rails_helper'
RSpec.describe PreviewCard, type: :model do
end

View File

@ -1,5 +1,5 @@
require 'rails_helper' require 'rails_helper'
RSpec.describe Subscription, type: :model do RSpec.describe Subscription, type: :model do
pending "add some examples to (or delete) #{__FILE__}"
end end