Merge branch 'master' into glitch-soc/master

Conflicts:
- `streaming/index.js`:
  Upstream entirely refactored it.
  Ported our changes to upstream's refactor. Hopefuly.
This commit is contained in:
Thibaut Girka 2020-08-11 19:19:27 +02:00
commit 78fa15d08f
6 changed files with 835 additions and 339 deletions

View File

@ -1,3 +1,5 @@
// @ts-check
import { connectStream } from '../stream'; import { connectStream } from '../stream';
import { import {
updateTimeline, updateTimeline,
@ -19,24 +21,59 @@ import { getLocale } from '../locales';
const { messages } = getLocale(); const { messages } = getLocale();
export function connectTimelineStream (timelineId, path, pollingRefresh = null, accept = null) { /**
* @param {number} max
* @return {number}
*/
const randomUpTo = max =>
Math.floor(Math.random() * Math.floor(max));
return connectStream (path, pollingRefresh, (dispatch, getState) => { /**
* @param {string} timelineId
* @param {string} channelName
* @param {Object.<string, string>} params
* @param {Object} options
* @param {function(Function, Function): void} [options.fallback]
* @param {function(object): boolean} [options.accept]
* @return {function(): void}
*/
export const connectTimelineStream = (timelineId, channelName, params = {}, options = {}) =>
connectStream(channelName, params, (dispatch, getState) => {
const locale = getState().getIn(['meta', 'locale']); const locale = getState().getIn(['meta', 'locale']);
let pollingId;
/**
* @param {function(Function, Function): void} fallback
*/
const useFallback = fallback => {
fallback(dispatch, () => {
pollingId = setTimeout(() => useFallback(fallback), 20000 + randomUpTo(20000));
});
};
return { return {
onConnect() { onConnect() {
dispatch(connectTimeline(timelineId)); dispatch(connectTimeline(timelineId));
if (pollingId) {
clearTimeout(pollingId);
pollingId = null;
}
}, },
onDisconnect() { onDisconnect() {
dispatch(disconnectTimeline(timelineId)); dispatch(disconnectTimeline(timelineId));
if (options.fallback) {
pollingId = setTimeout(() => useFallback(options.fallback), randomUpTo(40000));
}
}, },
onReceive (data) { onReceive (data) {
switch(data.event) { switch(data.event) {
case 'update': case 'update':
dispatch(updateTimeline(timelineId, JSON.parse(data.payload), accept)); dispatch(updateTimeline(timelineId, JSON.parse(data.payload), options.accept));
break; break;
case 'delete': case 'delete':
dispatch(deleteFromTimelines(data.payload)); dispatch(deleteFromTimelines(data.payload));
@ -63,17 +100,59 @@ export function connectTimelineStream (timelineId, path, pollingRefresh = null,
}, },
}; };
}); });
}
/**
* @param {Function} dispatch
* @param {function(): void} done
*/
const refreshHomeTimelineAndNotification = (dispatch, done) => { const refreshHomeTimelineAndNotification = (dispatch, done) => {
dispatch(expandHomeTimeline({}, () => dispatch(expandHomeTimeline({}, () =>
dispatch(expandNotifications({}, () => dispatch(expandNotifications({}, () =>
dispatch(fetchAnnouncements(done)))))); dispatch(fetchAnnouncements(done))))));
}; };
export const connectUserStream = () => connectTimelineStream('home', 'user', refreshHomeTimelineAndNotification); /**
export const connectCommunityStream = ({ onlyMedia } = {}) => connectTimelineStream(`community${onlyMedia ? ':media' : ''}`, `public:local${onlyMedia ? ':media' : ''}`); * @return {function(): void}
export const connectPublicStream = ({ onlyMedia, onlyRemote } = {}) => connectTimelineStream(`public${onlyRemote ? ':remote' : ''}${onlyMedia ? ':media' : ''}`, `public${onlyRemote ? ':remote' : ''}${onlyMedia ? ':media' : ''}`); */
export const connectHashtagStream = (id, tag, local, accept) => connectTimelineStream(`hashtag:${id}${local ? ':local' : ''}`, `hashtag${local ? ':local' : ''}&tag=${tag}`, null, accept); export const connectUserStream = () =>
export const connectDirectStream = () => connectTimelineStream('direct', 'direct'); connectTimelineStream('home', 'user', {}, { fallback: refreshHomeTimelineAndNotification });
export const connectListStream = id => connectTimelineStream(`list:${id}`, `list&list=${id}`);
/**
* @param {Object} options
* @param {boolean} [options.onlyMedia]
* @return {function(): void}
*/
export const connectCommunityStream = ({ onlyMedia } = {}) =>
connectTimelineStream(`community${onlyMedia ? ':media' : ''}`, `public:local${onlyMedia ? ':media' : ''}`);
/**
* @param {Object} options
* @param {boolean} [options.onlyMedia]
* @param {boolean} [options.onlyRemote]
* @return {function(): void}
*/
export const connectPublicStream = ({ onlyMedia, onlyRemote } = {}) =>
connectTimelineStream(`public${onlyRemote ? ':remote' : ''}${onlyMedia ? ':media' : ''}`, `public${onlyRemote ? ':remote' : ''}${onlyMedia ? ':media' : ''}`);
/**
* @param {string} columnId
* @param {string} tagName
* @param {boolean} onlyLocal
* @param {function(object): boolean} accept
* @return {function(): void}
*/
export const connectHashtagStream = (columnId, tagName, onlyLocal, accept) =>
connectTimelineStream(`hashtag:${columnId}${onlyLocal ? ':local' : ''}`, `hashtag${onlyLocal ? ':local' : ''}`, { tag: tagName }, { accept });
/**
* @return {function(): void}
*/
export const connectDirectStream = () =>
connectTimelineStream('direct', 'direct');
/**
* @param {string} listId
* @return {function(): void}
*/
export const connectListStream = listId =>
connectTimelineStream(`list:${listId}`, 'list', { list: listId });

View File

@ -1,87 +1,236 @@
// @ts-check
import WebSocketClient from '@gamestdio/websocket'; import WebSocketClient from '@gamestdio/websocket';
const randomIntUpTo = max => Math.floor(Math.random() * Math.floor(max)); /**
* @type {WebSocketClient | undefined}
*/
let sharedConnection;
const knownEventTypes = [ /**
'update', * @typedef Subscription
'delete', * @property {string} channelName
'notification', * @property {Object.<string, string>} params
'conversation', * @property {function(): void} onConnect
'filters_changed', * @property {function(StreamEvent): void} onReceive
]; * @property {function(): void} onDisconnect
*/
export function connectStream(path, pollingRefresh = null, callbacks = () => ({ onConnect() {}, onDisconnect() {}, onReceive() {} })) { /**
return (dispatch, getState) => { * @typedef StreamEvent
const streamingAPIBaseURL = getState().getIn(['meta', 'streaming_api_base_url']); * @property {string} event
const accessToken = getState().getIn(['meta', 'access_token']); * @property {object} payload
const { onConnect, onDisconnect, onReceive } = callbacks(dispatch, getState); */
let polling = null; /**
* @type {Array.<Subscription>}
*/
const subscriptions = [];
const setupPolling = () => { /**
pollingRefresh(dispatch, () => { * @type {Object.<string, number>}
polling = setTimeout(() => setupPolling(), 20000 + randomIntUpTo(20000)); */
}); const subscriptionCounters = {};
};
const clearPolling = () => { /**
if (polling) { * @param {Subscription} subscription
clearTimeout(polling); */
polling = null; const addSubscription = subscription => {
subscriptions.push(subscription);
};
/**
* @param {Subscription} subscription
*/
const removeSubscription = subscription => {
const index = subscriptions.indexOf(subscription);
if (index !== -1) {
subscriptions.splice(index, 1);
}
};
/**
* @param {Subscription} subscription
*/
const subscribe = ({ channelName, params, onConnect }) => {
const key = channelNameWithInlineParams(channelName, params);
subscriptionCounters[key] = subscriptionCounters[key] || 0;
if (subscriptionCounters[key] === 0) {
sharedConnection.send(JSON.stringify({ type: 'subscribe', stream: channelName, ...params }));
}
subscriptionCounters[key] += 1;
onConnect();
};
/**
* @param {Subscription} subscription
*/
const unsubscribe = ({ channelName, params, onDisconnect }) => {
const key = channelNameWithInlineParams(channelName, params);
subscriptionCounters[key] = subscriptionCounters[key] || 1;
if (subscriptionCounters[key] === 1 && sharedConnection.readyState === WebSocketClient.OPEN) {
sharedConnection.send(JSON.stringify({ type: 'unsubscribe', stream: channelName, ...params }));
}
subscriptionCounters[key] -= 1;
onDisconnect();
};
const sharedCallbacks = {
connected () {
subscriptions.forEach(subscription => subscribe(subscription));
},
received (data) {
const { stream } = data;
subscriptions.filter(({ channelName, params }) => {
const streamChannelName = stream[0];
if (stream.length === 1) {
return channelName === streamChannelName;
} }
};
const subscription = getStream(streamingAPIBaseURL, accessToken, path, { const streamIdentifier = stream[1];
if (['hashtag', 'hashtag:local'].includes(channelName)) {
return channelName === streamChannelName && params.tag === streamIdentifier;
} else if (channelName === 'list') {
return channelName === streamChannelName && params.list === streamIdentifier;
}
return false;
}).forEach(subscription => {
subscription.onReceive(data);
});
},
disconnected () {
subscriptions.forEach(({ onDisconnect }) => onDisconnect());
},
reconnected () {
subscriptions.forEach(subscription => subscribe(subscription));
},
};
/**
* @param {string} channelName
* @param {Object.<string, string>} params
* @return {string}
*/
const channelNameWithInlineParams = (channelName, params) => {
if (Object.keys(params).length === 0) {
return channelName;
}
return `${channelName}&${Object.keys(params).map(key => `${key}=${params[key]}`).join('&')}`;
};
/**
* @param {string} channelName
* @param {Object.<string, string>} params
* @param {function(Function, Function): { onConnect: (function(): void), onReceive: (function(StreamEvent): void), onDisconnect: (function(): void) }} callbacks
* @return {function(): void}
*/
export const connectStream = (channelName, params, callbacks) => (dispatch, getState) => {
const streamingAPIBaseURL = getState().getIn(['meta', 'streaming_api_base_url']);
const accessToken = getState().getIn(['meta', 'access_token']);
const { onConnect, onReceive, onDisconnect } = callbacks(dispatch, getState);
// If we cannot use a websockets connection, we must fall back
// to using individual connections for each channel
if (!streamingAPIBaseURL.startsWith('ws')) {
const connection = createConnection(streamingAPIBaseURL, accessToken, channelNameWithInlineParams(channelName, params), {
connected () { connected () {
if (pollingRefresh) {
clearPolling();
}
onConnect(); onConnect();
}, },
disconnected () {
if (pollingRefresh) {
polling = setTimeout(() => setupPolling(), randomIntUpTo(40000));
}
onDisconnect();
},
received (data) { received (data) {
onReceive(data); onReceive(data);
}, },
reconnected () { disconnected () {
if (pollingRefresh) { onDisconnect();
clearPolling();
pollingRefresh(dispatch);
}
onConnect();
}, },
reconnected () {
onConnect();
},
}); });
const disconnect = () => { return () => {
if (subscription) { connection.close();
subscription.close();
}
clearPolling();
}; };
}
return disconnect; const subscription = {
channelName,
params,
onConnect,
onReceive,
onDisconnect,
}; };
}
addSubscription(subscription);
export default function getStream(streamingAPIBaseURL, accessToken, stream, { connected, received, disconnected, reconnected }) { // If a connection is open, we can execute the subscription right now. Otherwise,
const params = stream.split('&'); // because we have already registered it, it will be executed on connect
stream = params.shift();
if (!sharedConnection) {
sharedConnection = /** @type {WebSocketClient} */ (createConnection(streamingAPIBaseURL, accessToken, '', sharedCallbacks));
} else if (sharedConnection.readyState === WebSocketClient.OPEN) {
subscribe(subscription);
}
return () => {
removeSubscription(subscription);
unsubscribe(subscription);
};
};
const KNOWN_EVENT_TYPES = [
'update',
'delete',
'notification',
'conversation',
'filters_changed',
'encrypted_message',
'announcement',
'announcement.delete',
'announcement.reaction',
];
/**
* @param {MessageEvent} e
* @param {function(StreamEvent): void} received
*/
const handleEventSourceMessage = (e, received) => {
received({
event: e.type,
payload: e.data,
});
};
/**
* @param {string} streamingAPIBaseURL
* @param {string} accessToken
* @param {string} channelName
* @param {{ connected: Function, received: function(StreamEvent): void, disconnected: Function, reconnected: Function }} callbacks
* @return {WebSocketClient | EventSource}
*/
const createConnection = (streamingAPIBaseURL, accessToken, channelName, { connected, received, disconnected, reconnected }) => {
const params = channelName.split('&');
channelName = params.shift();
if (streamingAPIBaseURL.startsWith('ws')) { if (streamingAPIBaseURL.startsWith('ws')) {
params.unshift(`stream=${stream}`);
const ws = new WebSocketClient(`${streamingAPIBaseURL}/api/v1/streaming/?${params.join('&')}`, accessToken); const ws = new WebSocketClient(`${streamingAPIBaseURL}/api/v1/streaming/?${params.join('&')}`, accessToken);
ws.onopen = connected; ws.onopen = connected;
@ -92,11 +241,19 @@ export default function getStream(streamingAPIBaseURL, accessToken, stream, { co
return ws; return ws;
} }
stream = stream.replace(/:/g, '/'); channelName = channelName.replace(/:/g, '/');
if (channelName.endsWith(':media')) {
channelName = channelName.replace('/media', '');
params.push('only_media=true');
}
params.push(`access_token=${accessToken}`); params.push(`access_token=${accessToken}`);
const es = new EventSource(`${streamingAPIBaseURL}/api/v1/streaming/${stream}?${params.join('&')}`);
const es = new EventSource(`${streamingAPIBaseURL}/api/v1/streaming/${channelName}?${params.join('&')}`);
let firstConnect = true; let firstConnect = true;
es.onopen = () => { es.onopen = () => {
if (firstConnect) { if (firstConnect) {
firstConnect = false; firstConnect = false;
@ -105,15 +262,12 @@ export default function getStream(streamingAPIBaseURL, accessToken, stream, { co
reconnected(); reconnected();
} }
}; };
for (let type of knownEventTypes) {
es.addEventListener(type, (e) => { KNOWN_EVENT_TYPES.forEach(type => {
received({ es.addEventListener(type, e => handleEventSourceMessage(/** @type {MessageEvent} */ (e), received));
event: e.type, });
payload: e.data,
}); es.onerror = /** @type {function(): void} */ (disconnected);
});
}
es.onerror = disconnected;
return es; return es;
}; };

View File

@ -30,7 +30,7 @@ class Form::CustomEmojiBatch
private private
def custom_emojis def custom_emojis
CustomEmoji.where(id: custom_emoji_ids) @custom_emojis ||= CustomEmoji.where(id: custom_emoji_ids)
end end
def update! def update!

View File

@ -173,11 +173,7 @@ Rails.application.routes.draw do
get '/dashboard', to: 'dashboard#index' get '/dashboard', to: 'dashboard#index'
resources :domain_allows, only: [:new, :create, :show, :destroy] resources :domain_allows, only: [:new, :create, :show, :destroy]
resources :domain_blocks, only: [:new, :create, :show, :destroy, :update] do resources :domain_blocks, only: [:new, :create, :show, :destroy, :update, :edit]
member do
get :edit
end
end
resources :email_domain_blocks, only: [:index, :new, :create, :destroy] resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
resources :action_logs, only: [:index] resources :action_logs, only: [:index]

View File

@ -89,7 +89,7 @@ module Mastodon
path_segments = object.key.split('/') path_segments = object.key.split('/')
path_segments.delete('cache') path_segments.delete('cache')
if path_segments.size != 7 unless [7, 10].include?(path_segments.size)
progress.log(pastel.yellow("Unrecognized file found: #{object.key}")) progress.log(pastel.yellow("Unrecognized file found: #{object.key}"))
next next
end end
@ -133,7 +133,7 @@ module Mastodon
path_segments = key.split(File::SEPARATOR) path_segments = key.split(File::SEPARATOR)
path_segments.delete('cache') path_segments.delete('cache')
if path_segments.size != 7 unless [7, 10].include?(path_segments.size)
progress.log(pastel.yellow("Unrecognized file found: #{key}")) progress.log(pastel.yellow("Unrecognized file found: #{key}"))
next next
end end
@ -258,7 +258,7 @@ module Mastodon
path_segments = path.split('/')[2..-1] path_segments = path.split('/')[2..-1]
path_segments.delete('cache') path_segments.delete('cache')
if path_segments.size != 7 unless [7, 10].include?(path_segments.size)
say('Not a media URL', :red) say('Not a media URL', :red)
exit(1) exit(1)
end end
@ -311,7 +311,7 @@ module Mastodon
segments = object.key.split('/') segments = object.key.split('/')
segments.delete('cache') segments.delete('cache')
next if segments.size != 7 next unless [7, 10].include?(segments.size)
model_name = segments.first.classify model_name = segments.first.classify
record_id = segments[2..-2].join.to_i record_id = segments[2..-2].join.to_i

File diff suppressed because it is too large Load Diff