Add a per-room mutex to federationapi when processing transactions (#1810)

* Add a per-room mutex to federationapi when processing transactions

This has numerous benefits:
 - Prevents us doing lots of state resolutions in busy rooms. Previously, room forks would always result
   in a state resolution being performed immediately, without checking if we were already doing this in
   a different transaction. Now they will queue up, resulting in fewer calls to `/state_ids`, `/g_m_e`, etc.
 - Prevents memory usage from growing too large as a result and potentially OOMing.

And costs:
 - High traffic rooms will be slightly slower due to head-of-line blocking from other servers,
   though this has always been an issue as roomserver has a per-room mutex already.

* Fix unit tests

* Correct mutex lock ordering
This commit is contained in:
Kegsay 2021-03-30 10:01:32 +01:00 committed by GitHub
parent 0ee1c56ffd
commit f8d3a762c4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 49 additions and 1 deletions

View File

@ -21,6 +21,7 @@ import (
"github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/jsonerror"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api" federationSenderAPI "github.com/matrix-org/dendrite/federationsender/api"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/httputil" "github.com/matrix-org/dendrite/internal/httputil"
keyserverAPI "github.com/matrix-org/dendrite/keyserver/api" keyserverAPI "github.com/matrix-org/dendrite/keyserver/api"
roomserverAPI "github.com/matrix-org/dendrite/roomserver/api" roomserverAPI "github.com/matrix-org/dendrite/roomserver/api"
@ -92,12 +93,13 @@ func Setup(
v2keysmux.Handle("/query", notaryKeys).Methods(http.MethodPost) v2keysmux.Handle("/query", notaryKeys).Methods(http.MethodPost)
v2keysmux.Handle("/query/{serverName}/{keyID}", notaryKeys).Methods(http.MethodGet) v2keysmux.Handle("/query/{serverName}/{keyID}", notaryKeys).Methods(http.MethodGet)
mu := internal.NewMutexByRoom()
v1fedmux.Handle("/send/{txnID}", httputil.MakeFedAPI( v1fedmux.Handle("/send/{txnID}", httputil.MakeFedAPI(
"federation_send", cfg.Matrix.ServerName, keys, wakeup, "federation_send", cfg.Matrix.ServerName, keys, wakeup,
func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse { func(httpReq *http.Request, request *gomatrixserverlib.FederationRequest, vars map[string]string) util.JSONResponse {
return Send( return Send(
httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]), httpReq, request, gomatrixserverlib.TransactionID(vars["txnID"]),
cfg, rsAPI, eduAPI, keyAPI, keys, federation, cfg, rsAPI, eduAPI, keyAPI, keys, federation, mu,
) )
}, },
)).Methods(http.MethodPut, http.MethodOptions) )).Methods(http.MethodPut, http.MethodOptions)

View File

@ -26,6 +26,7 @@ import (
"github.com/getsentry/sentry-go" "github.com/getsentry/sentry-go"
"github.com/matrix-org/dendrite/clientapi/jsonerror" "github.com/matrix-org/dendrite/clientapi/jsonerror"
eduserverAPI "github.com/matrix-org/dendrite/eduserver/api" eduserverAPI "github.com/matrix-org/dendrite/eduserver/api"
"github.com/matrix-org/dendrite/internal"
keyapi "github.com/matrix-org/dendrite/keyserver/api" keyapi "github.com/matrix-org/dendrite/keyserver/api"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/dendrite/setup/config" "github.com/matrix-org/dendrite/setup/config"
@ -98,6 +99,7 @@ func Send(
keyAPI keyapi.KeyInternalAPI, keyAPI keyapi.KeyInternalAPI,
keys gomatrixserverlib.JSONVerifier, keys gomatrixserverlib.JSONVerifier,
federation *gomatrixserverlib.FederationClient, federation *gomatrixserverlib.FederationClient,
mu *internal.MutexByRoom,
) util.JSONResponse { ) util.JSONResponse {
t := txnReq{ t := txnReq{
rsAPI: rsAPI, rsAPI: rsAPI,
@ -107,6 +109,7 @@ func Send(
haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent),
newEvents: make(map[string]bool), newEvents: make(map[string]bool),
keyAPI: keyAPI, keyAPI: keyAPI,
roomsMu: mu,
} }
var txnEvents struct { var txnEvents struct {
@ -163,6 +166,7 @@ type txnReq struct {
federation txnFederationClient federation txnFederationClient
servers []gomatrixserverlib.ServerName servers []gomatrixserverlib.ServerName
serversMutex sync.RWMutex serversMutex sync.RWMutex
roomsMu *internal.MutexByRoom
// local cache of events for auth checks, etc - this may include events // local cache of events for auth checks, etc - this may include events
// which the roomserver is unaware of. // which the roomserver is unaware of.
haveEvents map[string]*gomatrixserverlib.HeaderedEvent haveEvents map[string]*gomatrixserverlib.HeaderedEvent
@ -494,6 +498,8 @@ func (t *txnReq) getServers(ctx context.Context, roomID string) []gomatrixserver
} }
func (t *txnReq) processEvent(ctx context.Context, e *gomatrixserverlib.Event) error { func (t *txnReq) processEvent(ctx context.Context, e *gomatrixserverlib.Event) error {
t.roomsMu.Lock(e.RoomID())
defer t.roomsMu.Unlock(e.RoomID())
logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID()) logger := util.GetLogger(ctx).WithField("event_id", e.EventID()).WithField("room_id", e.RoomID())
t.work = "" // reset from previous event t.work = "" // reset from previous event

View File

@ -9,6 +9,7 @@ import (
"time" "time"
eduAPI "github.com/matrix-org/dendrite/eduserver/api" eduAPI "github.com/matrix-org/dendrite/eduserver/api"
"github.com/matrix-org/dendrite/internal"
"github.com/matrix-org/dendrite/internal/test" "github.com/matrix-org/dendrite/internal/test"
"github.com/matrix-org/dendrite/roomserver/api" "github.com/matrix-org/dendrite/roomserver/api"
"github.com/matrix-org/gomatrixserverlib" "github.com/matrix-org/gomatrixserverlib"
@ -370,6 +371,7 @@ func mustCreateTransaction(rsAPI api.RoomserverInternalAPI, fedClient txnFederat
federation: fedClient, federation: fedClient,
haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent), haveEvents: make(map[string]*gomatrixserverlib.HeaderedEvent),
newEvents: make(map[string]bool), newEvents: make(map[string]bool),
roomsMu: internal.NewMutexByRoom(),
} }
t.PDUs = pdus t.PDUs = pdus
t.Origin = testOrigin t.Origin = testOrigin

38
internal/mutex.go Normal file
View File

@ -0,0 +1,38 @@
package internal
import "sync"
type MutexByRoom struct {
mu *sync.Mutex // protects the map
roomToMu map[string]*sync.Mutex
}
func NewMutexByRoom() *MutexByRoom {
return &MutexByRoom{
mu: &sync.Mutex{},
roomToMu: make(map[string]*sync.Mutex),
}
}
func (m *MutexByRoom) Lock(roomID string) {
m.mu.Lock()
roomMu := m.roomToMu[roomID]
if roomMu == nil {
roomMu = &sync.Mutex{}
}
m.roomToMu[roomID] = roomMu
m.mu.Unlock()
// don't lock inside m.mu else we can deadlock
roomMu.Lock()
}
func (m *MutexByRoom) Unlock(roomID string) {
m.mu.Lock()
roomMu := m.roomToMu[roomID]
if roomMu == nil {
panic("MutexByRoom: Unlock before Lock")
}
m.mu.Unlock()
roomMu.Unlock()
}