From ce3090394b75369878030c7f0aa964f310567d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Viktor=20Strate=20Kl=C3=B8vedal?= Date: Sun, 7 Feb 2021 22:17:33 +0100 Subject: [PATCH] Add dataloaders to combine database queries (#190) See also https://gqlgen.com/reference/dataloaders/ --- api/go.mod | 6 +- api/go.sum | 6 + api/graphql/auth/auth.go | 7 +- api/graphql/dataloader/gen_mediaurlloader.go | 224 ++++++++++++++++++ .../dataloader/gen_userfavoritesloader.go | 221 +++++++++++++++++ api/graphql/dataloader/gen_userloader.go | 224 ++++++++++++++++++ api/graphql/dataloader/loaders.go | 42 ++++ api/graphql/dataloader/mediaURLLoader.go | 72 ++++++ api/graphql/dataloader/userFavoriteLoader.go | 59 +++++ api/graphql/dataloader/userLoader.go | 65 +++++ api/graphql/models/user.go | 17 -- api/graphql/resolvers/media.go | 66 +----- api/server.go | 2 + 13 files changed, 931 insertions(+), 80 deletions(-) create mode 100644 api/graphql/dataloader/gen_mediaurlloader.go create mode 100644 api/graphql/dataloader/gen_userfavoritesloader.go create mode 100644 api/graphql/dataloader/gen_userloader.go create mode 100644 api/graphql/dataloader/loaders.go create mode 100644 api/graphql/dataloader/mediaURLLoader.go create mode 100644 api/graphql/dataloader/userFavoriteLoader.go create mode 100644 api/graphql/dataloader/userLoader.go diff --git a/api/go.mod b/api/go.mod index 49ae1ca4..287c57fd 100644 --- a/api/go.mod +++ b/api/go.mod @@ -9,7 +9,7 @@ require ( github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 github.com/gorilla/websocket v1.4.2 - github.com/h2non/filetype v1.1.0 + github.com/h2non/filetype v1.1.1 github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/joho/godotenv v1.3.0 github.com/mitchellh/mapstructure v1.4.1 // indirect @@ -21,8 +21,8 @@ require ( golang.org/x/image v0.0.0-20201208152932-35266b937fa6 gopkg.in/vansante/go-ffprobe.v2 v2.0.2 gopkg.in/yaml.v2 v2.4.0 // indirect - gorm.io/driver/mysql v1.0.3 - gorm.io/driver/postgres v1.0.7 + gorm.io/driver/mysql v1.0.4 + gorm.io/driver/postgres v1.0.8 gorm.io/driver/sqlite v1.1.4 gorm.io/gorm v1.20.12 ) diff --git a/api/go.sum b/api/go.sum index 2a865013..82d0d0ce 100644 --- a/api/go.sum +++ b/api/go.sum @@ -44,6 +44,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/h2non/filetype v1.1.0 h1:Or/gjocJrJRNK/Cri/TDEKFjAR+cfG6eK65NGYB6gBA= github.com/h2non/filetype v1.1.0/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/filetype v1.1.1 h1:xvOwnXKAckvtLWsN398qS9QhlxlnVXBjXBydK2/UFB4= +github.com/h2non/filetype v1.1.1/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= @@ -259,8 +261,12 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gorm.io/driver/mysql v1.0.3 h1:+JKBYPfn1tygR1/of/Fh2T8iwuVwzt+PEJmKaXzMQXg= gorm.io/driver/mysql v1.0.3/go.mod h1:twGxftLBlFgNVNakL7F+P/x9oYqoymG3YYT8cAfI9oI= +gorm.io/driver/mysql v1.0.4 h1:TATTzt+kR+IV0+h3iUB3dHUe8omCvQ0rOkmfCsUBohk= +gorm.io/driver/mysql v1.0.4/go.mod h1:MEgp8tk2n60cSBCq5iTcPDw3ns8Gs+zOva9EUhkknTs= gorm.io/driver/postgres v1.0.7 h1:uCVjh1w7DSZ20Duo10JadA+1a0OZpgJk/o/z8pFpNQs= gorm.io/driver/postgres v1.0.7/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= +gorm.io/driver/postgres v1.0.8 h1:PAgM+PaHOSAeroTjHkCHCBIHHoBIf9RgPWGo8dF2DA8= +gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= gorm.io/driver/sqlite v1.1.4 h1:PDzwYE+sI6De2+mxAneV9Xs11+ZyKV6oxD3wDGkaNvM= gorm.io/driver/sqlite v1.1.4/go.mod h1:mJCeTFr7+crvS+TRnWc5Z3UvwxUN1BGBLMrf5LA9DYw= gorm.io/gorm v1.20.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= diff --git a/api/graphql/auth/auth.go b/api/graphql/auth/auth.go index 80a51302..205081e7 100644 --- a/api/graphql/auth/auth.go +++ b/api/graphql/auth/auth.go @@ -8,6 +8,7 @@ import ( "regexp" "github.com/99designs/gqlgen/handler" + "github.com/photoview/photoview/api/graphql/dataloader" "github.com/photoview/photoview/api/graphql/models" "gorm.io/gorm" ) @@ -28,7 +29,8 @@ func Middleware(db *gorm.DB) func(http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if tokenCookie, err := r.Cookie("auth-token"); err == nil { - user, err := models.VerifyTokenAndGetUser(db, tokenCookie.Value) + user, err := dataloader.For(r.Context()).UserFromAccessToken.Load(tokenCookie.Value) + // user, err := models.VerifyTokenAndGetUser(db, tokenCookie.Value) if err != nil { log.Printf("Invalid token: %s\n", err) http.Error(w, "invalid authorization token", http.StatusForbidden) @@ -80,7 +82,8 @@ func AuthWebsocketInit(db *gorm.DB) func(context.Context, handler.InitPayload) ( return nil, err } - user, err := models.VerifyTokenAndGetUser(db, *token) + user, err := dataloader.For(ctx).UserFromAccessToken.Load(*token) + // user, err := models.VerifyTokenAndGetUser(db, *token) if err != nil { log.Printf("Invalid token in websocket: %s\n", err) return nil, errors.New("invalid authorization token") diff --git a/api/graphql/dataloader/gen_mediaurlloader.go b/api/graphql/dataloader/gen_mediaurlloader.go new file mode 100644 index 00000000..99c9d92e --- /dev/null +++ b/api/graphql/dataloader/gen_mediaurlloader.go @@ -0,0 +1,224 @@ +// Code generated by github.com/vektah/dataloaden, DO NOT EDIT. + +package dataloader + +import ( + "sync" + "time" + + "github.com/photoview/photoview/api/graphql/models" +) + +// MediaURLLoaderConfig captures the config to create a new MediaURLLoader +type MediaURLLoaderConfig struct { + // Fetch is a method that provides the data for the loader + Fetch func(keys []int) ([]*models.MediaURL, []error) + + // Wait is how long wait before sending a batch + Wait time.Duration + + // MaxBatch will limit the maximum number of keys to send in one batch, 0 = not limit + MaxBatch int +} + +// NewMediaURLLoader creates a new MediaURLLoader given a fetch, wait, and maxBatch +func NewMediaURLLoader(config MediaURLLoaderConfig) *MediaURLLoader { + return &MediaURLLoader{ + fetch: config.Fetch, + wait: config.Wait, + maxBatch: config.MaxBatch, + } +} + +// MediaURLLoader batches and caches requests +type MediaURLLoader struct { + // this method provides the data for the loader + fetch func(keys []int) ([]*models.MediaURL, []error) + + // how long to done before sending a batch + wait time.Duration + + // this will limit the maximum number of keys to send in one batch, 0 = no limit + maxBatch int + + // INTERNAL + + // lazily created cache + cache map[int]*models.MediaURL + + // the current batch. keys will continue to be collected until timeout is hit, + // then everything will be sent to the fetch method and out to the listeners + batch *mediaURLLoaderBatch + + // mutex to prevent races + mu sync.Mutex +} + +type mediaURLLoaderBatch struct { + keys []int + data []*models.MediaURL + error []error + closing bool + done chan struct{} +} + +// Load a MediaURL by key, batching and caching will be applied automatically +func (l *MediaURLLoader) Load(key int) (*models.MediaURL, error) { + return l.LoadThunk(key)() +} + +// LoadThunk returns a function that when called will block waiting for a MediaURL. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *MediaURLLoader) LoadThunk(key int) func() (*models.MediaURL, error) { + l.mu.Lock() + if it, ok := l.cache[key]; ok { + l.mu.Unlock() + return func() (*models.MediaURL, error) { + return it, nil + } + } + if l.batch == nil { + l.batch = &mediaURLLoaderBatch{done: make(chan struct{})} + } + batch := l.batch + pos := batch.keyIndex(l, key) + l.mu.Unlock() + + return func() (*models.MediaURL, error) { + <-batch.done + + var data *models.MediaURL + if pos < len(batch.data) { + data = batch.data[pos] + } + + var err error + // its convenient to be able to return a single error for everything + if len(batch.error) == 1 { + err = batch.error[0] + } else if batch.error != nil { + err = batch.error[pos] + } + + if err == nil { + l.mu.Lock() + l.unsafeSet(key, data) + l.mu.Unlock() + } + + return data, err + } +} + +// LoadAll fetches many keys at once. It will be broken into appropriate sized +// sub batches depending on how the loader is configured +func (l *MediaURLLoader) LoadAll(keys []int) ([]*models.MediaURL, []error) { + results := make([]func() (*models.MediaURL, error), len(keys)) + + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + + mediaURLs := make([]*models.MediaURL, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + mediaURLs[i], errors[i] = thunk() + } + return mediaURLs, errors +} + +// LoadAllThunk returns a function that when called will block waiting for a MediaURLs. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *MediaURLLoader) LoadAllThunk(keys []int) func() ([]*models.MediaURL, []error) { + results := make([]func() (*models.MediaURL, error), len(keys)) + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + return func() ([]*models.MediaURL, []error) { + mediaURLs := make([]*models.MediaURL, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + mediaURLs[i], errors[i] = thunk() + } + return mediaURLs, errors + } +} + +// Prime the cache with the provided key and value. If the key already exists, no change is made +// and false is returned. +// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).) +func (l *MediaURLLoader) Prime(key int, value *models.MediaURL) bool { + l.mu.Lock() + var found bool + if _, found = l.cache[key]; !found { + // make a copy when writing to the cache, its easy to pass a pointer in from a loop var + // and end up with the whole cache pointing to the same value. + cpy := *value + l.unsafeSet(key, &cpy) + } + l.mu.Unlock() + return !found +} + +// Clear the value at key from the cache, if it exists +func (l *MediaURLLoader) Clear(key int) { + l.mu.Lock() + delete(l.cache, key) + l.mu.Unlock() +} + +func (l *MediaURLLoader) unsafeSet(key int, value *models.MediaURL) { + if l.cache == nil { + l.cache = map[int]*models.MediaURL{} + } + l.cache[key] = value +} + +// keyIndex will return the location of the key in the batch, if its not found +// it will add the key to the batch +func (b *mediaURLLoaderBatch) keyIndex(l *MediaURLLoader, key int) int { + for i, existingKey := range b.keys { + if key == existingKey { + return i + } + } + + pos := len(b.keys) + b.keys = append(b.keys, key) + if pos == 0 { + go b.startTimer(l) + } + + if l.maxBatch != 0 && pos >= l.maxBatch-1 { + if !b.closing { + b.closing = true + l.batch = nil + go b.end(l) + } + } + + return pos +} + +func (b *mediaURLLoaderBatch) startTimer(l *MediaURLLoader) { + time.Sleep(l.wait) + l.mu.Lock() + + // we must have hit a batch limit and are already finalizing this batch + if b.closing { + l.mu.Unlock() + return + } + + l.batch = nil + l.mu.Unlock() + + b.end(l) +} + +func (b *mediaURLLoaderBatch) end(l *MediaURLLoader) { + b.data, b.error = l.fetch(b.keys) + close(b.done) +} diff --git a/api/graphql/dataloader/gen_userfavoritesloader.go b/api/graphql/dataloader/gen_userfavoritesloader.go new file mode 100644 index 00000000..6ab333c5 --- /dev/null +++ b/api/graphql/dataloader/gen_userfavoritesloader.go @@ -0,0 +1,221 @@ +// Code generated by github.com/vektah/dataloaden, DO NOT EDIT. + +package dataloader + +import ( + "sync" + "time" + + "github.com/photoview/photoview/api/graphql/models" +) + +// UserFavoritesLoaderConfig captures the config to create a new UserFavoritesLoader +type UserFavoritesLoaderConfig struct { + // Fetch is a method that provides the data for the loader + Fetch func(keys []*models.UserMediaData) ([]bool, []error) + + // Wait is how long wait before sending a batch + Wait time.Duration + + // MaxBatch will limit the maximum number of keys to send in one batch, 0 = not limit + MaxBatch int +} + +// NewUserFavoritesLoader creates a new UserFavoritesLoader given a fetch, wait, and maxBatch +func NewUserFavoritesLoader(config UserFavoritesLoaderConfig) *UserFavoritesLoader { + return &UserFavoritesLoader{ + fetch: config.Fetch, + wait: config.Wait, + maxBatch: config.MaxBatch, + } +} + +// UserFavoritesLoader batches and caches requests +type UserFavoritesLoader struct { + // this method provides the data for the loader + fetch func(keys []*models.UserMediaData) ([]bool, []error) + + // how long to done before sending a batch + wait time.Duration + + // this will limit the maximum number of keys to send in one batch, 0 = no limit + maxBatch int + + // INTERNAL + + // lazily created cache + cache map[*models.UserMediaData]bool + + // the current batch. keys will continue to be collected until timeout is hit, + // then everything will be sent to the fetch method and out to the listeners + batch *userFavoritesLoaderBatch + + // mutex to prevent races + mu sync.Mutex +} + +type userFavoritesLoaderBatch struct { + keys []*models.UserMediaData + data []bool + error []error + closing bool + done chan struct{} +} + +// Load a bool by key, batching and caching will be applied automatically +func (l *UserFavoritesLoader) Load(key *models.UserMediaData) (bool, error) { + return l.LoadThunk(key)() +} + +// LoadThunk returns a function that when called will block waiting for a bool. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *UserFavoritesLoader) LoadThunk(key *models.UserMediaData) func() (bool, error) { + l.mu.Lock() + if it, ok := l.cache[key]; ok { + l.mu.Unlock() + return func() (bool, error) { + return it, nil + } + } + if l.batch == nil { + l.batch = &userFavoritesLoaderBatch{done: make(chan struct{})} + } + batch := l.batch + pos := batch.keyIndex(l, key) + l.mu.Unlock() + + return func() (bool, error) { + <-batch.done + + var data bool + if pos < len(batch.data) { + data = batch.data[pos] + } + + var err error + // its convenient to be able to return a single error for everything + if len(batch.error) == 1 { + err = batch.error[0] + } else if batch.error != nil { + err = batch.error[pos] + } + + if err == nil { + l.mu.Lock() + l.unsafeSet(key, data) + l.mu.Unlock() + } + + return data, err + } +} + +// LoadAll fetches many keys at once. It will be broken into appropriate sized +// sub batches depending on how the loader is configured +func (l *UserFavoritesLoader) LoadAll(keys []*models.UserMediaData) ([]bool, []error) { + results := make([]func() (bool, error), len(keys)) + + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + + bools := make([]bool, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + bools[i], errors[i] = thunk() + } + return bools, errors +} + +// LoadAllThunk returns a function that when called will block waiting for a bools. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *UserFavoritesLoader) LoadAllThunk(keys []*models.UserMediaData) func() ([]bool, []error) { + results := make([]func() (bool, error), len(keys)) + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + return func() ([]bool, []error) { + bools := make([]bool, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + bools[i], errors[i] = thunk() + } + return bools, errors + } +} + +// Prime the cache with the provided key and value. If the key already exists, no change is made +// and false is returned. +// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).) +func (l *UserFavoritesLoader) Prime(key *models.UserMediaData, value bool) bool { + l.mu.Lock() + var found bool + if _, found = l.cache[key]; !found { + l.unsafeSet(key, value) + } + l.mu.Unlock() + return !found +} + +// Clear the value at key from the cache, if it exists +func (l *UserFavoritesLoader) Clear(key *models.UserMediaData) { + l.mu.Lock() + delete(l.cache, key) + l.mu.Unlock() +} + +func (l *UserFavoritesLoader) unsafeSet(key *models.UserMediaData, value bool) { + if l.cache == nil { + l.cache = map[*models.UserMediaData]bool{} + } + l.cache[key] = value +} + +// keyIndex will return the location of the key in the batch, if its not found +// it will add the key to the batch +func (b *userFavoritesLoaderBatch) keyIndex(l *UserFavoritesLoader, key *models.UserMediaData) int { + for i, existingKey := range b.keys { + if key == existingKey { + return i + } + } + + pos := len(b.keys) + b.keys = append(b.keys, key) + if pos == 0 { + go b.startTimer(l) + } + + if l.maxBatch != 0 && pos >= l.maxBatch-1 { + if !b.closing { + b.closing = true + l.batch = nil + go b.end(l) + } + } + + return pos +} + +func (b *userFavoritesLoaderBatch) startTimer(l *UserFavoritesLoader) { + time.Sleep(l.wait) + l.mu.Lock() + + // we must have hit a batch limit and are already finalizing this batch + if b.closing { + l.mu.Unlock() + return + } + + l.batch = nil + l.mu.Unlock() + + b.end(l) +} + +func (b *userFavoritesLoaderBatch) end(l *UserFavoritesLoader) { + b.data, b.error = l.fetch(b.keys) + close(b.done) +} diff --git a/api/graphql/dataloader/gen_userloader.go b/api/graphql/dataloader/gen_userloader.go new file mode 100644 index 00000000..ec4d53fe --- /dev/null +++ b/api/graphql/dataloader/gen_userloader.go @@ -0,0 +1,224 @@ +// Code generated by github.com/vektah/dataloaden, DO NOT EDIT. + +package dataloader + +import ( + "sync" + "time" + + "github.com/photoview/photoview/api/graphql/models" +) + +// UserLoaderConfig captures the config to create a new UserLoader +type UserLoaderConfig struct { + // Fetch is a method that provides the data for the loader + Fetch func(keys []string) ([]*models.User, []error) + + // Wait is how long wait before sending a batch + Wait time.Duration + + // MaxBatch will limit the maximum number of keys to send in one batch, 0 = not limit + MaxBatch int +} + +// NewUserLoader creates a new UserLoader given a fetch, wait, and maxBatch +func NewUserLoader(config UserLoaderConfig) *UserLoader { + return &UserLoader{ + fetch: config.Fetch, + wait: config.Wait, + maxBatch: config.MaxBatch, + } +} + +// UserLoader batches and caches requests +type UserLoader struct { + // this method provides the data for the loader + fetch func(keys []string) ([]*models.User, []error) + + // how long to done before sending a batch + wait time.Duration + + // this will limit the maximum number of keys to send in one batch, 0 = no limit + maxBatch int + + // INTERNAL + + // lazily created cache + cache map[string]*models.User + + // the current batch. keys will continue to be collected until timeout is hit, + // then everything will be sent to the fetch method and out to the listeners + batch *userLoaderBatch + + // mutex to prevent races + mu sync.Mutex +} + +type userLoaderBatch struct { + keys []string + data []*models.User + error []error + closing bool + done chan struct{} +} + +// Load a User by key, batching and caching will be applied automatically +func (l *UserLoader) Load(key string) (*models.User, error) { + return l.LoadThunk(key)() +} + +// LoadThunk returns a function that when called will block waiting for a User. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *UserLoader) LoadThunk(key string) func() (*models.User, error) { + l.mu.Lock() + if it, ok := l.cache[key]; ok { + l.mu.Unlock() + return func() (*models.User, error) { + return it, nil + } + } + if l.batch == nil { + l.batch = &userLoaderBatch{done: make(chan struct{})} + } + batch := l.batch + pos := batch.keyIndex(l, key) + l.mu.Unlock() + + return func() (*models.User, error) { + <-batch.done + + var data *models.User + if pos < len(batch.data) { + data = batch.data[pos] + } + + var err error + // its convenient to be able to return a single error for everything + if len(batch.error) == 1 { + err = batch.error[0] + } else if batch.error != nil { + err = batch.error[pos] + } + + if err == nil { + l.mu.Lock() + l.unsafeSet(key, data) + l.mu.Unlock() + } + + return data, err + } +} + +// LoadAll fetches many keys at once. It will be broken into appropriate sized +// sub batches depending on how the loader is configured +func (l *UserLoader) LoadAll(keys []string) ([]*models.User, []error) { + results := make([]func() (*models.User, error), len(keys)) + + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + + users := make([]*models.User, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + users[i], errors[i] = thunk() + } + return users, errors +} + +// LoadAllThunk returns a function that when called will block waiting for a Users. +// This method should be used if you want one goroutine to make requests to many +// different data loaders without blocking until the thunk is called. +func (l *UserLoader) LoadAllThunk(keys []string) func() ([]*models.User, []error) { + results := make([]func() (*models.User, error), len(keys)) + for i, key := range keys { + results[i] = l.LoadThunk(key) + } + return func() ([]*models.User, []error) { + users := make([]*models.User, len(keys)) + errors := make([]error, len(keys)) + for i, thunk := range results { + users[i], errors[i] = thunk() + } + return users, errors + } +} + +// Prime the cache with the provided key and value. If the key already exists, no change is made +// and false is returned. +// (To forcefully prime the cache, clear the key first with loader.clear(key).prime(key, value).) +func (l *UserLoader) Prime(key string, value *models.User) bool { + l.mu.Lock() + var found bool + if _, found = l.cache[key]; !found { + // make a copy when writing to the cache, its easy to pass a pointer in from a loop var + // and end up with the whole cache pointing to the same value. + cpy := *value + l.unsafeSet(key, &cpy) + } + l.mu.Unlock() + return !found +} + +// Clear the value at key from the cache, if it exists +func (l *UserLoader) Clear(key string) { + l.mu.Lock() + delete(l.cache, key) + l.mu.Unlock() +} + +func (l *UserLoader) unsafeSet(key string, value *models.User) { + if l.cache == nil { + l.cache = map[string]*models.User{} + } + l.cache[key] = value +} + +// keyIndex will return the location of the key in the batch, if its not found +// it will add the key to the batch +func (b *userLoaderBatch) keyIndex(l *UserLoader, key string) int { + for i, existingKey := range b.keys { + if key == existingKey { + return i + } + } + + pos := len(b.keys) + b.keys = append(b.keys, key) + if pos == 0 { + go b.startTimer(l) + } + + if l.maxBatch != 0 && pos >= l.maxBatch-1 { + if !b.closing { + b.closing = true + l.batch = nil + go b.end(l) + } + } + + return pos +} + +func (b *userLoaderBatch) startTimer(l *UserLoader) { + time.Sleep(l.wait) + l.mu.Lock() + + // we must have hit a batch limit and are already finalizing this batch + if b.closing { + l.mu.Unlock() + return + } + + l.batch = nil + l.mu.Unlock() + + b.end(l) +} + +func (b *userLoaderBatch) end(l *UserLoader) { + b.data, b.error = l.fetch(b.keys) + close(b.done) +} diff --git a/api/graphql/dataloader/loaders.go b/api/graphql/dataloader/loaders.go new file mode 100644 index 00000000..6904bda1 --- /dev/null +++ b/api/graphql/dataloader/loaders.go @@ -0,0 +1,42 @@ +package dataloader + +import ( + "context" + "net/http" + + "github.com/gorilla/mux" + "gorm.io/gorm" +) + +const loadersKey = "dataloaders" + +type Loaders struct { + MediaThumbnail *MediaURLLoader + MediaHighres *MediaURLLoader + MediaVideoWeb *MediaURLLoader + UserFromAccessToken *UserLoader + UserMediaFavorite *UserFavoritesLoader +} + +func Middleware(db *gorm.DB) mux.MiddlewareFunc { + return mux.MiddlewareFunc(func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + ctx := context.WithValue(r.Context(), loadersKey, &Loaders{ + MediaThumbnail: NewThumbnailMediaURLLoader(db), + MediaHighres: NewHighresMediaURLLoader(db), + MediaVideoWeb: NewVideoWebMediaURLLoader(db), + UserFromAccessToken: NewUserLoaderByToken(db), + UserMediaFavorite: NewUserFavoriteLoader(db), + }) + + r = r.WithContext(ctx) + + next.ServeHTTP(w, r) + }) + }) +} + +func For(ctx context.Context) *Loaders { + return ctx.Value(loadersKey).(*Loaders) +} diff --git a/api/graphql/dataloader/mediaURLLoader.go b/api/graphql/dataloader/mediaURLLoader.go new file mode 100644 index 00000000..78955976 --- /dev/null +++ b/api/graphql/dataloader/mediaURLLoader.go @@ -0,0 +1,72 @@ +package dataloader + +import ( + "time" + + "github.com/photoview/photoview/api/graphql/models" + "github.com/photoview/photoview/api/scanner" + "github.com/pkg/errors" + + "gorm.io/gorm" +) + +func makeMediaURLLoader(db *gorm.DB, filter func(query *gorm.DB) *gorm.DB) func(keys []int) ([]*models.MediaURL, []error) { + return func(mediaIDs []int) ([]*models.MediaURL, []error) { + + var urls []*models.MediaURL + query := db.Where("media_id IN (?)", mediaIDs) + + filter(query) + + if err := query.Find(&urls).Error; err != nil { + return nil, []error{errors.Wrap(err, "media url loader database query")} + } + + resultMap := make(map[int]*models.MediaURL, len(mediaIDs)) + for _, url := range urls { + resultMap[url.MediaID] = url + } + + result := make([]*models.MediaURL, len(mediaIDs)) + for i, mediaID := range mediaIDs { + mediaURL, found := resultMap[mediaID] + if found { + result[i] = mediaURL + } else { + result[i] = nil + } + } + + return result, nil + } +} + +func NewThumbnailMediaURLLoader(db *gorm.DB) *MediaURLLoader { + return &MediaURLLoader{ + maxBatch: 100, + wait: 5 * time.Millisecond, + fetch: makeMediaURLLoader(db, func(query *gorm.DB) *gorm.DB { + return query.Where("purpose = ? OR purpose = ?", models.PhotoThumbnail, models.VideoThumbnail) + }), + } +} + +func NewHighresMediaURLLoader(db *gorm.DB) *MediaURLLoader { + return &MediaURLLoader{ + maxBatch: 100, + wait: 5 * time.Millisecond, + fetch: makeMediaURLLoader(db, func(query *gorm.DB) *gorm.DB { + return query.Where("purpose = ? OR (purpose = ? AND content_type IN ?)", models.PhotoHighRes, models.MediaOriginal, scanner.WebMimetypes) + }), + } +} + +func NewVideoWebMediaURLLoader(db *gorm.DB) *MediaURLLoader { + return &MediaURLLoader{ + maxBatch: 100, + wait: 5 * time.Millisecond, + fetch: makeMediaURLLoader(db, func(query *gorm.DB) *gorm.DB { + return query.Where("purpose = ? OR purpose = ?", models.VideoWeb, models.MediaOriginal) + }), + } +} diff --git a/api/graphql/dataloader/userFavoriteLoader.go b/api/graphql/dataloader/userFavoriteLoader.go new file mode 100644 index 00000000..54e4234d --- /dev/null +++ b/api/graphql/dataloader/userFavoriteLoader.go @@ -0,0 +1,59 @@ +package dataloader + +import ( + "time" + + "github.com/photoview/photoview/api/graphql/models" + "gorm.io/gorm" +) + +func NewUserFavoriteLoader(db *gorm.DB) *UserFavoritesLoader { + return &UserFavoritesLoader{ + maxBatch: 100, + wait: 5 * time.Millisecond, + fetch: func(keys []*models.UserMediaData) ([]bool, []error) { + + userIDMap := make(map[int]struct{}, len(keys)) + mediaIDMap := make(map[int]struct{}, len(keys)) + for _, key := range keys { + userIDMap[key.UserID] = struct{}{} + mediaIDMap[key.MediaID] = struct{}{} + } + + uniqueUserIDs := make([]int, len(userIDMap)) + uniqueMediaIDs := make([]int, len(mediaIDMap)) + + count := 0 + for id := range userIDMap { + uniqueUserIDs[count] = id + count++ + } + + count = 0 + for id := range mediaIDMap { + uniqueMediaIDs[count] = id + count++ + } + + var userMediaFavorites []*models.UserMediaData + err := db.Where("user_id IN (?)", uniqueUserIDs).Where("media_id IN (?)", uniqueMediaIDs).Where("favorite = 1").Find(&userMediaFavorites).Error + if err != nil { + return nil, []error{err} + } + + result := make([]bool, len(keys)) + for i, key := range keys { + favorite := false + for _, fav := range userMediaFavorites { + if fav.UserID == key.UserID && fav.MediaID == key.MediaID { + favorite = true + break + } + } + result[i] = favorite + } + + return result, nil + }, + } +} diff --git a/api/graphql/dataloader/userLoader.go b/api/graphql/dataloader/userLoader.go new file mode 100644 index 00000000..6338ee79 --- /dev/null +++ b/api/graphql/dataloader/userLoader.go @@ -0,0 +1,65 @@ +package dataloader + +import ( + "time" + + "github.com/photoview/photoview/api/graphql/models" + "gorm.io/gorm" +) + +func NewUserLoaderByToken(db *gorm.DB) *UserLoader { + return &UserLoader{ + maxBatch: 100, + wait: 5 * time.Millisecond, + fetch: func(tokens []string) ([]*models.User, []error) { + + var accessTokens []*models.AccessToken + err := db.Where("expire > ?", time.Now()).Where("value IN (?)", tokens).Find(&accessTokens).Error + if err != nil { + return nil, []error{err} + } + + rows, err := db.Table("access_tokens").Select("distinct user_id").Where("expire > ?", time.Now()).Where("value IN (?)", tokens).Rows() + if err != nil { + return nil, []error{err} + } + userIDs := make([]int, 0) + for rows.Next() { + var id int + if err := db.ScanRows(rows, &id); err != nil { + return nil, []error{err} + } + userIDs = append(userIDs, id) + } + rows.Close() + + var users []*models.User + if err := db.Where("id IN (?)", userIDs).Find(&users).Error; err != nil { + return nil, []error{err} + } + + userMap := make(map[int]*models.User, len(users)) + for _, user := range users { + userMap[user.ID] = user + } + + tokenMap := make(map[string]*models.AccessToken, len(tokens)) + for _, token := range accessTokens { + tokenMap[token.Value] = token + } + + result := make([]*models.User, len(tokens)) + for i, token := range tokens { + accessToken, tokenFound := tokenMap[token] + user, userFound := userMap[accessToken.UserID] + if tokenFound && userFound { + result[i] = user + } else { + result[i] = nil + } + } + + return result, nil + }, + } +} diff --git a/api/graphql/models/user.go b/api/graphql/models/user.go index a6b62d21..5b71e97f 100644 --- a/api/graphql/models/user.go +++ b/api/graphql/models/user.go @@ -137,23 +137,6 @@ func (user *User) GenerateAccessToken(db *gorm.DB) (*AccessToken, error) { return &token, nil } -func VerifyTokenAndGetUser(db *gorm.DB, token string) (*User, error) { - - var accessToken AccessToken - result := db.Where("expire > ? AND value = ?", time.Now(), token).First(&accessToken) - if result.Error != nil { - return nil, result.Error - } - - var user User - result = db.First(&user, accessToken.UserID) - if result.Error != nil { - return nil, result.Error - } - - return &user, nil -} - // FillAlbums fill user.Albums with albums from database func (user *User) FillAlbums(db *gorm.DB) error { // Albums already present diff --git a/api/graphql/resolvers/media.go b/api/graphql/resolvers/media.go index 2a6a1915..0f2109f3 100644 --- a/api/graphql/resolvers/media.go +++ b/api/graphql/resolvers/media.go @@ -5,10 +5,9 @@ import ( api "github.com/photoview/photoview/api/graphql" "github.com/photoview/photoview/api/graphql/auth" + "github.com/photoview/photoview/api/graphql/dataloader" "github.com/photoview/photoview/api/graphql/models" - "github.com/photoview/photoview/api/scanner" "github.com/pkg/errors" - "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -145,39 +144,11 @@ func (r *mediaResolver) HighRes(ctx context.Context, media *models.Media) (*mode return nil, nil } - var url models.MediaURL - err := r.Database. - Where("media_id = ?", media.ID). - Where("purpose = ? OR (purpose = ? AND content_type IN ?)", models.PhotoHighRes, models.MediaOriginal, scanner.WebMimetypes). - First(&url).Error - - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil - } else { - return nil, errors.Wrapf(err, "could not query high-res (%s)", media.Path) - } - } - - return &url, nil + return dataloader.For(ctx).MediaHighres.Load(media.ID) } func (r *mediaResolver) Thumbnail(ctx context.Context, media *models.Media) (*models.MediaURL, error) { - var url []*models.MediaURL - err := r.Database. - Where("media_id = ?", media.ID). - Where("purpose = ? OR purpose = ?", models.PhotoThumbnail, models.VideoThumbnail). - Find(&url).Error - - if err != nil { - return nil, errors.Wrapf(err, "could not query thumbnail (%s)", media.Path) - } - - if len(url) == 0 { - return nil, nil - } - - return url[0], nil + return dataloader.For(ctx).MediaThumbnail.Load(media.ID) } func (r *mediaResolver) VideoWeb(ctx context.Context, media *models.Media) (*models.MediaURL, error) { @@ -185,21 +156,7 @@ func (r *mediaResolver) VideoWeb(ctx context.Context, media *models.Media) (*mod return nil, nil } - var url models.MediaURL - err := r.Database. - Where("media_id = ?", media.ID). - Where("purpose = ? OR purpose = ?", models.VideoWeb, models.MediaOriginal). - First(&url).Error - - if err != nil { - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, nil - } else { - return nil, errors.Wrapf(err, "could not query video web-format url (%s)", media.Path) - } - } - - return &url, nil + return dataloader.For(ctx).MediaVideoWeb.Load(media.ID) } func (r *mediaResolver) Exif(ctx context.Context, media *models.Media) (*models.MediaEXIF, error) { @@ -221,17 +178,10 @@ func (r *mediaResolver) Favorite(ctx context.Context, media *models.Media) (bool return false, auth.ErrUnauthorized } - userMediaData := models.UserMediaData{ - UserID: user.ID, - MediaID: media.ID, - Favorite: false, - } - - if err := r.Database.FirstOrInit(&userMediaData).Error; err != nil { - return false, errors.Wrapf(err, "get user media data from database (user: %d, media: %d)", user.ID, media.ID) - } - - return userMediaData.Favorite, nil + return dataloader.For(ctx).UserMediaFavorite.Load(&models.UserMediaData{ + UserID: user.ID, + MediaID: media.ID, + }) } func (r *mutationResolver) FavoriteMedia(ctx context.Context, mediaID int, favorite bool) (*models.Media, error) { diff --git a/api/server.go b/api/server.go index 506d2b86..11699b70 100644 --- a/api/server.go +++ b/api/server.go @@ -12,6 +12,7 @@ import ( "github.com/photoview/photoview/api/database" "github.com/photoview/photoview/api/graphql/auth" + "github.com/photoview/photoview/api/graphql/dataloader" "github.com/photoview/photoview/api/routes" "github.com/photoview/photoview/api/scanner" "github.com/photoview/photoview/api/server" @@ -52,6 +53,7 @@ func main() { rootRouter := mux.NewRouter() + rootRouter.Use(dataloader.Middleware(db)) rootRouter.Use(auth.Middleware(db)) rootRouter.Use(server.LoggingMiddleware) rootRouter.Use(server.CORSMiddleware(devMode))