[MAJOR MIGRATION] Initial structure for a concept of "datastores"

**Caution**: This is a major migration that changes how files are tracked within the database. Although tested to ensure things won't go wrong, there is the possibility of something going haywire. It is recommended to run the following database commands before upgrading:

CREATE TABLE media_backup AS TABLE media;
CREATE TABLE thumbnails_backup AS TABLE thumbnails;

In the event you need to roll back, run the following:
DROP TABLE media;
DROP TABLE thumbnails;
CREATE TABLE media AS TABLE media_backup;
CREATE TABLE thumbnails AS TABLE thumbnails_backup;
DELETE FROM gomigrate WHERE migration_id = 7;

This will cause any files uploaded between upgrading and rolling back to be lost.

NOTE: The migration will take a while and runs on the first access of a file. If you have a lot of files, this could take a while.

-----

Fixes #106
Step towards #47

migrations/7_add_datastore_down.sql

@@ -0,0 +1,4 @@
+ALTER TABLE media DROP COLUMN datastore_id;
+ALTER TABLE thumbnails DROP COLUMN datastore_id;
+DROP INDEX datastores_index;
+DROP TABLE datastores;

migrations/7_add_datastore_up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS datastores (
+  datastore_id TEXT NOT NULL,
+  ds_type TEXT NOT NULL,
+  uri TEXT NOT NULL
+);
+CREATE UNIQUE INDEX IF NOT EXISTS datastores_index ON datastores (datastore_id);
+
+ALTER TABLE media ADD COLUMN datastore_id TEXT NOT NULL DEFAULT '';
+ALTER TABLE thumbnails ADD COLUMN datastore_id TEXT NOT NULL DEFAULT '';
+

src/github.com/turt2live/matrix-media-repo/controllers/download_controller/download_controller.go

@@ -48,7 +48,11 @@ func GetMedia(origin string, mediaId string, downloadRemote bool, ctx context.Co
 	}
 
 	log.Info("Reading media from disk")
-	stream, err := os.Open(media.Location)
+	filePath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+	if err != nil {
+		return nil, err
+	}
+	stream, err := os.Open(filePath)
 	if err != nil {
 		return nil, err
 	}

src/github.com/turt2live/matrix-media-repo/controllers/maintenance_controller/maintainance_controller.go

@@ -40,7 +40,12 @@ func PurgeRemoteMediaBefore(beforeTs int64, ctx context.Context, log *logrus.Ent
 		}
 
 		// Delete the file first
-		err = os.Remove(media.Location)
+		filePath, err := storage.ResolveMediaLocation(context.TODO(), &logrus.Entry{}, media.DatastoreId, media.Location)
+		if err != nil {
+			log.Error("Error resolving datastore path for media " + media.Origin + "/" + media.MediaId + " because: " + err.Error())
+			continue
+		}
+		err = os.Remove(filePath)
 		if err != nil {
 			log.Warn("Cannot remove media " + media.Origin + "/" + media.MediaId + " because: " + err.Error())
 		} else {

src/github.com/turt2live/matrix-media-repo/controllers/preview_controller/preview_resource_handler.go

@@ -98,15 +98,20 @@ func urlPreviewWorkFn(request *resource_handler.WorkRequest) interface{} {
 		if err != nil {
 			log.Warn("Non-fatal error storing preview thumbnail: " + err.Error())
 		} else {
-			img, err := imaging.Open(media.Location)
+			filePath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
 			if err != nil {
-				log.Warn("Non-fatal error getting thumbnail dimensions: " + err.Error())
+				log.Warn("Non-fatal error resolving datastore path: " + err.Error())
 			} else {
-				result.ImageMxc = media.MxcUri()
-				result.ImageType = media.ContentType
-				result.ImageSize = media.SizeBytes
-				result.ImageWidth = img.Bounds().Max.X
-				result.ImageHeight = img.Bounds().Max.Y
+				img, err := imaging.Open(filePath)
+				if err != nil {
+					log.Warn("Non-fatal error getting thumbnail dimensions: " + err.Error())
+				} else {
+					result.ImageMxc = media.MxcUri()
+					result.ImageType = media.ContentType
+					result.ImageSize = media.SizeBytes
+					result.ImageWidth = img.Bounds().Max.X
+					result.ImageHeight = img.Bounds().Max.Y
+				}
 			}
 		}
 	}

src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_controller.go

@@ -158,7 +158,11 @@ func GetThumbnail(origin string, mediaId string, desiredWidth int, desiredHeight
 	}
 
 	log.Info("Reading thumbnail from disk")
-	stream, err := os.Open(thumbnail.Location)
+	filePath, err := storage.ResolveMediaLocation(ctx, log, thumbnail.DatastoreId, thumbnail.Location)
+	if err != nil {
+		return nil, err
+	}
+	stream, err := os.Open(filePath)
 	if err != nil {
 		return nil, err
 	}

src/github.com/turt2live/matrix-media-repo/controllers/thumbnail_controller/thumbnail_resource_handler.go

@@ -21,6 +21,7 @@ import (
 	"github.com/turt2live/matrix-media-repo/types"
 	"github.com/turt2live/matrix-media-repo/util"
 	"github.com/turt2live/matrix-media-repo/util/resource_handler"
+	"github.com/turt2live/matrix-media-repo/util/util_exif"
 )
 
 type thumbnailResourceHandler struct {
@@ -42,6 +43,7 @@ type thumbnailResponse struct {
 
 type GeneratedThumbnail struct {
 	ContentType  string
+	DatastoreId  string
 	DiskLocation string
 	SizeBytes    int64
 	Animated     bool
@@ -94,6 +96,7 @@ func thumbnailWorkFn(request *resource_handler.WorkRequest) interface{} {
 		Animated:    generated.Animated,
 		CreationTs:  util.NowMillis(),
 		ContentType: generated.ContentType,
+		DatastoreId: generated.DatastoreId,
 		Location:    generated.DiskLocation,
 		SizeBytes:   generated.SizeBytes,
 		Sha256Hash:  generated.Sha256Hash,
@@ -130,9 +133,14 @@ func GenerateThumbnail(media *types.Media, width int, height int, method string,
 	var err error
 
 	if media.ContentType == "image/svg+xml" {
-		src, err = svgToImage(media)
+		src, err = svgToImage(media, ctx, log)
 	} else {
-		src, err = imaging.Open(media.Location)
+		mediaPath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+		if err != nil {
+			log.Error("Error resolving datastore path: ", err)
+			return nil, err
+		}
+		src, err = imaging.Open(mediaPath)
 	}
 
 	if err != nil {
@@ -162,6 +170,7 @@ func GenerateThumbnail(media *types.Media, width int, height int, method string,
 		} else {
 			// Image is too small - don't upscale
 			thumb.ContentType = media.ContentType
+			thumb.DatastoreId = media.DatastoreId
 			thumb.DiskLocation = media.Location
 			thumb.SizeBytes = media.SizeBytes
 			thumb.Sha256Hash = media.Sha256Hash
@@ -170,9 +179,9 @@ func GenerateThumbnail(media *types.Media, width int, height int, method string,
 		}
 	}
 
-	var orientation *util.ExifOrientation = nil
+	var orientation *util_exif.ExifOrientation = nil
 	if media.ContentType == "image/jpeg" || media.ContentType == "image/jpg" {
-		orientation, err = util.GetExifOrientation(media)
+		orientation, err = util_exif.GetExifOrientation(media)
 		if err != nil {
 			log.Warn("Non-fatal error getting EXIF orientation: " + err.Error())
 			orientation = nil // just in case
@@ -188,7 +197,12 @@ func GenerateThumbnail(media *types.Media, width int, height int, method string,
 		// Animated GIFs are a bit more special because we need to do it frame by frame.
 		// This is fairly resource intensive. The calling code is responsible for limiting this case.
 
-		inputFile, err := os.Open(media.Location)
+		mediaPath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+		if err != nil {
+			log.Error("Error resolving datastore path: ", err)
+			return nil, err
+		}
+		inputFile, err := os.Open(mediaPath)
 		if err != nil {
 			log.Error("Error generating animated thumbnail: " + err.Error())
 			return nil, err
@@ -252,12 +266,14 @@ func GenerateThumbnail(media *types.Media, width int, height int, method string,
 	}
 
 	// Reset the buffer pointer and store the file
-	location, err := storage.PersistFile(imgData, ctx, log)
+	datastore, relPath, err := storage.PersistFile(imgData, ctx, log)
 	if err != nil {
 		log.Error("Unexpected error saving thumbnail: " + err.Error())
 		return nil, err
 	}
 
+	location := datastore.ResolveFilePath(relPath)
+
 	fileSize, err := util.FileSize(location)
 	if err != nil {
 		log.Error("Unexpected error getting the size of the thumbnail: " + err.Error())
@@ -270,15 +286,16 @@ func GenerateThumbnail(media *types.Media, width int, height int, method string,
 		return nil, err
 	}
 
-	thumb.DiskLocation = location
+	thumb.DiskLocation = relPath
+	thumb.DatastoreId = datastore.DatastoreId
 	thumb.ContentType = contentType
 	thumb.SizeBytes = fileSize
 	thumb.Sha256Hash = hash
 
 	return thumb, nil
 }
 
-func thumbnailFrame(src image.Image, method string, width int, height int, filter imaging.ResampleFilter, orientation *util.ExifOrientation) (image.Image, error) {
+func thumbnailFrame(src image.Image, method string, width int, height int, filter imaging.ResampleFilter, orientation *util_exif.ExifOrientation) (image.Image, error) {
 	var result image.Image
 	if method == "scale" {
 		result = imaging.Fit(src, width, height, filter)
@@ -310,12 +327,17 @@ func thumbnailFrame(src image.Image, method string, width int, height int, filte
 	return result, nil
 }
 
-func svgToImage(media *types.Media) (image.Image, error) {
+func svgToImage(media *types.Media, ctx context.Context, log *logrus.Entry) (image.Image, error) {
 	tempFile := path.Join(os.TempDir(), "media_repo."+media.Origin+"."+media.MediaId+".png")
 	defer os.Remove(tempFile)
 
 	// requires imagemagick
-	err := exec.Command("convert", media.Location, tempFile).Run()
+	mediaPath, err := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+	if err != nil {
+		log.Error("Error resolving datastore path: ", err)
+		return nil, err
+	}
+	err = exec.Command("convert", mediaPath, tempFile).Run()
 	if err != nil {
 		return nil, err
 	}

src/github.com/turt2live/matrix-media-repo/controllers/upload_controller/upload_controller.go

@@ -63,11 +63,13 @@ func trackUploadAsLastAccess(ctx context.Context, log *logrus.Entry, media *type
 }
 
 func StoreDirect(contents io.Reader, contentType string, filename string, userId string, origin string, mediaId string, ctx context.Context, log *logrus.Entry) (*types.Media, error) {
-	fileLocation, err := storage.PersistFile(contents, ctx, log)
+	datastore, location, err := storage.PersistFile(contents, ctx, log)
 	if err != nil {
 		return nil, err
 	}
 
+	fileLocation := datastore.ResolveFilePath(location)
+
 	fileMime, err := util.GetMimeType(fileLocation)
 	if err != nil {
 		log.Error("Error while checking content type of file: ", err.Error())
@@ -159,10 +161,14 @@ func StoreDirect(contents io.Reader, contentType string, filename string, userId
 
 		// If the media's file exists, we'll delete the temp file
 		// If the media's file doesn't exist, we'll move the temp file to where the media expects it to be
-		exists, err := util.FileExists(media.Location)
+		targetPath, err2 := storage.ResolveMediaLocation(ctx, log, media.DatastoreId, media.Location)
+		if err2 != nil {
+			return nil, err2
+		}
+		exists, err := util.FileExists(targetPath)
 		if err != nil || !exists {
 			// We'll assume an error means it doesn't exist
-			os.Rename(fileLocation, media.Location)
+			os.Rename(fileLocation, targetPath)
 		} else {
 			os.Remove(fileLocation)
 		}
@@ -189,7 +195,8 @@ func StoreDirect(contents io.Reader, contentType string, filename string, userId
 		UserId:      userId,
 		Sha256Hash:  hash,
 		SizeBytes:   fileSize,
-		Location:    fileLocation,
+		DatastoreId: datastore.DatastoreId,
+		Location:    location,
 		CreationTs:  util.NowMillis(),
 	}
 

src/github.com/turt2live/matrix-media-repo/internal_cache/media_cache.go

@@ -3,6 +3,7 @@ package internal_cache
 import (
 	"bytes"
 	"container/list"
+	"context"
 	"fmt"
 	"io/ioutil"
 	"sync"
@@ -11,6 +12,7 @@ import (
 	"github.com/patrickmn/go-cache"
 	"github.com/sirupsen/logrus"
 	"github.com/turt2live/matrix-media-repo/common/config"
+	"github.com/turt2live/matrix-media-repo/storage"
 	"github.com/turt2live/matrix-media-repo/types"
 	"github.com/turt2live/matrix-media-repo/util"
 	"github.com/turt2live/matrix-media-repo/util/download_tracker"
@@ -92,7 +94,11 @@ func (c *MediaCache) GetMedia(media *types.Media, log *logrus.Entry) (*cachedFil
 	}
 
 	cacheFn := func() (*cachedFile, error) {
-		data, err := ioutil.ReadFile(media.Location)
+		filePath, err := storage.ResolveMediaLocation(context.TODO(), log, media.DatastoreId, media.Location)
+		if err != nil {
+			return nil, err
+		}
+		data, err := ioutil.ReadFile(filePath)
 		if err != nil {
 			return nil, err
 		}
@@ -109,7 +115,11 @@ func (c *MediaCache) GetThumbnail(thumbnail *types.Thumbnail, log *logrus.Entry)
 	}
 
 	cacheFn := func() (*cachedFile, error) {
-		data, err := ioutil.ReadFile(thumbnail.Location)
+		filePath, err := storage.ResolveMediaLocation(context.TODO(), log, thumbnail.DatastoreId, thumbnail.Location)
+		if err != nil {
+			return nil, err
+		}
+		data, err := ioutil.ReadFile(filePath)
 		if err != nil {
 			return nil, err
 		}

src/github.com/turt2live/matrix-media-repo/storage/ds_utils.go

@@ -0,0 +1,59 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+
+	"github.com/sirupsen/logrus"
+	"github.com/turt2live/matrix-media-repo/storage/stores"
+	"github.com/turt2live/matrix-media-repo/types"
+	"github.com/turt2live/matrix-media-repo/util"
+)
+
+func GetOrCreateDatastore(ctx context.Context, log *logrus.Entry, basePath string) (*types.Datastore, error) {
+	logrus.Info("Init DS lookup")
+	mediaService := GetDatabase().GetMediaStore(ctx, log)
+
+	return getOrCreateDatastoreWithMediaService(mediaService, basePath)
+}
+
+func getOrCreateDatastoreWithMediaService(mediaService *stores.MediaStore, basePath string) (*types.Datastore, error) {
+	datastore, err := mediaService.GetDatastoreByUri(basePath)
+	if err != nil && err == sql.ErrNoRows {
+		id, err2 := util.GenerateRandomString(32)
+		if err2 != nil {
+			logrus.Error("Error generating datastore ID for base path ", basePath, ": ", err)
+			return nil, err2
+		}
+		datastore = &types.Datastore{
+			DatastoreId: id,
+			Type:        "file",
+			Uri:         basePath,
+		}
+		err2 = mediaService.InsertDatastore(datastore)
+		if err2 != nil {
+			logrus.Error("Error creating datastore for base path ", basePath, ": ", err)
+			return nil, err2
+		}
+	} else if err != nil {
+		logrus.Error("Error getting datastore for base path ", basePath, ": ", err)
+		return nil, err
+	}
+
+	return datastore, nil
+}
+
+func ResolveMediaLocation(ctx context.Context, log *logrus.Entry, datastoreId string, location string) (string, error) {
+	svc := GetDatabase().GetMediaStore(ctx, log)
+	ds, err := svc.GetDatastore(datastoreId)
+	if err != nil {
+		return "", err
+	}
+
+	if ds.Type != "file" {
+		return "", errors.New("unrecognized datastore type: " + ds.Type)
+	}
+
+	return ds.ResolveFilePath(location), nil
+}