Makes db thread-safe

It was needed for simplicity and various optimization

album_database.go

@@ -3,7 +3,9 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"sort"
+	"sync"
 
 	"github.com/peterbourgon/diskv"
 )
@@ -11,8 +13,14 @@ import (
 type AlbumDatabase struct {
 	dv *diskv.Diskv
 
-	sorted     bool
+	consolidated bool
-	sortedList []AlbumID
+
+	byID           map[AlbumID]*Album
+	byPurchaseDate []AlbumID
+	byCollection   map[string][]AlbumID
+	byAuthor       map[string][]AlbumID
+
+	lock *sync.RWMutex
 }
 
 func OpenAlbumDatabase(basepath string) *AlbumDatabase {
@@ -20,37 +28,79 @@ func OpenAlbumDatabase(basepath string) *AlbumDatabase {
 	return &AlbumDatabase{
 		dv: diskv.New(diskv.Options{
 			BasePath:     basepath,
-			CacheSizeMax: 100 * 1024 * 1024, // 100 MB
+			CacheSizeMax: 50 * 1024 * 1024, // 50 MB
 			Compression:  diskv.NewGzipCompression(),
 		}),
-		sorted:     false,
+		consolidated:   false,
-		sortedList: make([]AlbumID, 0, 10),
+		byID:           make(map[AlbumID]*Album),
+		byPurchaseDate: make([]AlbumID, 0, 10),
+		byCollection:   make(map[string][]AlbumID),
+		byAuthor:       make(map[string][]AlbumID),
+		lock:           &sync.RWMutex{},
 	}
+
 }
 
 func (db *AlbumDatabase) AddOrUpdate(a *Album) error {
+
+	aInDb, err := db.Get(a.ID)
+	dbNeedUpdate := true
+	if err == nil {
+		dbNeedUpdate = (a.PurchaseDate != aInDb.PurchaseDate ||
+			!reflect.DeepEqual(a.Designers, aInDb.Designers) ||
+			!reflect.DeepEqual(a.Scenarists, aInDb.Scenarists) ||
+			!reflect.DeepEqual(a.Colorists, aInDb.Colorists) ||
+			a.Collection != aInDb.Collection)
+	}
+
 	data, err := json.Marshal(a)
 	if err != nil {
-		return err
+		return fmt.Errorf("json encoding: %s", err)
-	}
-	aInDb, err := db.Get(a.ID)
-	dbNewSorted := false
-	if err == nil {
-		dbNewSorted = (a.PurchaseDate != aInDb.PurchaseDate)
 	}
 
+	db.lock.Lock()
 	err = db.dv.Write(AlbumIDString(a.ID), data)
 	if err == nil {
-		db.sorted = dbNewSorted
+		db.consolidated = !dbNeedUpdate
+		db.byID[a.ID] = a
 	}
-
+	db.lock.Unlock()
 	return err
 }
 
+func deleteFromList(list []AlbumID, idToDelete AlbumID) []AlbumID {
+	toDeleteIdx := len(list)
+	for i, aID := range list {
+		if aID == idToDelete {
+			toDeleteIdx = i
+			break
+		}
+	}
+
+	if toDeleteIdx == len(list) {
+		return list
+	}
+	return append(list[:toDeleteIdx], list[toDeleteIdx+1:]...)
+}
+
 func (db *AlbumDatabase) Delete(ID AlbumID) error {
-	if db.dv.Has(AlbumIDString(ID)) == false {
+	inDB, err := db.Get(ID)
+	if err != nil {
 		return fmt.Errorf("Album %d not found", ID)
 	}
+
+	db.lock.Lock()
+	defer db.lock.Unlock()
+	delete(db.byID, ID)
+	db.byPurchaseDate = deleteFromList(db.byPurchaseDate, ID)
+
+	for _, a := range inDB.Authors() {
+		db.byAuthor[a] = deleteFromList(db.byAuthor[a], ID)
+	}
+
+	if len(inDB.Collection) > 0 {
+		db.byCollection[inDB.Collection] = deleteFromList(db.byCollection[inDB.Collection], ID)
+	}
 	return db.dv.Erase(AlbumIDString(ID))
 }
 
@@ -67,11 +117,14 @@ func (db *AlbumDatabase) get(ID string) (*Album, error) {
 }
 
 func (db *AlbumDatabase) Get(ID AlbumID) (*Album, error) {
-	return db.get(AlbumIDString(ID))
+	db.lock.RLock()
-}
+	defer db.lock.RUnlock()
 
-func (db *AlbumDatabase) GetJSON(ID AlbumID) ([]byte, error) {
+	a, ok := db.byID[ID]
-	return db.dv.Read(AlbumIDString(ID))
+	if ok == true {
+		return a, nil
+	}
+	return db.get(AlbumIDString(ID))
 }
 
 type ListOfAlbum []*Album
@@ -88,25 +141,67 @@ func (l ListOfAlbum) Swap(i, j int) {
 	l[i], l[j] = l[j], l[i]
 }
 
-func (db *AlbumDatabase) ByPurchaseDate() ([]AlbumID, error) {
+func maxOfInt(a, b int) int {
-	if db.sorted == false {
+	if a < b {
-		allAlbums := []*Album{}
+		return b
-		for aIDStr := range db.dv.Keys(nil) {
+	}
-			a, err := db.get(aIDStr)
+	return a
-			if err != nil {
+}
-				return nil, err
-			}
-			allAlbums = append(allAlbums, a)
-		}
 
-		sort.Sort(sort.Reverse(ListOfAlbum(allAlbums)))
+func (db *AlbumDatabase) ensureConsolidated() error {
+	db.lock.RLock()
+	if db.consolidated == true {
+		db.lock.RUnlock()
+		return nil
+	}
+	db.lock.RUnlock()
 
-		db.sortedList = make([]AlbumID, 0, len(allAlbums))
+	db.lock.Lock()
-		for _, a := range allAlbums {
+	defer db.lock.Unlock()
-			db.sortedList = append(db.sortedList, a.ID)
+
+	allAlbums := make([]*Album, 0, maxOfInt(len(db.byID), len(db.byPurchaseDate)))
+	db.byPurchaseDate = make([]AlbumID, 0, maxOfInt(len(db.byID), len(db.byPurchaseDate)))
+	db.byID = make(map[AlbumID]*Album)
+	db.byAuthor = make(map[string][]AlbumID)
+	db.byCollection = make(map[string][]AlbumID)
+	for aIDStr := range db.dv.Keys(nil) {
+		a, err := db.get(aIDStr)
+		if err != nil {
+			return err
 		}
-		db.sorted = true
+		allAlbums = append(allAlbums, a)
+		db.byID[a.ID] = a
 	}
 
-	return db.sortedList, nil
+	sort.Sort(sort.Reverse(ListOfAlbum(allAlbums)))
+
+	for _, album := range allAlbums {
+		db.byPurchaseDate = append(db.byPurchaseDate, album.ID)
+	}
+
+	for aID, album := range db.byID {
+		if len(album.Collection) > 0 {
+			db.byCollection[album.Collection] = append(db.byCollection[album.Collection], aID)
+		}
+
+		for _, author := range album.Authors() {
+			db.byAuthor[author] = append(db.byAuthor[author], aID)
+		}
+	}
+
+	db.consolidated = true
+	return nil
+}
+
+func (db *AlbumDatabase) ByPurchaseDate() ([]AlbumID, error) {
+	err := db.ensureConsolidated()
+	if err != nil {
+		return nil, err
+	}
+
+	db.lock.RLock()
+	defer db.lock.RUnlock()
+	res := make([]AlbumID, len(db.byPurchaseDate))
+	copy(res, db.byPurchaseDate)
+	return res, nil
 }

album_database_test.go

@@ -17,7 +17,9 @@ var _ = Suite(&AlbumDatabaseSuite{})
 func (s *AlbumDatabaseSuite) SetUpSuite(c *C) {
 	s.db = OpenAlbumDatabase(filepath.Join(c.MkDir(), "satdb.bar.satellite/db"))
 	for _, a := range albumsDataTest {
-		c.Assert(s.db.AddOrUpdate(&a), IsNil)
+		//we need to copy the value to avoid aliasing
+		aCopied := a
+		c.Assert(s.db.AddOrUpdate(&aCopied), IsNil)
 	}
 }
 
@@ -31,7 +33,6 @@ func (s *AlbumDatabaseSuite) TestCanDelete(c *C) {
 }
 
 func (s *AlbumDatabaseSuite) TestCanGet(c *C) {
-
 	for _, a := range albumsDataTest {
 		fromDb, err := s.db.Get(a.ID)
 		if c.Check(err, IsNil) == true {
@@ -41,7 +42,6 @@ func (s *AlbumDatabaseSuite) TestCanGet(c *C) {
 }
 
 func (s *AlbumDatabaseSuite) TestCanSort(c *C) {
-	// here
 	start := time.Now()
 	data := []AlbumID{160366, 58595, 15875, 9935, 84448, 46005, 19762, 164, 52100, 8179, 44989, 32043, 22737, 754}
 	sorted, err := s.db.ByPurchaseDate()