From 575655b6a4f8b634c243eea46dcf5c4ede83fa4d Mon Sep 17 00:00:00 2001 From: Alexandre Tuleu Date: Mon, 25 Jan 2016 13:14:31 +0100 Subject: [PATCH] Makes db thread-safe It was needed for simplicity and various optimization --- album_database.go | 163 ++++++++++++++++++++++++++++++++--------- album_database_test.go | 6 +- 2 files changed, 132 insertions(+), 37 deletions(-) diff --git a/album_database.go b/album_database.go index 6eafce9..349862a 100644 --- a/album_database.go +++ b/album_database.go @@ -3,7 +3,9 @@ package main import ( "encoding/json" "fmt" + "reflect" "sort" + "sync" "github.com/peterbourgon/diskv" ) @@ -11,8 +13,14 @@ import ( type AlbumDatabase struct { dv *diskv.Diskv - sorted bool - sortedList []AlbumID + consolidated bool + + byID map[AlbumID]*Album + byPurchaseDate []AlbumID + byCollection map[string][]AlbumID + byAuthor map[string][]AlbumID + + lock *sync.RWMutex } func OpenAlbumDatabase(basepath string) *AlbumDatabase { @@ -20,37 +28,79 @@ func OpenAlbumDatabase(basepath string) *AlbumDatabase { return &AlbumDatabase{ dv: diskv.New(diskv.Options{ BasePath: basepath, - CacheSizeMax: 100 * 1024 * 1024, // 100 MB + CacheSizeMax: 50 * 1024 * 1024, // 50 MB Compression: diskv.NewGzipCompression(), }), - sorted: false, - sortedList: make([]AlbumID, 0, 10), + consolidated: false, + byID: make(map[AlbumID]*Album), + byPurchaseDate: make([]AlbumID, 0, 10), + byCollection: make(map[string][]AlbumID), + byAuthor: make(map[string][]AlbumID), + lock: &sync.RWMutex{}, } + } func (db *AlbumDatabase) AddOrUpdate(a *Album) error { + + aInDb, err := db.Get(a.ID) + dbNeedUpdate := true + if err == nil { + dbNeedUpdate = (a.PurchaseDate != aInDb.PurchaseDate || + !reflect.DeepEqual(a.Designers, aInDb.Designers) || + !reflect.DeepEqual(a.Scenarists, aInDb.Scenarists) || + !reflect.DeepEqual(a.Colorists, aInDb.Colorists) || + a.Collection != aInDb.Collection) + } + data, err := json.Marshal(a) if err != nil { - return err - } - aInDb, err := db.Get(a.ID) - dbNewSorted := false - if err == nil { - dbNewSorted = (a.PurchaseDate != aInDb.PurchaseDate) + return fmt.Errorf("json encoding: %s", err) } + db.lock.Lock() err = db.dv.Write(AlbumIDString(a.ID), data) if err == nil { - db.sorted = dbNewSorted + db.consolidated = !dbNeedUpdate + db.byID[a.ID] = a } - + db.lock.Unlock() return err } +func deleteFromList(list []AlbumID, idToDelete AlbumID) []AlbumID { + toDeleteIdx := len(list) + for i, aID := range list { + if aID == idToDelete { + toDeleteIdx = i + break + } + } + + if toDeleteIdx == len(list) { + return list + } + return append(list[:toDeleteIdx], list[toDeleteIdx+1:]...) +} + func (db *AlbumDatabase) Delete(ID AlbumID) error { - if db.dv.Has(AlbumIDString(ID)) == false { + inDB, err := db.Get(ID) + if err != nil { return fmt.Errorf("Album %d not found", ID) } + + db.lock.Lock() + defer db.lock.Unlock() + delete(db.byID, ID) + db.byPurchaseDate = deleteFromList(db.byPurchaseDate, ID) + + for _, a := range inDB.Authors() { + db.byAuthor[a] = deleteFromList(db.byAuthor[a], ID) + } + + if len(inDB.Collection) > 0 { + db.byCollection[inDB.Collection] = deleteFromList(db.byCollection[inDB.Collection], ID) + } return db.dv.Erase(AlbumIDString(ID)) } @@ -67,11 +117,14 @@ func (db *AlbumDatabase) get(ID string) (*Album, error) { } func (db *AlbumDatabase) Get(ID AlbumID) (*Album, error) { - return db.get(AlbumIDString(ID)) -} + db.lock.RLock() + defer db.lock.RUnlock() -func (db *AlbumDatabase) GetJSON(ID AlbumID) ([]byte, error) { - return db.dv.Read(AlbumIDString(ID)) + a, ok := db.byID[ID] + if ok == true { + return a, nil + } + return db.get(AlbumIDString(ID)) } type ListOfAlbum []*Album @@ -88,25 +141,67 @@ func (l ListOfAlbum) Swap(i, j int) { l[i], l[j] = l[j], l[i] } -func (db *AlbumDatabase) ByPurchaseDate() ([]AlbumID, error) { - if db.sorted == false { - allAlbums := []*Album{} - for aIDStr := range db.dv.Keys(nil) { - a, err := db.get(aIDStr) - if err != nil { - return nil, err - } - allAlbums = append(allAlbums, a) - } +func maxOfInt(a, b int) int { + if a < b { + return b + } + return a +} - sort.Sort(sort.Reverse(ListOfAlbum(allAlbums))) +func (db *AlbumDatabase) ensureConsolidated() error { + db.lock.RLock() + if db.consolidated == true { + db.lock.RUnlock() + return nil + } + db.lock.RUnlock() - db.sortedList = make([]AlbumID, 0, len(allAlbums)) - for _, a := range allAlbums { - db.sortedList = append(db.sortedList, a.ID) + db.lock.Lock() + defer db.lock.Unlock() + + allAlbums := make([]*Album, 0, maxOfInt(len(db.byID), len(db.byPurchaseDate))) + db.byPurchaseDate = make([]AlbumID, 0, maxOfInt(len(db.byID), len(db.byPurchaseDate))) + db.byID = make(map[AlbumID]*Album) + db.byAuthor = make(map[string][]AlbumID) + db.byCollection = make(map[string][]AlbumID) + for aIDStr := range db.dv.Keys(nil) { + a, err := db.get(aIDStr) + if err != nil { + return err } - db.sorted = true + allAlbums = append(allAlbums, a) + db.byID[a.ID] = a } - return db.sortedList, nil + sort.Sort(sort.Reverse(ListOfAlbum(allAlbums))) + + for _, album := range allAlbums { + db.byPurchaseDate = append(db.byPurchaseDate, album.ID) + } + + for aID, album := range db.byID { + if len(album.Collection) > 0 { + db.byCollection[album.Collection] = append(db.byCollection[album.Collection], aID) + } + + for _, author := range album.Authors() { + db.byAuthor[author] = append(db.byAuthor[author], aID) + } + } + + db.consolidated = true + return nil +} + +func (db *AlbumDatabase) ByPurchaseDate() ([]AlbumID, error) { + err := db.ensureConsolidated() + if err != nil { + return nil, err + } + + db.lock.RLock() + defer db.lock.RUnlock() + res := make([]AlbumID, len(db.byPurchaseDate)) + copy(res, db.byPurchaseDate) + return res, nil } diff --git a/album_database_test.go b/album_database_test.go index 4d18446..19b7924 100644 --- a/album_database_test.go +++ b/album_database_test.go @@ -17,7 +17,9 @@ var _ = Suite(&AlbumDatabaseSuite{}) func (s *AlbumDatabaseSuite) SetUpSuite(c *C) { s.db = OpenAlbumDatabase(filepath.Join(c.MkDir(), "satdb.bar.satellite/db")) for _, a := range albumsDataTest { - c.Assert(s.db.AddOrUpdate(&a), IsNil) + //we need to copy the value to avoid aliasing + aCopied := a + c.Assert(s.db.AddOrUpdate(&aCopied), IsNil) } } @@ -31,7 +33,6 @@ func (s *AlbumDatabaseSuite) TestCanDelete(c *C) { } func (s *AlbumDatabaseSuite) TestCanGet(c *C) { - for _, a := range albumsDataTest { fromDb, err := s.db.Get(a.ID) if c.Check(err, IsNil) == true { @@ -41,7 +42,6 @@ func (s *AlbumDatabaseSuite) TestCanGet(c *C) { } func (s *AlbumDatabaseSuite) TestCanSort(c *C) { - // here start := time.Now() data := []AlbumID{160366, 58595, 15875, 9935, 84448, 46005, 19762, 164, 52100, 8179, 44989, 32043, 22737, 754} sorted, err := s.db.ByPurchaseDate()