satellite/satbd-explorer
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Makes db thread-safe

Browse Source 
It was needed for simplicity and various optimization
This commit is contained in:
Alexandre Tuleu
2016-01-25 13:14:31 +01:00
parent 594598ed6b
commit 575655b6a4
2 changed files with 132 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

163
album_database.go
View File

@@ -3,7 +3,9 @@ package main
import (
"encoding/json"
"fmt"
"reflect"
"sort"
"sync"
"github.com/peterbourgon/diskv"
)
@@ -11,8 +13,14 @@ import (
type AlbumDatabase struct {
dv *diskv.Diskv
sorted bool
sortedList []AlbumID
consolidated bool
byID map[AlbumID]*Album
byPurchaseDate []AlbumID
byCollection map[string][]AlbumID
byAuthor map[string][]AlbumID
lock *sync.RWMutex
}
func OpenAlbumDatabase(basepath string) *AlbumDatabase {
@@ -20,37 +28,79 @@ func OpenAlbumDatabase(basepath string) *AlbumDatabase {
return &AlbumDatabase{
dv: diskv.New(diskv.Options{
BasePath: basepath,
CacheSizeMax: 100 * 1024 * 1024, // 100 MB
CacheSizeMax: 50 * 1024 * 1024, // 50 MB
Compression: diskv.NewGzipCompression(),
}),
sorted: false,
sortedList: make([]AlbumID, 0, 10),
consolidated: false,
byID: make(map[AlbumID]*Album),
byPurchaseDate: make([]AlbumID, 0, 10),
byCollection: make(map[string][]AlbumID),
byAuthor: make(map[string][]AlbumID),
lock: &sync.RWMutex{},
}
}
func (db *AlbumDatabase) AddOrUpdate(a *Album) error {
aInDb, err := db.Get(a.ID)
dbNeedUpdate := true
if err == nil {
dbNeedUpdate = (a.PurchaseDate != aInDb.PurchaseDate ||
!reflect.DeepEqual(a.Designers, aInDb.Designers) ||
!reflect.DeepEqual(a.Scenarists, aInDb.Scenarists) ||
!reflect.DeepEqual(a.Colorists, aInDb.Colorists) ||
a.Collection != aInDb.Collection)
}
data, err := json.Marshal(a)
if err != nil {
return err
}
aInDb, err := db.Get(a.ID)
dbNewSorted := false
if err == nil {
dbNewSorted = (a.PurchaseDate != aInDb.PurchaseDate)
return fmt.Errorf("json encoding: %s", err)
}
db.lock.Lock()
err = db.dv.Write(AlbumIDString(a.ID), data)
if err == nil {
db.sorted = dbNewSorted
db.consolidated = !dbNeedUpdate
db.byID[a.ID] = a
}
db.lock.Unlock()
return err
}
func deleteFromList(list []AlbumID, idToDelete AlbumID) []AlbumID {
toDeleteIdx := len(list)
for i, aID := range list {
if aID == idToDelete {
toDeleteIdx = i
break
}
}
if toDeleteIdx == len(list) {
return list
}
return append(list[:toDeleteIdx], list[toDeleteIdx+1:]...)
}
func (db *AlbumDatabase) Delete(ID AlbumID) error {
if db.dv.Has(AlbumIDString(ID)) == false {
inDB, err := db.Get(ID)
if err != nil {
return fmt.Errorf("Album %d not found", ID)
}
db.lock.Lock()
defer db.lock.Unlock()
delete(db.byID, ID)
db.byPurchaseDate = deleteFromList(db.byPurchaseDate, ID)
for _, a := range inDB.Authors() {
db.byAuthor[a] = deleteFromList(db.byAuthor[a], ID)
}
if len(inDB.Collection) > 0 {
db.byCollection[inDB.Collection] = deleteFromList(db.byCollection[inDB.Collection], ID)
}
return db.dv.Erase(AlbumIDString(ID))
}
@@ -67,11 +117,14 @@ func (db *AlbumDatabase) get(ID string) (*Album, error) {
}
func (db *AlbumDatabase) Get(ID AlbumID) (*Album, error) {
return db.get(AlbumIDString(ID))
}
db.lock.RLock()
defer db.lock.RUnlock()
func (db *AlbumDatabase) GetJSON(ID AlbumID) ([]byte, error) {
return db.dv.Read(AlbumIDString(ID))
a, ok := db.byID[ID]
if ok == true {
return a, nil
}
return db.get(AlbumIDString(ID))
}
type ListOfAlbum []*Album
@@ -88,25 +141,67 @@ func (l ListOfAlbum) Swap(i, j int) {
l[i], l[j] = l[j], l[i]
}
func (db *AlbumDatabase) ByPurchaseDate() ([]AlbumID, error) {
if db.sorted == false {
allAlbums := []*Album{}
for aIDStr := range db.dv.Keys(nil) {
a, err := db.get(aIDStr)
if err != nil {
return nil, err
}
allAlbums = append(allAlbums, a)
}
func maxOfInt(a, b int) int {
if a < b {
return b
}
return a
}
sort.Sort(sort.Reverse(ListOfAlbum(allAlbums)))
func (db *AlbumDatabase) ensureConsolidated() error {
db.lock.RLock()
if db.consolidated == true {
db.lock.RUnlock()
return nil
}
db.lock.RUnlock()
db.sortedList = make([]AlbumID, 0, len(allAlbums))
for _, a := range allAlbums {
db.sortedList = append(db.sortedList, a.ID)
db.lock.Lock()
defer db.lock.Unlock()
allAlbums := make([]*Album, 0, maxOfInt(len(db.byID), len(db.byPurchaseDate)))
db.byPurchaseDate = make([]AlbumID, 0, maxOfInt(len(db.byID), len(db.byPurchaseDate)))
db.byID = make(map[AlbumID]*Album)
db.byAuthor = make(map[string][]AlbumID)
db.byCollection = make(map[string][]AlbumID)
for aIDStr := range db.dv.Keys(nil) {
a, err := db.get(aIDStr)
if err != nil {
return err
}
db.sorted = true
allAlbums = append(allAlbums, a)
db.byID[a.ID] = a
}
return db.sortedList, nil
sort.Sort(sort.Reverse(ListOfAlbum(allAlbums)))
for _, album := range allAlbums {
db.byPurchaseDate = append(db.byPurchaseDate, album.ID)
}
for aID, album := range db.byID {
if len(album.Collection) > 0 {
db.byCollection[album.Collection] = append(db.byCollection[album.Collection], aID)
}
for _, author := range album.Authors() {
db.byAuthor[author] = append(db.byAuthor[author], aID)
}
}
db.consolidated = true
return nil
}
func (db *AlbumDatabase) ByPurchaseDate() ([]AlbumID, error) {
err := db.ensureConsolidated()
if err != nil {
return nil, err
}
db.lock.RLock()
defer db.lock.RUnlock()
res := make([]AlbumID, len(db.byPurchaseDate))
copy(res, db.byPurchaseDate)
return res, nil
}

6
album_database_test.go
View File

@@ -17,7 +17,9 @@ var _ = Suite(&AlbumDatabaseSuite{})
func (s *AlbumDatabaseSuite) SetUpSuite(c *C) {
s.db = OpenAlbumDatabase(filepath.Join(c.MkDir(), "satdb.bar.satellite/db"))
for _, a := range albumsDataTest {
c.Assert(s.db.AddOrUpdate(&a), IsNil)
//we need to copy the value to avoid aliasing
aCopied := a
c.Assert(s.db.AddOrUpdate(&aCopied), IsNil)
}
}
@@ -31,7 +33,6 @@ func (s *AlbumDatabaseSuite) TestCanDelete(c *C) {
}
func (s *AlbumDatabaseSuite) TestCanGet(c *C) {
for _, a := range albumsDataTest {
fromDb, err := s.db.Get(a.ID)
if c.Check(err, IsNil) == true {
@@ -41,7 +42,6 @@ func (s *AlbumDatabaseSuite) TestCanGet(c *C) {
}
func (s *AlbumDatabaseSuite) TestCanSort(c *C) {
// here
start := time.Now()
data := []AlbumID{160366, 58595, 15875, 9935, 84448, 46005, 19762, 164, 52100, 8179, 44989, 32043, 22737, 754}
sorted, err := s.db.ByPurchaseDate()