From 4a18666fe2ce8aa17487abc82a704f3f92f0510d Mon Sep 17 00:00:00 2001 From: Alexandre Tuleu Date: Thu, 21 Jan 2016 20:24:36 +0100 Subject: [PATCH] Implements the database --- album.go | 4 +- album_database.go | 107 ++++++++++++++++++++++++++++++++++++----- album_database_test.go | 15 +++--- bleve_indexer_test.go | 2 +- main.go | 2 +- 5 files changed, 109 insertions(+), 21 deletions(-) diff --git a/album.go b/album.go index cfb62f7..753ff56 100644 --- a/album.go +++ b/album.go @@ -67,6 +67,6 @@ type Album struct { FetchDate time.Time } -func (a *Album) IDString() string { - return strconv.FormatUint(uint64(a.ID), 10) +func AlbumIDString(ID AlbumID) string { + return strconv.FormatUint(uint64(ID), 10) } diff --git a/album_database.go b/album_database.go index dd1ec8b..6eafce9 100644 --- a/album_database.go +++ b/album_database.go @@ -1,27 +1,112 @@ package main -import "github.com/peterbourgon/diskv" +import ( + "encoding/json" + "fmt" + "sort" + + "github.com/peterbourgon/diskv" +) type AlbumDatabase struct { - db *diskv.Diskv + dv *diskv.Diskv + + sorted bool + sortedList []AlbumID } -func OpenAlbumDatabase(basepath string) (*AlbumDatabase, error) { - return nil, notYetImplemented() +func OpenAlbumDatabase(basepath string) *AlbumDatabase { + + return &AlbumDatabase{ + dv: diskv.New(diskv.Options{ + BasePath: basepath, + CacheSizeMax: 100 * 1024 * 1024, // 100 MB + Compression: diskv.NewGzipCompression(), + }), + sorted: false, + sortedList: make([]AlbumID, 0, 10), + } } -func (db *AlbumDatabase) AddOrUpdate(*Album) error { - return notYetImplemented() +func (db *AlbumDatabase) AddOrUpdate(a *Album) error { + data, err := json.Marshal(a) + if err != nil { + return err + } + aInDb, err := db.Get(a.ID) + dbNewSorted := false + if err == nil { + dbNewSorted = (a.PurchaseDate != aInDb.PurchaseDate) + } + + err = db.dv.Write(AlbumIDString(a.ID), data) + if err == nil { + db.sorted = dbNewSorted + } + + return err } -func (db *AlbumDatabase) Delete(AlbumID) error { - return notYetImplemented() +func (db *AlbumDatabase) Delete(ID AlbumID) error { + if db.dv.Has(AlbumIDString(ID)) == false { + return fmt.Errorf("Album %d not found", ID) + } + return db.dv.Erase(AlbumIDString(ID)) } -func (db *AlbumDatabase) Get(AlbumID) (*Album, error) { - return nil, notYetImplemented() +func (db *AlbumDatabase) get(ID string) (*Album, error) { + res := &Album{} + + r, err := db.dv.ReadStream(ID, false) + if err != nil { + return nil, fmt.Errorf("Album %s not found: %s", ID, err) + } + dec := json.NewDecoder(r) + defer closeOrPanic(r, "db record "+ID) + return res, dec.Decode(res) +} + +func (db *AlbumDatabase) Get(ID AlbumID) (*Album, error) { + return db.get(AlbumIDString(ID)) +} + +func (db *AlbumDatabase) GetJSON(ID AlbumID) ([]byte, error) { + return db.dv.Read(AlbumIDString(ID)) +} + +type ListOfAlbum []*Album + +func (l ListOfAlbum) Len() int { + return len(l) +} + +func (l ListOfAlbum) Less(i, j int) bool { + return l[i].PurchaseDate.Before(l[j].PurchaseDate) +} + +func (l ListOfAlbum) Swap(i, j int) { + l[i], l[j] = l[j], l[i] } func (db *AlbumDatabase) ByPurchaseDate() ([]AlbumID, error) { - return nil, notYetImplemented() + if db.sorted == false { + allAlbums := []*Album{} + for aIDStr := range db.dv.Keys(nil) { + a, err := db.get(aIDStr) + if err != nil { + return nil, err + } + allAlbums = append(allAlbums, a) + } + + sort.Sort(sort.Reverse(ListOfAlbum(allAlbums))) + + db.sortedList = make([]AlbumID, 0, len(allAlbums)) + for _, a := range allAlbums { + db.sortedList = append(db.sortedList, a.ID) + } + db.sorted = true + } + + return db.sortedList, nil } diff --git a/album_database_test.go b/album_database_test.go index ef0bea4..4d18446 100644 --- a/album_database_test.go +++ b/album_database_test.go @@ -1,7 +1,9 @@ package main import ( + "log" "path/filepath" + "time" . "gopkg.in/check.v1" ) @@ -13,10 +15,7 @@ type AlbumDatabaseSuite struct { var _ = Suite(&AlbumDatabaseSuite{}) func (s *AlbumDatabaseSuite) SetUpSuite(c *C) { - var err error - s.db, err = OpenAlbumDatabase(filepath.Join(c.MkDir(), "satdb.bar.satellite/db")) - c.Assert(err, IsNil) - + s.db = OpenAlbumDatabase(filepath.Join(c.MkDir(), "satdb.bar.satellite/db")) for _, a := range albumsDataTest { c.Assert(s.db.AddOrUpdate(&a), IsNil) } @@ -43,11 +42,15 @@ func (s *AlbumDatabaseSuite) TestCanGet(c *C) { func (s *AlbumDatabaseSuite) TestCanSort(c *C) { // here - data := []AlbumID{} + start := time.Now() + data := []AlbumID{160366, 58595, 15875, 9935, 84448, 46005, 19762, 164, 52100, 8179, 44989, 32043, 22737, 754} sorted, err := s.db.ByPurchaseDate() + + log.Printf("sorting took %s", time.Since(start)) + c.Assert(err, IsNil) c.Assert(len(sorted), Equals, len(data)) for i, a := range sorted { - c.Check(a, Equals, data[i]) + c.Check(a, Equals, data[i], Commentf("expected %d", a)) } } diff --git a/bleve_indexer_test.go b/bleve_indexer_test.go index 9980309..a7e1b7b 100644 --- a/bleve_indexer_test.go +++ b/bleve_indexer_test.go @@ -22,7 +22,7 @@ func (s *BleveIndexerSuite) SetUpSuite(c *C) { s.i, err = bleve.New(filepath.Join(c.MkDir(), "satbd-test.bar.satellite"), buildAlbumMapping()) c.Assert(err, IsNil) for _, a := range albumsDataTest { - c.Assert(s.i.Index((&a).IDString(), &a), IsNil) + c.Assert(s.i.Index(AlbumIDString(a.ID), &a), IsNil) } log.Printf("Indexing took %s", time.Since(start)) } diff --git a/main.go b/main.go index de730b4..e50ea53 100644 --- a/main.go +++ b/main.go @@ -54,7 +54,7 @@ func indexAlbums(i bleve.Index, albums chan *Album, errors chan error) { iAlbum := 0 start := time.Now() for a := range albums { - err := i.Index(a.IDString(), a) + err := i.Index(AlbumIDString(a.ID), a) if err != nil { errors <- err }