Makes a loop to fetch everything

main.go

@@ -1,14 +1,14 @@
 package main
 
 import (
-	"encoding/csv"
 	"fmt"
+	"io"
 	"log"
-	"net/http"
 	"os"
-	"strconv"
+	"path/filepath"
-	"strings"
 	"time"
+
+	"launchpad.net/go-xdg/v0"
 )
 
 // Execute executes the job
@@ -17,74 +17,87 @@ func Execute() error {
 		return fmt.Errorf("Missing mandatory .csv parameter")
 	}
 
+	albums := make(chan *Album, 1000)
+	errors := make(chan error, 10)
+	parsed := make(chan bool, 1000)
+	cached := make(chan bool, 1000)
+	cacheBase, err := xdg.Cache.Ensure(filepath.Join("org.satellite.satbd", "test"))
+	if err != nil {
+		return err
+	}
+	cacheBase = filepath.Dir(cacheBase)
+	c, err := NewAlbumDescriptionCache(cacheBase, 10, 10*time.Second)
+	if err != nil {
+		return err
+	}
+	start := time.Now()
+	go func() {
+		defer close(albums)
 		csvFile, err := os.Open(os.Args[1])
 		if err != nil {
-		return err
+			errors <- err
+			return
 		}
 		defer csvFile.Close()
-
+		csvReader, err := NewAlbumCsvReader(csvFile)
-	csvReader := csv.NewReader(csvFile)
-	csvReader.Comma = ';'
-	_, err = csvReader.Read()
 		if err != nil {
-		return err
+			errors <- err
+			return
 		}
 
-	errorFile, err := os.Create("errors.csv")
-	if err != nil {
-		return err
-	}
-
-	waitUntil := time.Now()
 		for {
-		waitUntil = waitUntil.Add(2 * time.Second)
+			a, err := csvReader.Read()
-		data, err := csvReader.Read()
 			if err != nil {
-			return err
+				if err == io.EOF {
+					break
 				}
+				errors <- err
+			}
+			parsed <- true
+			albums <- a
+		}
+		log.Printf("Done parsing albums in %s", time.Since(start))
+	}()
 
-		a := Album{}
+	go func() {
-		// ID
+		defer close(errors)
-		ID, err := strconv.ParseInt(data[0], 0, 64)
+		for {
-		if err != nil {
+			a, ok := <-albums
-			return err
+			if ok == false {
+				break
 			}
-		a.ID = AlbumID(ID)
+			_, err := c.GetDescription(a.ID)
-		// ISBN
+			cached <- true
-		a.ISBN = data[1]
+			if err != nil {
-		// Series
+				errors <- err
-		a.Series = data[2]
+			}
-		//Num
+		}
-		if len(data[3]) == 0 {
+		log.Printf("DONE caching in %s", time.Since(start))
-			a.Num = -1
+	}()
+
+	pCount := 0
+	cCount := 0
+	errCount := 0
+	done := false
+	for done == false {
+		select {
+		case err, ok := <-errors:
+			if ok == false {
+				done = true
 			} else {
-			Num, err := strconv.ParseInt(data[3], 0, 64)
+				errCount++
+			}
 			if err != nil {
-				return err
+				fmt.Printf("\033[1A\033[KGot error: %s\n\n", err)
 			}
-			a.Num = int(Num)
+		case <-parsed:
+			pCount++
+		case <-cached:
+			cCount++
 		}
-		// NumA
+		fmt.Printf("\033[1A\033[K%d/%d (errors:%d)\n", cCount, pCount, errCount)
-		a.NumA = data[4]
-		// Titre
-		a.Title = data[5]
-
-		URL := "http://www.bedetheque.com/" + a.GetBedethequeComURI()
-		log.Printf("Contacting %s", URL)
-		resp, err := http.Get(URL)
-		if err != nil {
-			return err
 	}
-		defer resp.Body.Close()
+	return nil
-
-		log.Printf("Got %d", resp.StatusCode)
-		if resp.StatusCode >= 400 {
-			fmt.Fprintf(errorFile, "%s;%d", strings.Join(data, ";"), resp.StatusCode)
-		}
-
-		time.Sleep(waitUntil.Sub(time.Now()))
-	}
-
 }
 
 func main() {