Makes a loop to fetch everything

main.go

@@ -1,14 +1,14 @@
 package main
 
 import (
-	"encoding/csv"
 	"fmt"
+	"io"
 	"log"
-	"net/http"
 	"os"
-	"strconv"
-	"strings"
+	"path/filepath"
 	"time"
+
+	"launchpad.net/go-xdg/v0"
 )
 
 // Execute executes the job
@@ -17,74 +17,87 @@ func Execute() error {
 		return fmt.Errorf("Missing mandatory .csv parameter")
 	}
 
-	csvFile, err := os.Open(os.Args[1])
+	albums := make(chan *Album, 1000)
+	errors := make(chan error, 10)
+	parsed := make(chan bool, 1000)
+	cached := make(chan bool, 1000)
+	cacheBase, err := xdg.Cache.Ensure(filepath.Join("org.satellite.satbd", "test"))
 	if err != nil {
 		return err
 	}
-	defer csvFile.Close()
-
-	csvReader := csv.NewReader(csvFile)
-	csvReader.Comma = ';'
-	_, err = csvReader.Read()
+	cacheBase = filepath.Dir(cacheBase)
+	c, err := NewAlbumDescriptionCache(cacheBase, 10, 10*time.Second)
 	if err != nil {
 		return err
 	}
-
-	errorFile, err := os.Create("errors.csv")
-	if err != nil {
-		return err
-	}
-
-	waitUntil := time.Now()
-	for {
-		waitUntil = waitUntil.Add(2 * time.Second)
-		data, err := csvReader.Read()
+	start := time.Now()
+	go func() {
+		defer close(albums)
+		csvFile, err := os.Open(os.Args[1])
 		if err != nil {
-			return err
+			errors <- err
+			return
+		}
+		defer csvFile.Close()
+		csvReader, err := NewAlbumCsvReader(csvFile)
+		if err != nil {
+			errors <- err
+			return
 		}
 
-		a := Album{}
-		// ID
-		ID, err := strconv.ParseInt(data[0], 0, 64)
-		if err != nil {
-			return err
-		}
-		a.ID = AlbumID(ID)
-		// ISBN
-		a.ISBN = data[1]
-		// Series
-		a.Series = data[2]
-		//Num
-		if len(data[3]) == 0 {
-			a.Num = -1
-		} else {
-			Num, err := strconv.ParseInt(data[3], 0, 64)
+		for {
+			a, err := csvReader.Read()
 			if err != nil {
-				return err
+				if err == io.EOF {
+					break
+				}
+				errors <- err
 			}
-			a.Num = int(Num)
+			parsed <- true
+			albums <- a
 		}
-		// NumA
-		a.NumA = data[4]
-		// Titre
-		a.Title = data[5]
+		log.Printf("Done parsing albums in %s", time.Since(start))
+	}()
 
-		URL := "http://www.bedetheque.com/" + a.GetBedethequeComURI()
-		log.Printf("Contacting %s", URL)
-		resp, err := http.Get(URL)
-		if err != nil {
-			return err
+	go func() {
+		defer close(errors)
+		for {
+			a, ok := <-albums
+			if ok == false {
+				break
+			}
+			_, err := c.GetDescription(a.ID)
+			cached <- true
+			if err != nil {
+				errors <- err
+			}
 		}
-		defer resp.Body.Close()
+		log.Printf("DONE caching in %s", time.Since(start))
+	}()
 
-		log.Printf("Got %d", resp.StatusCode)
-		if resp.StatusCode >= 400 {
-			fmt.Fprintf(errorFile, "%s;%d", strings.Join(data, ";"), resp.StatusCode)
+	pCount := 0
+	cCount := 0
+	errCount := 0
+	done := false
+	for done == false {
+		select {
+		case err, ok := <-errors:
+			if ok == false {
+				done = true
+			} else {
+				errCount++
+			}
+			if err != nil {
+				fmt.Printf("\033[1A\033[KGot error: %s\n\n", err)
+			}
+		case <-parsed:
+			pCount++
+		case <-cached:
+			cCount++
 		}
-
-		time.Sleep(waitUntil.Sub(time.Now()))
+		fmt.Printf("\033[1A\033[K%d/%d (errors:%d)\n", cCount, pCount, errCount)
 	}
-
+	return nil
 }
 
 func main() {