Alexandre Tuleu
c070c63463
We don't want to have a large bandwidth on bedetheque.com, so the pace at which we perform GET request is limited by a maximal number of request over a window (per example no more than 10 request over 10 second) If more request are required, the request is simply paused until we go back within the limit ;). Go rulez !
48 lines
974 B
Go
48 lines
974 B
Go
package main
|
|
|
|
import (
|
|
"net/http"
|
|
"time"
|
|
)
|
|
|
|
// An HTTPGetter can GET over HTTP
|
|
type HTTPGetter interface {
|
|
Get(URL string) (*http.Response, error)
|
|
}
|
|
|
|
type simpleHTTPGetter struct{}
|
|
|
|
// NewHTTPGetter returns the simplest object that can GET over HTTP
|
|
func NewHTTPGetter() HTTPGetter {
|
|
return &simpleHTTPGetter{}
|
|
}
|
|
|
|
func (g *simpleHTTPGetter) Get(URL string) (*http.Response, error) {
|
|
return http.Get(URL)
|
|
}
|
|
|
|
type rateLimitedHTTPGetter struct {
|
|
tokens chan bool
|
|
window time.Duration
|
|
}
|
|
|
|
// NewRateLimitedGetter returns an HTTPGetter that is limited by a
|
|
// maximal amount of request over a time window
|
|
func NewRateLimitedGetter(maxRequest uint, window time.Duration) HTTPGetter {
|
|
return &rateLimitedHTTPGetter{
|
|
tokens: make(chan bool, maxRequest),
|
|
window: window,
|
|
}
|
|
}
|
|
|
|
func (g *rateLimitedHTTPGetter) Get(URL string) (*http.Response, error) {
|
|
g.tokens <- true
|
|
defer func() {
|
|
go func() {
|
|
time.Sleep(g.window)
|
|
<-g.tokens
|
|
}()
|
|
}()
|
|
return http.Get(URL)
|
|
}
|