crawler

type Crawler struct {
	// contains filtered or unexported fields
}

type FilterFunc func(ctx context.Context, r io.Reader, yieldTitle func(pos int, title string) error, yieldLink func(pos int, link string) error) error

type Options struct {
	Client *http.Client

	UserAgent string

	// Zero for unlimited depth
	Depth int

	// Zero for no limit
	ParallelRequestsPerHost int
}

type YieldErrorFunc func(origin, link string, pos int, err error)

type YieldTitleFunc func(depth, pos int, origin string, title string)

type YieldURLFunc func(depth, pos int, origin string, originalLink string, link *url.URL, external bool) bool