webextractor

WebExtractor

Package webextractor provides a ready-to-use and extensible web crawling and scraping framework for extracting structured data from the web.

See Colibri.

Installation

 $ go get github.com/gonzxlezs/webextractor

Do

package main

import (
	"encoding/json"
	"fmt"
	"net/url"

	"github.com/gonzxlezs/webextractor"
	"github.com/gonzxlezs/webextractor/colibri"
)

var rawRules = `{
	"Method": "GET",
	"IgnoreRobotsTxt": true
}`

func main() {
	we, err := webextractor.New()
	if err != nil {
		panic(err)
	}

	var rules colibri.Rules
	err = json.Unmarshal([]byte(rawRules), &rules)
	if err != nil {
		panic(err)
	}

	u, err := url.Parse("https://example.com")
	if err != nil {
		panic(err)
	}

	out, err := we.Do(u, &rules)
	if err != nil {
		panic(err)
	}

	fmt.Println("URL:", out.Response.URL())
	fmt.Println("Status code:", out.Response.StatusCode())
	fmt.Println("Content-Type", out.Response.Header().Get("Content-Type"))
}

URL: https://example.com
Status code: 200
Content-Type text/html

Extract

package main

import (
	"encoding/json"
	"fmt"
	"net/url"

	"github.com/gonzxlezs/webextractor"
	"github.com/gonzxlezs/webextractor/colibri"
)

var rawRules = `{
	"Method": "GET",
	"Selectors": {
		"title": "//title"
	}
}`

func main() {
	we, err := webextractor.New()
	if err != nil {
		panic(err)
	}

	var rules colibri.Rules
	err = json.Unmarshal([]byte(rawRules), &rules)
	if err != nil {
		panic(err)
	}

	u, err := url.Parse("https://example.com")
	if err != nil {
		panic(err)
	}

	output, err := we.Extract(u, &rules)
	if err != nil {
		panic(err)
	}

	fmt.Println("URL:", output.Response.URL())
	fmt.Println("Status code:", output.Response.StatusCode())
	fmt.Println("Content-Type", output.Response.Header().Get("Content-Type"))
	fmt.Println("Data:", output.Data)
}

URL: https://example.com
Status code: 200
Content-Type text/html
Data: map[title:Example Domain]