hath

package module
v0.0.22 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Jun 30, 2015 License: MIT Imports: 25 Imported by: 0

README

hath

Hath client in golang

Build status Build Status Coverage Status GoDoc License

Status

Near alpha stage, client is already usable, but lacks testing

Tested by me Client stats

Unit-tested on linux and windows;

Tested on real workload by ernado on ubuntu 15.04 and windows 8.1 x64

Using

Download client, copy to cache folder and start.

Documentation

Overview

Package hath is Hentai@Home client implementation in golang

Index

Constants

View Source 
const (

	// HashSize is length of sha1 hash in bytes
	HashSize = 20

	// FileMaximumSize is maximum image size in hath
	FileMaximumSize = size10MB
)

Variables

View Source 
var (
	// ErrFileNotFound should be returned when file does not exist in frontend
	ErrFileNotFound = errors.New("File not found in cache")
	// ErrFileBadLength means that file size does not equals read size
	ErrFileBadLength = errors.New("Bad lenght in file")

	// ErrFileInconsistent should be returned if file failed to check sha1 hash
	ErrFileInconsistent = errors.New("File has bad hash")
)
View Source 
var (
	// FileTypes list for allowerd images
	FileTypes = []string{"jpg", "png", "gif"}
	// FileTypesN count of FileTypes
	FileTypesN = len(FileTypes)
)
View Source 
var (
	// ErrClientKeyExpired timestampt drift occured
	ErrClientKeyExpired = errors.New("Client key is expired")
	// ErrClientFailedConnectionTest client failed to response on test correctly
	ErrClientFailedConnectionTest = errors.New("Client failed connection test")
	// ErrClientStartupFlood api rpc server flood protection is enabled
	// client should wait
	ErrClientStartupFlood = errors.New("API flood protection enabled")
	// ErrClientOtherConnected other client with same clientID connected
	ErrClientOtherConnected = errors.New("Other client is connected")
	// ErrClientUnexpectedResponse unexpected/unhandler error
	ErrClientUnexpectedResponse = errors.New("Unexpected error")
	// ErrTimeDesync timestamp delta too bit
	ErrTimeDesync = errors.New("Time on server and on client differ too much")
	// ErrClientVersionOld api outdated
	ErrClientVersionOld = errors.New("Client version is too old")
)
View Source 
var (
	// ErrFileTypeUnknown when FileType is UnknownImage
	ErrFileTypeUnknown = errors.New("hath => file type unknown")
	// ErrHashBadLength when hash size is not HashSize
	ErrHashBadLength = errors.New("hath => hash of image has bad length")
)
View Source 
var (
	// ContentTypes is map of file types to conent types
	ContentTypes = map[FileType]string{
		JPG:          "image/jpeg",
		PNG:          "image/png",
		GIF:          "image/gif",
		UnknownImage: "application/octet-stream",
	}
)
View Source 
var (
	// ErrNoFilesToRemove is flag that there is 0 files to remove
	ErrNoFilesToRemove = errors.New("No more unused files")
)
View Source 
var (
	// LocalNetworks is slice of net.IPNet for all local networks subnets
	LocalNetworks []net.IPNet
)

Functions

func FileFromBytesTo 

func FileFromBytesTo(result []byte, f *File) error

FileFromBytesTo deserializes byte slice into file by pointer

func FromRequest 

func FromRequest(req *http.Request) (net.IP, error)

FromRequest extracts the user IP address from req, if present.

func GetRoughCacheSize 

func GetRoughCacheSize(count int64) uint64

GetRoughCacheSize is cache size estimation based on average file size

func IsUnexpected 

func IsUnexpected(err error) bool

IsUnexpected return true if err is ErrUnexpected or ErrClientUnexpectedResponse

func UnmarshalFileTo 

func UnmarshalFileTo(data []byte, f *File) error

UnmarshalFileTo deserializes file info fron byte array by pointer

Types

type APIResponse 

type APIResponse struct {
	Success bool
	Message string
	Data    []string
}

APIResponse represents response from rpc api

func (APIResponse) ParseVars 

func (r APIResponse) ParseVars() Vars

ParseVars parses k=v map from r.Data

type Args 

type Args map[string]string

Args represents additional arguments in request string

func ParseArgs 

func ParseArgs(s string) (a Args)

ParseArgs parses arguments from string 

arg1=val1;arg2=val2

func (Args) Get 

func (a Args) Get(k string) string

Get returns string value

func (Args) GetInt 

func (a Args) GetInt(k string) int

GetInt parses and returns integer value

func (Args) GetInt64 

func (a Args) GetInt64(k string) int64

GetInt64 parses and returns 64bit integer value

func (Args) String 

func (a Args) String() string

type BoltDB 

type BoltDB struct {
	// contains filtered or unexported fields
}

BoltDB stores info about files in cache stores data in b-tree structure stores index on LastUsage+Hash implements DataBase interface

func NewDB 

func NewDB(dbPath string) (d *BoltDB, err error)

NewDB new db

func (BoltDB) Add 

func (d BoltDB) Add(f File) error

Add inserts file info to db

func (BoltDB) AddBatch 

func (d BoltDB) AddBatch(files []File) error

AddBatch inserts slice of files into db

func (BoltDB) Close 

func (d BoltDB) Close() error

Close closes boltdb internal database

func (BoltDB) Count 

func (d BoltDB) Count() (count int)

Count is count of files in database

func (BoltDB) Exists 

func (d BoltDB) Exists(f File) (exists bool)

Exists return true if file exists in db

func (BoltDB) Get 

func (d BoltDB) Get(id []byte) (f File, err error)

Get loads file from database

func (BoltDB) GetBatch 

func (d BoltDB) GetBatch(files chan File, max int64) (err error)

GetBatch reads files and sends it to channel

func (BoltDB) GetOldFiles 

func (d BoltDB) GetOldFiles(maxCount int, deadline time.Time) (files []File, err error)

GetOldFiles returns maxCount or less expired files

func (BoltDB) GetOldFilesCount 

func (d BoltDB) GetOldFilesCount(deadline time.Time) (count int64, err error)

GetOldFilesCount count of files that LastUsage is older than deadline

func (BoltDB) Remove 

func (d BoltDB) Remove(f File) error

Remove deletes file info and index from db

func (BoltDB) RemoveBatch 

func (d BoltDB) RemoveBatch(files []File) error

RemoveBatch remove file and corresponding index records

func (BoltDB) Size 

func (d BoltDB) Size() (sum int64, err error)

Size of all files in cache

func (BoltDB) Use 

func (d BoltDB) Use(f File) error

Use updates LastUsage of provided file

func (BoltDB) UseBatch 

func (d BoltDB) UseBatch(files []File) error

UseBatch updates lastUsage for list of files

type Client 

type Client struct {
	// contains filtered or unexported fields
}

Client is api for hath rpc

func NewClient 

func NewClient(cfg ClientConfig) *Client

NewClient creates new client for api

func (Client) ActionURL 

func (c Client) ActionURL(args ...string) *url.URL

ActionURL - get url for action

func (Client) AddFiles 

func (c Client) AddFiles(files []File) error

AddFiles notifies api server of registered files

func (Client) CheckStats 

func (c Client) CheckStats() error

CheckStats checks time desync and minumum client build returns nil, of time is synced and client version is up to date

func (Client) Close 

func (c Client) Close() error

Close server

func (Client) GetFile 

func (c Client) GetFile(u *url.URL) (rc io.ReadCloser, err error)

GetFile returns io.ReadCloser for given url

func (Client) Login 

func (c Client) Login() error

Login performs sync check and authentication

func (Client) More 

func (c Client) More() error

More files

func (Client) RemoveFiles 

func (c Client) RemoveFiles(files []File) error

RemoveFiles notifies api server of removed files

func (Client) RequestFile 

func (c Client) RequestFile(f File, u *url.URL) (rc io.ReadCloser, err error)

RequestFile from hath server

func (Client) Resume 

func (c Client) Resume() error

Resume server

func (Client) Settings 

func (c Client) Settings() (cfg Settings, err error)

Settings from server

func (Client) Start 

func (c Client) Start() error

Start starts api client

func (Client) StillAlive 

func (c Client) StillAlive() error

StillAlive sends heartbeat

func (Client) Suspend 

func (c Client) Suspend() error

Suspend server

func (Client) Tokens 

func (c Client) Tokens(files []File) (result map[string]string, err error)

Tokens issues requests to obtain tokens for downloading files

type ClientConfig 

type ClientConfig struct {
	Credentials
	Host  string
	Debug bool
}

ClientConfig is configuration for client

type Credentials 

type Credentials struct {
	ClientID int64
	Key      string
}

Credentials of hath client

type DataBase 

type DataBase interface {
	Add(f File) error
	AddBatch(f []File) error
	Use(f File) error
	UseBatch(files []File) error
	Remove(f File) error
	RemoveBatch(f []File) error
	Close() error
	Count() int
	GetOldFiles(maxCount int, deadline time.Time) (files []File, err error)
	GetOldFilesCount(deadline time.Time) (count int64, err error)
	Size() (int64, error)
	Exists(f File) bool
	Get(id []byte) (File, error)
	GetBatch(files chan File, max int64) (err error)
}

DataBase is interface for storing info about files in some DB

type DefaultServer 

type DefaultServer struct {
	// contains filtered or unexported fields
}

DefaultServer uses hard drive to respond

func NewServer 

func NewServer(cfg ServerConfig) *DefaultServer

NewServer cleares default server with provided client and frontend

func (*DefaultServer) Close 

func (s *DefaultServer) Close() error

Close stops server

func (*DefaultServer) Listen 

func (s *DefaultServer) Listen() error

Listen on confgured port

func (DefaultServer) PopulateFromFrontend 

func (s DefaultServer) PopulateFromFrontend() error

PopulateFromFrontend scans frontend and adds all files in it to database

func (*DefaultServer) ServeHTTP 

func (s *DefaultServer) ServeHTTP(w http.ResponseWriter, r *http.Request)

func (*DefaultServer) Start 

func (s *DefaultServer) Start() error

Start server internal goroutines

type DirectCache 

type DirectCache interface {
	Get(file File) (io.ReadCloser, error)
	Remove(f File) error
	RemoveBatch(f []File) error
	Add(file File, r io.Reader) error
	Check(file File) error
	Scan(chan File, chan Progress) error
}

DirectCache is engine for serving files in hath directly from block devices i.e. not using any redirects

type DirectFrontend 

type DirectFrontend struct {
	// contains filtered or unexported fields
}

DirectFrontend is frontend that uses DirectCache

func (*DirectFrontend) Add 

func (d *DirectFrontend) Add(file File, r io.Reader) error

Add file to frontend

func (*DirectFrontend) Check 

func (d *DirectFrontend) Check(file File) error

Check check da file

func (*DirectFrontend) Get 

func (d *DirectFrontend) Get(file File) (io.ReadCloser, error)

Get returns file from fontend

func (*DirectFrontend) Handle 

func (d *DirectFrontend) Handle(file File, w http.ResponseWriter) error

Handle request for file returns ErrFileNotFound, ErrFileBadLength can return unexpected errors

func (*DirectFrontend) Remove 

func (d *DirectFrontend) Remove(file File) error

Remove file

func (*DirectFrontend) RemoveBatch 

func (d *DirectFrontend) RemoveBatch(files []File) error

RemoveBatch removes files

func (*DirectFrontend) Scan 

func (d *DirectFrontend) Scan(files chan File, progress chan Progress) error

Scan scans storage for all files, checks ids and sends files to chan

type ErrUnexpected 

type ErrUnexpected struct {
	Err      error
	Response APIResponse
}

ErrUnexpected error while processing request/response

func (ErrUnexpected) Error 

func (e ErrUnexpected) Error() string

type Event 

type Event struct {
	Type EventType
	File File
}

Event from server

type EventType 

type EventType byte

EventType is type of event from server 

const (
	// EventSent issued when file is sent to hath network
	EventSent EventType = iota
	// EventDownloaded issued when file is downloaded by server from hath
	EventDownloaded
	// EventAdded issued when file is added to cache
	EventAdded
	// EventRemoved issued when file is removed from cache
	EventRemoved
)

func (EventType) String 

func (t EventType) String() string

type File 

type File struct {
	Hash [HashSize]byte `json:"hash"` // 20 byte
	Type FileType       `json:"type"` // 1 byte
	// Static files should never be removed
	Static bool  `json:"static"` // 1 byte
	Size   int64 `json:"size"`   // 4 byte (maximum size 4095mb)
	Width  int   `json:"width"`  // 2 byte
	Height int   `json:"height"` // 2 byte
	// LastUsage is Unix timestamp
	LastUsage int64 `json:"last_usage"` // 8 byte (can be optimized)
}

File is hath file representation total 20 + 4 + 2 + 2 + 1 + 8 + 1 = 38 bytes in memory = 56 bytes

func FileFromBytes 

func FileFromBytes(result []byte) (f File, err error)

FileFromBytes deserializes byte slice into file

func FileFromID 

func FileFromID(fileid string) (f File, err error)

FileFromID generates new File from provided ID

func UnmarshalFile 

func UnmarshalFile(data []byte) (f File, err error)

UnmarshalFile deserializes file info fron byte array

func (File) Basex 

func (f File) Basex() string

Basex returns basex representation of hash

func (*File) Buffer 

func (f *File) Buffer() *bytes.Buffer

Buffer creates buffer with size of file

func (File) ByteID 

func (f File) ByteID() []byte

ByteID returns []byte for file hash

func (File) Bytes 

func (f File) Bytes() []byte

Bytes serializes file info into byte array

func (File) ContentType 

func (f File) ContentType() string

ContentType of image

func (File) Dir 

func (f File) Dir() string

Dir is first prefixLenght chars of file hash

func (File) HexID 

func (f File) HexID() string

HexID returns hex representation of hash

func (File) InRange 

func (f File) InRange(r StaticRange) bool

InRange returns true if file is in static range r

func (File) KeyStamp 

func (f File) KeyStamp(key string, timestamp int64) string

KeyStamp generates file key for provided timestamp

func (File) LastUsageBefore 

func (f File) LastUsageBefore(t time.Time) bool

LastUsageBefore returns true, if last usage occured before deadline t

func (File) Marshal 

func (f File) Marshal() ([]byte, error)

Marshal serializes file info

func (File) Path 

func (f File) Path() string

Path returns relative path to file

func (File) Range 

func (f File) Range() (r StaticRange)

Range returns static range of file

func (*File) SetHash 

func (f *File) SetHash(s string) error

SetHash sets hash from string

func (File) String 

func (f File) String() string

func (*File) Use 

func (f *File) Use()

Use sets LastUsage to current time

type FileCache 

type FileCache struct {
	// contains filtered or unexported fields
}

FileCache serves files from disk no internal buffering, caching or rate limiting is done and should be implement separetaly

func (*FileCache) Add 

func (c *FileCache) Add(file File, r io.Reader) error

Add saves file to storage

func (*FileCache) Check 

func (c *FileCache) Check(file File) error

Check performs sha1 hash checking on file returns nil if all ok

func (*FileCache) Get 

func (c *FileCache) Get(file File) (io.ReadCloser, error)

Get returns readcloser for file if file does not exist, it will return ErrFileNotFound

func (*FileCache) Remove 

func (c *FileCache) Remove(file File) error

Remove removes file from storage

func (*FileCache) RemoveBatch 

func (c *FileCache) RemoveBatch(files []File) error

RemoveBatch removes files from storage

func (*FileCache) Scan 

func (c *FileCache) Scan(results chan File, progress chan Progress) error

Scan storage for files

type FileGenerator 

type FileGenerator struct {
	SizeMax       int64
	SizeMin       int64
	ResolutionMax int
	ResolutionMin int
	Dir           string
	TimeDelta     int64
}

FileGenerator is factory for random files

func (FileGenerator) New 

func (g FileGenerator) New() (f File, err error)

New generates random file and returns it, writing on disk

func (FileGenerator) NewFake 

func (g FileGenerator) NewFake() (f File)

NewFake generates random file without writing it on disk

type FileType 

type FileType byte

FileType represents file format of image 

const (
	// JPG image
	JPG FileType = iota
	// PNG image
	PNG
	// GIF animation
	GIF
	// UnknownImage is not supported format
	UnknownImage
)

func ParseFileType 

func ParseFileType(filetype string) FileType

ParseFileType returns FileType from string

func (FileType) String 

func (f FileType) String() string

type Frontend 

type Frontend interface {
	Handle(file File, w http.ResponseWriter) error
	DirectCache
}

Frontend is cache backend that should processes requests to specidif files, returning them with correct headers and processing IO errors

func NewDirectFrontend 

func NewDirectFrontend(cache DirectCache) Frontend

NewDirectFrontend create direct frontend

func NewFrontend 

func NewFrontend(dir string) Frontend

type HTTPClient 

type HTTPClient interface {
	Get(url string) (*http.Response, error)
	Do(req *http.Request) (*http.Response, error)
}

HTTPClient is underlying http client

type Progress 

type Progress struct {
	Total   int
	Current int
	Percent float32
}

Progress represents status of some ongoing operation

func (Progress) String 

func (p Progress) String() string

type ProxyMode 

type ProxyMode byte

ProxyMode sets proxy security politics 

const (
	// ProxyDisabled no requests allowed
	ProxyDisabled ProxyMode = iota + 1 // starts with 1
	// ProxyLocalNetworksProtected allows requests from local network with passkey
	ProxyLocalNetworksProtected // 2
	// ProxyLocalNetworksOpen allows any requests from local network
	ProxyLocalNetworksOpen // 3
	// ProxyAllNetworksProtected allows requests from any network with passkey (not recommended)
	ProxyAllNetworksProtected // 4
	// ProxyAllNetworksOpen allows any requests from any network (very not recommended)
	ProxyAllNetworksOpen // 5
)

func (ProxyMode) String 

func (p ProxyMode) String() string

type Server 

type Server interface {
	http.Handler
}

Server should handle requests from users (and rpc?) speedtests server commands: 

/servercmd/<command>/<additional:kwds>/<timestamp:int>/<key>'

image file request: 

/h/<fileid>/<additional:kwds>/<filename>

type ServerConfig 

type ServerConfig struct {
	Credentials
	Frontend            Frontend
	DataBase            DataBase
	Client              *Client
	DontCheckTimestamps bool
	DontCheckSHA1       bool
	RemoveTimeout       time.Duration
	RemoveRate          time.Duration
	UpdateRate          time.Duration
	MaxDownloadAttemps  int
	Settings            Settings
	Debug               bool
}

ServerConfig cfg for server

func (*ServerConfig) PopulateDefaults 

func (cfg *ServerConfig) PopulateDefaults()

PopulateDefaults of the config

type Settings 

type Settings struct {
	RPCServers            []net.IP
	ImageServer           string
	RequestServer         string
	LowMemory             bool
	ProxyMode             ProxyMode
	StaticRanges          StaticRanges
	Name                  string
	Host                  net.IP
	Port                  int
	MaximumBytesPerSecond int64
	MaximumCacheSize      int64
	DiskReamainingBytes   int64
}

Settings of hath client

func (Settings) IsRPCServer 

func (s Settings) IsRPCServer(r *http.Request) bool

IsRPCServer returns true, if request is sent from hath rpc server

type StaticRange 

type StaticRange [staticRangeBytes]byte

StaticRange is prefix for static ranges assigned to user

func ParseStaticRange 

func ParseStaticRange(s string) (r StaticRange, err error)

ParseStaticRange parses hex string static range start

func (StaticRange) String 

func (s StaticRange) String() string

type StaticRanges 

type StaticRanges map[StaticRange]bool

StaticRanges contain ranges

func (StaticRanges) Add 

func (s StaticRanges) Add(r StaticRange)

Add static range

func (StaticRanges) Contains 

func (s StaticRanges) Contains(f File) bool

Contains returns true if file f is in static ranges

func (StaticRanges) Count 

func (s StaticRanges) Count() int

Count of static ranges

func (StaticRanges) Remove 

func (s StaticRanges) Remove(r StaticRange)

Remove static range

func (StaticRanges) String 

func (s StaticRanges) String() string

type Stats 

type Stats struct {
	FilesTotal           int
	FilesSent            int
	FilesSentBytes       int64
	FilesDownloaded      int
	FilesDownloadedBytes int64
	Started              time.Time
	Uptime               time.Duration
}

Stats for server

func (*Stats) Process 

func (s *Stats) Process(e Event)

Process event

type Vars 

type Vars map[string]string

Vars represents k-v map from APIResponse.Data

func (Vars) Get 

func (v Vars) Get(k string) string

Get string

func (Vars) GetInt 

func (v Vars) GetInt(k string) (int, error)

GetInt parses int

func (Vars) GetInt64 

func (v Vars) GetInt64(k string) (int64, error)

GetInt64 parses int64

func (Vars) GetProxyMode 

func (v Vars) GetProxyMode(k string) (p ProxyMode, err error)

GetProxyMode parses ProxyMode

func (Vars) GetStaticRange 

func (v Vars) GetStaticRange(k string) (s StaticRanges, err error)

GetStaticRange parses static range list

func (Vars) GetUint64 

func (v Vars) GetUint64(k string) (uint64, error)

GetUint64 parses uint64

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.