47 lines
884 B
Go
47 lines
884 B
Go
package metadata
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"net/http"
|
|
"net/url"
|
|
|
|
"borodyadka.dev/borodyadka/minireader/internal/minireader/dto"
|
|
)
|
|
|
|
func fetch(ctx context.Context, cl *http.Client, page string) (io.ReadCloser, error) {
|
|
req, err := http.NewRequestWithContext(ctx, "GET", page, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
resp, err := cl.Do(req)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return resp.Body, nil
|
|
}
|
|
|
|
func FetchPageInfo(ctx context.Context, page string) (dto.WebsiteInfo, error) {
|
|
info := dto.WebsiteInfo{}
|
|
|
|
u, err := url.Parse(page)
|
|
if err != nil {
|
|
return info, err
|
|
}
|
|
|
|
cl := &http.Client{}
|
|
for {
|
|
body, err := fetch(ctx, cl, u.String())
|
|
if err != nil {
|
|
return info, err
|
|
}
|
|
_ = body
|
|
// TODO: handle http status codes (404, 500, 502)
|
|
// TODO: if no info on this page go to parent dir ant try again (maybe add some sleep time?)
|
|
}
|
|
|
|
return info, nil
|
|
}
|