From d02ff08aa75c0880e5882c4f31419c756dbbb02e Mon Sep 17 00:00:00 2001 From: Pavel Kilin Date: Wed, 25 Oct 2023 21:49:11 +0700 Subject: [PATCH] init --- .air.toml | 46 +++ .env.dist | 0 .gitignore | 4 + .golangci.yml | 348 ++++++++++++++++++ Dockerfile | 12 + Dockerfile.dev | 9 + LICENSE | 19 + Makefile | 11 + README.md | 11 + cmd/minireader/main.go | 25 ++ docker-compose.dev.yml | 9 + docker-compose.yml | 32 ++ go.mod | 61 +++ go.sum | 171 +++++++++ internal/minireader/app.go | 152 ++++++++ internal/minireader/config/config.go | 43 +++ internal/minireader/dto/errors.go | 9 + internal/minireader/dto/reader.go | 77 ++++ internal/minireader/dto/site.go | 7 + internal/minireader/dto/user.go | 5 + internal/minireader/fetcher/fetcher.go | 148 ++++++++ internal/minireader/fetcher/options.go | 38 ++ internal/minireader/fetcher/rss/fetcher.go | 64 ++++ internal/minireader/logger.go | 83 +++++ internal/minireader/metadata/urlinfo.go | 46 +++ internal/minireader/reader/options.go | 41 +++ internal/minireader/reader/reader.go | 157 ++++++++ internal/minireader/repositories/database.go | 77 ++++ internal/minireader/repositories/feed.go | 92 +++++ internal/minireader/repositories/item.go | 53 +++ .../minireader/repositories/sql/.gitignore | 1 + .../minireader/repositories/sql/query.sql | 63 ++++ .../minireader/repositories/subscription.go | 36 ++ internal/minireader/repositories/user.go | 19 + internal/minireader/transport/http/auth.go | 90 +++++ internal/minireader/transport/http/dto.go | 27 ++ .../minireader/transport/http/handlers.go | 86 +++++ internal/minireader/transport/http/options.go | 43 +++ internal/minireader/transport/http/server.go | 103 ++++++ internal/pkg/errx/details.go | 66 ++++ internal/pkg/errx/details_test.go | 101 +++++ internal/pkg/utils/deref.go | 9 + internal/pkg/utils/slices.go | 13 + internal/pkg/validator/validator.go | 19 + migrations/20231021105313_init.sql | 145 ++++++++ migrations/embed.go | 9 + sqlc.yaml | 32 ++ 47 files changed, 2712 insertions(+) create mode 100644 .air.toml create mode 100644 .env.dist create mode 100644 .gitignore create mode 100644 .golangci.yml create mode 100644 Dockerfile create mode 100644 Dockerfile.dev create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 README.md create mode 100644 cmd/minireader/main.go create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.yml create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/minireader/app.go create mode 100644 internal/minireader/config/config.go create mode 100644 internal/minireader/dto/errors.go create mode 100644 internal/minireader/dto/reader.go create mode 100644 internal/minireader/dto/site.go create mode 100644 internal/minireader/dto/user.go create mode 100644 internal/minireader/fetcher/fetcher.go create mode 100644 internal/minireader/fetcher/options.go create mode 100644 internal/minireader/fetcher/rss/fetcher.go create mode 100644 internal/minireader/logger.go create mode 100644 internal/minireader/metadata/urlinfo.go create mode 100644 internal/minireader/reader/options.go create mode 100644 internal/minireader/reader/reader.go create mode 100644 internal/minireader/repositories/database.go create mode 100644 internal/minireader/repositories/feed.go create mode 100644 internal/minireader/repositories/item.go create mode 100644 internal/minireader/repositories/sql/.gitignore create mode 100644 internal/minireader/repositories/sql/query.sql create mode 100644 internal/minireader/repositories/subscription.go create mode 100644 internal/minireader/repositories/user.go create mode 100644 internal/minireader/transport/http/auth.go create mode 100644 internal/minireader/transport/http/dto.go create mode 100644 internal/minireader/transport/http/handlers.go create mode 100644 internal/minireader/transport/http/options.go create mode 100644 internal/minireader/transport/http/server.go create mode 100644 internal/pkg/errx/details.go create mode 100644 internal/pkg/errx/details_test.go create mode 100644 internal/pkg/utils/deref.go create mode 100644 internal/pkg/utils/slices.go create mode 100644 internal/pkg/validator/validator.go create mode 100644 migrations/20231021105313_init.sql create mode 100644 migrations/embed.go create mode 100644 sqlc.yaml diff --git a/.air.toml b/.air.toml new file mode 100644 index 0000000..5c697f9 --- /dev/null +++ b/.air.toml @@ -0,0 +1,46 @@ +root = "." +testdata_dir = "testdata" +tmp_dir = "/tmp" + +[build] + args_bin = [] + bin = "/tmp/minireader" + cmd = "go build -o /tmp/minireader ./cmd/minireader" + delay = 1000 + exclude_dir = ["assets", "tmp", "vendor", "testdata"] + exclude_file = [] + exclude_regex = ["_test.go"] + exclude_unchanged = false + follow_symlink = false + full_bin = "" + include_dir = [] + include_ext = ["go", "tpl", "tmpl", "html"] + include_file = [] + kill_delay = "0s" + log = "/tmp/build-errors.log" + poll = false + poll_interval = 0 + post_cmd = [] + pre_cmd = [] + rerun = false + rerun_delay = 500 + send_interrupt = false + stop_on_error = false + +[color] + app = "" + build = "yellow" + main = "magenta" + runner = "green" + watcher = "cyan" + +[log] + main_only = false + time = false + +[misc] + clean_on_exit = false + +[screen] + clear_on_rebuild = false + keep_scroll = true diff --git a/.env.dist b/.env.dist new file mode 100644 index 0000000..e69de29 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e6567bb --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.idea + +.env +tmp/* diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..6fc0bfe --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,348 @@ +# Options for analysis running. +run: + # Timeout for analysis, e.g. 30s, 5m. + # Default: 1m + timeout: 10m + # Exit code when at least one issue was found. + # Default: 1 + issues-exit-code: 1 + # Include test files or not. + # Default: true + tests: true + # List of build tags, all linters use it. + # Default: []. + build-tags: [ ] + # Which dirs to skip: issues from them won't be reported. + # Can use regexp here: `generated.*`, regexp is applied on full path, + # including the path prefix if one is set. + # Default value is empty list, + # but default dirs are skipped independently of this option's value (see skip-dirs-use-default). + # "/" will be replaced by current OS file path separator to properly work on Windows. + skip-dirs: + - pkg/ + - migrations/ + # If set we pass it to "go list -mod={option}". From "go help modules": + # If invoked with -mod=readonly, the go command is disallowed from the implicit + # automatic updating of go.mod described above. Instead, it fails when any changes + # to go.mod are needed. This setting is most useful to check that go.mod does + # not need updates, such as in a continuous integration and testing system. + # If invoked with -mod=vendor, the go command assumes that the vendor + # directory holds the correct copies of dependencies and ignores + # the dependency descriptions in go.mod. + # + # Allowed values: readonly|vendor|mod + # By default, it isn't set. + modules-download-mode: readonly + # Allow multiple parallel golangci-lint instances running. + allow-parallel-runners: true + +output: + sort-results: true + +linters: + enable: + - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers + - containedctx # containedctx is a linter that detects struct contained context.Context field + - cyclop # checks function and package cyclomatic complexity + - dupl # Tool for code clone detection + - errname # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error` + - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13 + - exhaustive # check exhaustiveness of enum switch statements + - exhaustruct # Checks if all structure fields are initialized + - funlen # Tool for detection of long functions + - gocognit # Computes and checks the cognitive complexity of functions + - goconst # Finds repeated strings that could be replaced by a constant + - gocritic # Provides diagnostics that check for bugs, performance and style issues + - lll # Reports long lines + - makezero # Finds slice declarations with non-zero initial length + - nilerr # Finds the code that returns nil even if it checks that the error is not nil + - prealloc # Finds slice declarations that could potentially be pre-allocated + - tagliatelle # Checks the struct tags + # TODO: consider following linters + + # - fieldalignment + # - wrapcheck + +linters-settings: + cyclop: + # The maximal code complexity to report. + # Default: 10 + max-complexity: 10 + # The maximal average package complexity. + # If it's higher than 0.0 (float) the check is enabled + # Default: 0.0 + package-average: 0.0 + # Should ignore tests. + # Default: false + skip-tests: true + dupl: + # Tokens count to trigger issue. + # Default: 150 + threshold: 150 + errorlint: + # Check whether fmt.Errorf uses the %w verb for formatting errors. + # See the https://github.com/polyfloyd/go-errorlint for caveats. + # Default: true + errorf: true + # Permit more than 1 %w verb, valid per Go 1.20 (Requires errorf:true) + # Default: true + errorf-multi: true + # Check for plain type assertions and type switches. + # Default: true + asserts: true + # Check for plain error comparisons. + # Default: true + comparison: true + exhaustruct: + # List of regular expressions to exclude struct packages and names from check. + # Default: [] + exclude: + # public libs + - "^github.com/segmentio/kafka-go.Message$" + exhaustive: + # Program elements to check for exhaustiveness. + # Default: [ switch ] + check: + - switch + # - map + # Check switch statements in generated files also. + # Default: false + check-generated: false + # Presence of "default" case in switch statements satisfies exhaustiveness, + # even if all enum members are not listed. + # Default: false + default-signifies-exhaustive: false + # Enum members matching the supplied regex do not have to be listed in + # switch statements to satisfy exhaustiveness. + # Default: "" + ignore-enum-members: "Example.+" + # Enum types matching the supplied regex do not have to be listed in + # switch statements to satisfy exhaustiveness. + # Default: "" + ignore-enum-types: "Example.+" + # Consider enums only in package scopes, not in inner scopes. + # Default: false + package-scope-only: false + # Only run exhaustive check on switches with "//exhaustive:enforce" comment. + # Default: false + explicit-exhaustive-switch: false + # Only run exhaustive check on map literals with "//exhaustive:enforce" comment. + # Default: false + explicit-exhaustive-map: false + funlen: + # Checks the number of lines in a function. + # If lower than 0, disable the check. + # Default: 60 + lines: 60 + # Checks the number of statements in a function. + # If lower than 0, disable the check. + # Default: 40 + statements: 40 + gocognit: + # Minimal code complexity to report. + # Default: 30 + min-complexity: 30 + goconst: + # Minimal length of string constant. + # Default: 3 + min-len: 3 + # Minimum occurrences of constant string count to trigger issue. + # Default: 3 + min-occurrences: 3 + # Ignore test files. + # Default: false + ignore-tests: true + # Look for existing constants matching the values. + # Default: true + match-constant: true + # Search also for duplicated numbers. + # Default: false + numbers: false + # Ignore when constant is not used as function argument. + # Default: true + ignore-calls: true + gocritic: + # Which checks should be enabled; can't be combined with 'disabled-checks'. + # See https://go-critic.github.io/overview#checks-overview. + # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run`. + # By default, list of stable checks is used. + enabled-checks: + - appendAssign + - appendCombine + - badCond + - badLock + - badRegexp + - boolExprSimplify + - builtinShadow + - builtinShadowDecl + - captLocal + - caseOrder + - commentFormatting + - commentedOutCode + - commentedOutImport + - defaultCaseOrder + - deferInLoop + - deferUnlambda + - deprecatedComment + - docStub + - dupArg + - dupBranchBody + - dupCase + - dupImport + - dupSubExpr + - dynamicFmtString + - elseif + - emptyDecl + - emptyFallthrough + - emptyStringTest + - exposedSyncMutex + - externalErrorReassign + - filepathJoin + - flagDeref + - flagName + - hugeParam + - ifElseChain + - indexAlloc + - mapKey + - nestingReduce + - nilValReturn + - octalLiteral + - offBy1 + - preferDecodeRune + - preferFilepathJoin + - ptrToRefParam + - rangeExprCopy + - rangeValCopy + - regexpMust + - regexpSimplify + - singleCaseSwitch + - sliceClear + - sloppyLen + - sloppyReassign + - sloppyTypeAssert + - stringConcatSimplify + - stringXbytes + - stringsCompare + - switchTrue + - timeCmpSimplify + - timeExprSimplify + - truncateCmp + - typeAssertChain + - typeDefFirst + - typeSwitchVar + - typeUnparen + - uncheckedInlineErr + - underef + - unlambda + - unnecessaryDefer + - weakCond + - whyNoLint + - wrapperFunc + - yodaStyleExpr + # Settings passed to gocritic. + # The settings key is the name of a supported gocritic checker. + # The list of supported checkers can be find in https://go-critic.github.io/overview. + settings: + # Must be valid enabled check name. + captLocal: + # Whether to restrict checker to params only. + # Default: true + paramsOnly: true + elseif: + # Whether to skip balanced if-else pairs. + # Default: true + skipBalanced: true + hugeParam: + # Size in bytes that makes the warning trigger. + # Default: 80 + sizeThreshold: 512 + nestingReduce: + # Min number of statements inside a branch to trigger a warning. + # Default: 5 + bodyWidth: 3 + rangeExprCopy: + # Size in bytes that makes the warning trigger. + # Default: 512 + sizeThreshold: 512 + # Whether to check test functions + # Default: true + skipTestFuncs: true + rangeValCopy: + # Size in bytes that makes the warning trigger. + # Default: 128 + sizeThreshold: 128 + # Whether to check test functions. + # Default: true + skipTestFuncs: true + truncateCmp: + # Whether to skip int/uint/uintptr types. + # Default: true + skipArchDependent: true + underef: + # Whether to skip (*x).method() calls where x is a pointer receiver. + # Default: true + skipRecvDeref: true + prealloc: + # IMPORTANT: we don't recommend using this linter before doing performance profiling. + # For most programs usage of prealloc will be a premature optimization. + + # Report pre-allocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. + # Default: true + simple: true + # Report pre-allocation suggestions on range loops. + # Default: true + range-loops: true + # Report pre-allocation suggestions on for loops. + # Default: false + for-loops: true + tagliatelle: + # Check the struct tag name case. + case: + # Use the struct field name to check the name of the struct tag. + # Default: false + use-field-name: false + # `camel` is used for `json` and `yaml`, and `header` is used for `header` (can be overridden) + # Default: {} + rules: + # Any struct tag type can be used. + # Support string case: `camel`, `pascal`, `kebab`, `snake`, `goCamel`, `goPascal`, `goKebab`, `goSnake`, `upper`, `lower`, `header` + db: snake + json: camel + yaml: camel + mapstructure: camel + # xml: camel # TODO + +issues: + # List of regexps of issue texts to exclude. + # + # Independently of option `exclude` we use default exclude patterns, + # it can be disabled by this option. + # To list all excluded by default patterns execute `golangci-lint run --help`. + # Default: true. + exclude-use-default: true + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + # Exclude `lll` and `gocritic` issues for long lines with `go:generate` + - source: "^//go:generate " + linters: + - lll + - gocritic + # Exclude `lll` issues for long struct tags + - source: "`[^`\n]+`" + linters: + - lll + # Exclude `lll` issues for swagger annotation + - source: "^//[\\s]*@[\\w]+\\s+" + linters: + - lll + # Exclude `lll` issues for long lines with `swag annotations` + - linters: + - lll + source: "^//[\\s]*@" + # Exclude `lll` and `funlen` for test files + - path: _test\.go + linters: + - funlen + - lll + - containedctx + - exhaustruct diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..6ef3134 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM golang:1.21.3-alpine3.18 AS builder + +COPY go.mod go.sum /opt/minireader/ +RUN go mod download + +COPY . /opt/minireader +RUN go build -o /opt/minireader/minireader /opt/minireader/cmd/minireader + +FROM alpine:3.18 +COPY --from=builder /opt/minireader/minireader /app/minireader + +CMD ["/app/minireader"] diff --git a/Dockerfile.dev b/Dockerfile.dev new file mode 100644 index 0000000..4bc4c3b --- /dev/null +++ b/Dockerfile.dev @@ -0,0 +1,9 @@ +FROM golang:1.21.3-alpine3.18 + +EXPOSE 8080 +VOLUME /opt/app +WORKDIR /opt/app + +RUN go install github.com/cosmtrek/air@latest + +CMD ["air", "-c", ".air.toml"] diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d64b3b3 --- /dev/null +++ b/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2023 Pavel Kilin (aka borodyadka) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..8b53c9e --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +.PHONY: dev +dev: + docker-compose -f docker-compose.yml -f docker-compose.dev.yml up + +.PHONY: lint +lint: + docker run --rm -v $(shell pwd):/app -w /app golangci/golangci-lint:v1.55.2 golangci-lint run ./cmd/minireader + +.PHONY: sqlc +sqlc: + docker run --rm -v $(shell pwd):/app -w /app -u $(shell id -u):$(shell id -g) sqlc/sqlc:1.23.0 generate diff --git a/README.md b/README.md new file mode 100644 index 0000000..a2ed0f1 --- /dev/null +++ b/README.md @@ -0,0 +1,11 @@ +# MiniReader + +Minireader is simple, lightweight and extensible newsfeed aggregator. + +## TODO + +* [ ] swagger docs; +* [ ] filters (by name, by feed tags, by title, by url, regex/glob patterns) and rules (autotags); +* [ ] user-defined tags; +* [ ] strip `utm_` garbage from links; + diff --git a/cmd/minireader/main.go b/cmd/minireader/main.go new file mode 100644 index 0000000..2e44080 --- /dev/null +++ b/cmd/minireader/main.go @@ -0,0 +1,25 @@ +package main + +import ( + "context" + "log/slog" + "os" + + "borodyadka.dev/borodyadka/minireader/internal/minireader" +) + +func main() { + app, err := minireader.New() + if err != nil { + slog.Error("failed to init application", "error", err) + os.Exit(1) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if err := app.Start(ctx); err != nil { + app.Logger().Error("failed to start application", "error", err) + os.Exit(1) + } +} diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 0000000..f119ec7 --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,9 @@ +version: '3' + +services: + server: + build: + context: . + dockerfile: Dockerfile.dev + volumes: + - .:/opt/app diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..70bc461 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,32 @@ +version: '3' + +services: + postgres: + image: postgres:15-alpine + environment: + POSTGRES_USER: minireader + POSTGRES_PASSWORD: minireader + POSTGRES_DB: minireader + ports: + - "127.0.0.1:5432:5432" + volumes: + - postgres:/var/lib/postgresql/data + tmpfs: + - /tmp + healthcheck: + test: pg_isready -U minireader + interval: 3s + timeout: 3s + retries: 3 + server: + image: borodyadka/minireader:dev + env_file: + - .env + ports: + - "127.0.0.1:8080:8080" + depends_on: + postgres: + condition: service_healthy + +volumes: + postgres: {} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..a728545 --- /dev/null +++ b/go.mod @@ -0,0 +1,61 @@ +module borodyadka.dev/borodyadka/minireader + +go 1.21.1 + +require ( + github.com/go-playground/validator/v10 v10.16.0 + github.com/gofiber/contrib/jwt v1.0.7 + github.com/gofiber/fiber/v2 v2.50.0 + github.com/golang-jwt/jwt/v5 v5.0.0 + github.com/jackc/pgx/v5 v5.4.3 + github.com/mmcdole/gofeed v1.2.1 + github.com/pressly/goose/v3 v3.15.1 + github.com/samber/slog-fiber v1.6.0 + github.com/samber/slog-loki v1.0.0 + github.com/sethvargo/go-envconfig v0.9.0 + github.com/stretchr/testify v1.8.2 + golang.org/x/sync v0.4.0 + gopkg.in/guregu/null.v4 v4.0.0 +) + +require ( + github.com/MicahParks/keyfunc/v2 v2.1.0 // indirect + github.com/PuerkitoBio/goquery v1.8.0 // indirect + github.com/afiskon/promtail-client v0.0.0-20190305142237-506f3f921e9c // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.2 // indirect + github.com/go-playground/locales v0.14.1 // indirect + github.com/go-playground/universal-translator v0.18.1 // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/snappy v0.0.4 // indirect + github.com/google/uuid v1.4.0 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/puddle/v2 v2.2.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/leodido/go-urn v1.2.4 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/mmcdole/goxpp v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/samber/lo v1.38.1 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.50.0 // indirect + github.com/valyala/tcplisten v1.0.0 // indirect + golang.org/x/crypto v0.13.0 // indirect + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..de8a27c --- /dev/null +++ b/go.sum @@ -0,0 +1,171 @@ +github.com/MicahParks/keyfunc/v2 v2.1.0 h1:6ZXKb9Rp6qp1bDbJefnG7cTH8yMN1IC/4nf+GVjO99k= +github.com/MicahParks/keyfunc/v2 v2.1.0/go.mod h1:rW42fi+xgLJ2FRRXAfNx9ZA8WpD4OeE/yHVMteCkw9k= +github.com/PuerkitoBio/goquery v1.8.0 h1:PJTF7AmFCFKk1N6V6jmKfrNH9tV5pNE6lZMkG0gta/U= +github.com/PuerkitoBio/goquery v1.8.0/go.mod h1:ypIiRMtY7COPGk+I/YbZLbxsxn9g5ejnI2HSMtkjZvI= +github.com/afiskon/promtail-client v0.0.0-20190305142237-506f3f921e9c h1:AMDVOKGaiqse4qiRXSzRgpC9DCNTHCx6zpzdtXXrKM4= +github.com/afiskon/promtail-client v0.0.0-20190305142237-506f3f921e9c/go.mod h1:p/7Wos+jcfrnwLqqzJMZ0s323kfVtJPW+HUvAANklVQ= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= +github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= +github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= +github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= +github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= +github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= +github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= +github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= +github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE= +github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/gofiber/contrib/jwt v1.0.7 h1:LZuCnjEq8AjiDTUjBQSd2zg3H5uDWjHxSXjo7nj9iAc= +github.com/gofiber/contrib/jwt v1.0.7/go.mod h1:fA1apg9zQlUhax+Foc0BHATCDzBsemga1Yr9X0KSvrQ= +github.com/gofiber/fiber/v2 v2.50.0 h1:ia0JaB+uw3GpNSCR5nvC5dsaxXjRU5OEu36aytx+zGw= +github.com/gofiber/fiber/v2 v2.50.0/go.mod h1:21eytvay9Is7S6z+OgPi7c7n4++tnClWmhpimVHMimw= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= +github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= +github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mmcdole/gofeed v1.2.1 h1:tPbFN+mfOLcM1kDF1x2c/N68ChbdBatkppdzf/vDe1s= +github.com/mmcdole/gofeed v1.2.1/go.mod h1:2wVInNpgmC85q16QTTuwbuKxtKkHLCDDtf0dCmnrNr4= +github.com/mmcdole/goxpp v1.1.0 h1:WwslZNF7KNAXTFuzRtn/OKZxFLJAAyOA9w82mDz2ZGI= +github.com/mmcdole/goxpp v1.1.0/go.mod h1:v+25+lT2ViuQ7mVxcncQ8ch1URund48oH+jhjiwEgS8= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pressly/goose/v3 v3.15.1 h1:dKaJ1SdLvS/+HtS8PzFT0KBEtICC1jewLXM+b3emlv8= +github.com/pressly/goose/v3 v3.15.1/go.mod h1:0E3Yg/+EwYzO6Rz2P98MlClFgIcoujbVRs575yi3iIM= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/samber/slog-fiber v1.6.0 h1:UK37csaDv9MGPKz5gdJvfL5EsjWg8g/Aph8/rm6/qKo= +github.com/samber/slog-fiber v1.6.0/go.mod h1:TEQR0iowC2vWOqrVXCTVg+h05DpkQMN9Y5mgHgJSx1g= +github.com/samber/slog-loki v1.0.0 h1:T/Dv0LlCfFEEi2PDzpxJgQeoFxAvjCnriItaBGgNT6s= +github.com/samber/slog-loki v1.0.0/go.mod h1:7FpVL6ECdmstVLDogUDE7tLdyEIjW/B/u0+ctPmYzd0= +github.com/sethvargo/go-envconfig v0.9.0 h1:Q6FQ6hVEeTECULvkJZakq3dZMeBQ3JUpcKMfPQbKMDE= +github.com/sethvargo/go-envconfig v0.9.0/go.mod h1:Iz1Gy1Sf3T64TQlJSvee81qDhf7YIlt8GMUX6yyNFs0= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.50.0 h1:H7fweIlBm0rXLs2q0XbalvJ6r0CUPFWK3/bB4N13e9M= +github.com/valyala/fasthttp v1.50.0/go.mod h1:k2zXd82h/7UZc3VOdJ2WaUqt1uZ/XpXAfE9i+HBC3lA= +github.com/valyala/tcplisten v1.0.0 h1:rBHj/Xf+E1tRGZyWIWwJDiRY0zc1Js+CV5DqwacVSA8= +github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg= +gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +lukechampine.com/uint128 v1.3.0 h1:cDdUVfRwDUDovz610ABgFD17nXD4/uDgVHl2sC3+sbo= +lukechampine.com/uint128 v1.3.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= +modernc.org/cc/v3 v3.41.0 h1:QoR1Sn3YWlmA1T4vLaKZfawdVtSiGx8H+cEojbC7v1Q= +modernc.org/cc/v3 v3.41.0/go.mod h1:Ni4zjJYJ04CDOhG7dn640WGfwBzfE0ecX8TyMB0Fv0Y= +modernc.org/ccgo/v3 v3.16.15 h1:KbDR3ZAVU+wiLyMESPtbtE/Add4elztFyfsWoNTgxS0= +modernc.org/ccgo/v3 v3.16.15/go.mod h1:yT7B+/E2m43tmMOT51GMoM98/MtHIcQQSleGnddkUNI= +modernc.org/libc v1.24.1 h1:uvJSeCKL/AgzBo2yYIPPTy82v21KgGnizcGYfBHaNuM= +modernc.org/libc v1.24.1/go.mod h1:FmfO1RLrU3MHJfyi9eYYmZBfi/R+tqZ6+hQ3yQQUkak= +modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4= +modernc.org/mathutil v1.6.0/go.mod h1:Ui5Q9q1TR2gFm0AQRqQUaBWFLAhQpCwNcuhBOSedWPo= +modernc.org/memory v1.7.2 h1:Klh90S215mmH8c9gO98QxQFsY+W451E8AnzjoE2ee1E= +modernc.org/memory v1.7.2/go.mod h1:NO4NVCQy0N7ln+T9ngWqOQfi7ley4vpwvARR+Hjw95E= +modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= +modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= +modernc.org/sqlite v1.26.0 h1:SocQdLRSYlA8W99V8YH0NES75thx19d9sB/aFc4R8Lw= +modernc.org/sqlite v1.26.0/go.mod h1:FL3pVXie73rg3Rii6V/u5BoHlSoyeZeIgKZEgHARyCU= +modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA= +modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/minireader/app.go b/internal/minireader/app.go new file mode 100644 index 0000000..1e50eca --- /dev/null +++ b/internal/minireader/app.go @@ -0,0 +1,152 @@ +package minireader + +import ( + "context" + "database/sql" + "fmt" + "log/slog" + "os" + "os/signal" + "syscall" + "time" + + _ "github.com/jackc/pgx/v5/stdlib" + "github.com/pressly/goose/v3" + "github.com/sethvargo/go-envconfig" + "golang.org/x/sync/errgroup" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/config" + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" + "borodyadka.dev/borodyadka/minireader/internal/minireader/fetcher" + "borodyadka.dev/borodyadka/minireader/internal/minireader/fetcher/rss" + "borodyadka.dev/borodyadka/minireader/internal/minireader/reader" + "borodyadka.dev/borodyadka/minireader/internal/minireader/repositories" + "borodyadka.dev/borodyadka/minireader/internal/minireader/transport/http" + "borodyadka.dev/borodyadka/minireader/migrations" +) + +type App struct { + logger *slog.Logger + config config.Config +} + +func New() (*App, error) { + config := config.Config{} + if err := envconfig.Process(context.Background(), &config); err != nil { + return nil, err + } + + logger, err := newLogger(config.Log) + if err != nil { + return nil, err + } + + return &App{ + config: config, + logger: logger, + }, nil +} + +func (app *App) Start(ctx context.Context) error { + err := app.migrate( + ctx, + app.logger.With("module", "migrator"), + app.config.Postgres.DSN, + ) + if err != nil { + return fmt.Errorf("failed to apply migrations: %w", err) + } + + db, err := repositories.NewDatabase(ctx, app.config.Postgres.DSN) + if err != nil { + return err + } + + fetcher := fetcher.New( + fetcher.WithConfig(fetcher.Config{ + CheckInterval: app.config.Fetcher.Interval, + Fetchers: map[dto.Provider]fetcher.Fetcher{ + dto.ProviderRSS: rss.New(rss.Config{ + HTTPTimeout: 5 * time.Minute, + }), + }, + }), + fetcher.WithRepositories(fetcher.Repositories{ + Feed: repositories.NewFeedRepository(), + Item: repositories.NewItemRepository(), + }), + fetcher.WithLogger(app.logger.With("module", "fetcher")), + ) + + reader := reader.New( + reader.WithRepositories(reader.Repositories{ + Feed: repositories.NewFeedRepository(), + Subscription: repositories.NewSubscriptionRepository(), + Item: repositories.NewItemRepository(), + User: repositories.NewUserRepository(), + }), + reader.WithFetcher(fetcher), + reader.WithLogger(app.logger.With("module", "reader")), + ) + + server := http.New( + http.WithConfig(app.config.HTTP), + http.WithServices(http.Services{ + Reader: reader, + }), + http.WithSecurity(app.config.Security), + http.WithLogger(app.logger.With("module", "http")), + ) + + server.Inject(repositories.ConnKey, db) + + eg, ctx := errgroup.WithContext(ctx) + eg.Go(func() error { + return server.Start() + }) + + eg.Go(func() error { + fctx := context.WithValue(ctx, repositories.ConnKey, db) + return fetcher.Start(fctx) + }) + + eg.Go(func() error { + sigHandler := make(chan os.Signal, 1) + signal.Notify(sigHandler, syscall.SIGTERM, syscall.SIGINT) + select { + case <-ctx.Done(): + case <-sigHandler: + } + + _ = server.Shutdown() + return nil + }) + + return eg.Wait() +} + +func (app *App) migrate(ctx context.Context, logger *slog.Logger, dsn string) error { + goose.SetBaseFS(migrations.Migrations) + goose.SetLogger(&gooseWrapper{logger: logger}) + + db, err := sql.Open("pgx", dsn) + if err != nil { + return err + } + defer db.Close() + + goose.SetTableName("migrations") + if err := goose.SetDialect("postgres"); err != nil { + return err + } + + if err := goose.UpContext(ctx, db, "."); err != nil { + return err + } + + return nil +} + +func (app *App) Logger() *slog.Logger { + return app.logger +} diff --git a/internal/minireader/config/config.go b/internal/minireader/config/config.go new file mode 100644 index 0000000..d89799d --- /dev/null +++ b/internal/minireader/config/config.go @@ -0,0 +1,43 @@ +package config + +import ( + "time" +) + +type Config struct { + Postgres PostgresConfig `env:",prefix=POSTGRES_"` + HTTP HTTP `env:",prefix=HTTP_"` + Log LogConfig `env:",prefix=LOG_"` + Security Security `env:",prefix=SECURITY_"` + Fetcher Fetcher `env:",prefix=FETCHER_"` +} + +type Security struct { + JWKSURL string `env:"JWKS_URL" validate:"uri,required"` + UserInfoURL string `env:"USER_INFO_URL" validate:"uri,optional"` +} + +type HTTP struct { + Listen string `env:"LISTEN,default=0.0.0.0:8080"` + APIPrefix string `env:"API_PREFIX"` + AccessLog bool `env:"ENABLE_ACCESS_LOG,default=false"` + EnableGUI bool `env:"ENABLE_GUI,default=true"` // TODO +} + +type PostgresConfig struct { + DSN string `env:"DSN" validate:"uri"` +} + +type Fetcher struct { + Interval time.Duration `env:"INTERVAL,default=1m"` +} + +type LogConfig struct { + // possible values: DEBUG INFO WARN ERROR + Level string `env:"LEVEL,default=INFO" validate:"oneof=DEBUG INFO WARN ERROR"` + // possible values: + // * "none" + // * "stdout" + // * "loki+http://localhost:3100" + Output string `env:"OUTPUT,default=stdout"` +} diff --git a/internal/minireader/dto/errors.go b/internal/minireader/dto/errors.go new file mode 100644 index 0000000..552b414 --- /dev/null +++ b/internal/minireader/dto/errors.go @@ -0,0 +1,9 @@ +package dto + +import ( + "errors" +) + +var ( + ErrNotFound = errors.New("not_found") +) diff --git a/internal/minireader/dto/reader.go b/internal/minireader/dto/reader.go new file mode 100644 index 0000000..e73ffad --- /dev/null +++ b/internal/minireader/dto/reader.go @@ -0,0 +1,77 @@ +package dto + +import ( + "fmt" + "time" + + "github.com/google/uuid" + "gopkg.in/guregu/null.v4" +) + +type Provider string + +func (e *Provider) Scan(src interface{}) error { + switch s := src.(type) { + case []byte: + *e = Provider(s) + case string: + *e = Provider(s) + default: + return fmt.Errorf("unsupported scan type for Provider: %T", src) + } + return nil +} + +const ( + ProviderRSS Provider = "rss" +) + +type FeedState struct { + LastFetchedAt time.Time `json:"-"` + FetchInterval time.Duration `json:"-"` + LastError null.String `json:"-"` + ErrorsCount int `json:"-"` +} + +type Feed struct { + ID uuid.UUID `json:"id"` + Title string `json:"title"` + Provider Provider `json:"provider"` + Source string `json:"source"` + + UnreadCount int `json:"unread"` + + State FeedState `json:"-"` +} + +type CreateFeed struct { + Title null.String `json:"title" validate:"optional"` + Provider Provider `json:"provider" validate:"required,oneof=rss"` + Source string `json:"source" validate:"required"` +} + +type UpdateFeed struct { + ID uuid.UUID + Title null.String + FetchIntervalMultiplier float64 + LastFetchedAt time.Time +} + +type UpdateFeedError struct { + ID uuid.UUID + Error string + LastFetchedAt time.Time +} + +type Item struct { + ID uuid.UUID `json:"id"` + UID string `json:"uid"` + CreatedAt time.Time `json:"createdAt"` + FetchedAt time.Time `json:"fetchedAt"` + Title string `json:"title"` + Content string `json:"content"` + Link string `json:"link"` + Author string `json:"author"` + //Tags []string `json:"tags"` // TODO: user-defined tags + //Attachments []any `json:"attachments"` // TODO +} diff --git a/internal/minireader/dto/site.go b/internal/minireader/dto/site.go new file mode 100644 index 0000000..1263892 --- /dev/null +++ b/internal/minireader/dto/site.go @@ -0,0 +1,7 @@ +package dto + +type WebsiteInfo struct { + Title string + Favicon []byte + Feeds []string +} diff --git a/internal/minireader/dto/user.go b/internal/minireader/dto/user.go new file mode 100644 index 0000000..bf58cf5 --- /dev/null +++ b/internal/minireader/dto/user.go @@ -0,0 +1,5 @@ +package dto + +type User struct { + ID string `json:"id,omitempty"` +} diff --git a/internal/minireader/fetcher/fetcher.go b/internal/minireader/fetcher/fetcher.go new file mode 100644 index 0000000..1bd1a31 --- /dev/null +++ b/internal/minireader/fetcher/fetcher.go @@ -0,0 +1,148 @@ +package fetcher + +import ( + "context" + "log/slog" + "time" + + "github.com/google/uuid" + "gopkg.in/guregu/null.v4" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" +) + +type Fetcher interface { + Fetch(ctx context.Context, source string) (dto.Feed, []dto.Item, error) +} + +type feedRepository interface { + ListOutdatedFeeds(ctx context.Context) ([]dto.Feed, error) + UpdateFeed(ctx context.Context, input *dto.UpdateFeed) error + SaveFeedError(ctx context.Context, input *dto.UpdateFeedError) error +} + +type itemRepository interface { + UpsertItems(ctx context.Context, feed uuid.UUID, items []dto.Item) (inserted int, err error) +} + +type Config struct { + CheckInterval time.Duration + + Fetchers map[dto.Provider]Fetcher +} + +type FetchScheduler struct { + checkInterval time.Duration + logger *slog.Logger + + feeds feedRepository + items itemRepository + + fetchers map[dto.Provider]Fetcher +} + +func New(opts ...Option) *FetchScheduler { + fetcher := &FetchScheduler{ + fetchers: make(map[dto.Provider]Fetcher), + } + + for _, opt := range opts { + opt(fetcher) + } + + return fetcher +} + +func (s *FetchScheduler) Start(ctx context.Context) error { + t := time.NewTicker(s.checkInterval) + defer t.Stop() + + for { + select { + case <-ctx.Done(): + return nil + case <-t.C: + if err := s.refreshFeeds(ctx); err != nil { + s.logger.Error("failed to refresh feeds", "error", err) + } + } + } +} + +func (s *FetchScheduler) refreshFeeds(ctx context.Context) error { + logger := s.logger.With("method", "refreshFeeds") + + // TODO: make it parallel with N (configurable) workers + feeds, err := s.feeds.ListOutdatedFeeds(ctx) + if err != nil { + return err + } + + for _, feed := range feeds { + err := s.refreshFeed(ctx, feed) + if err != nil { + logger.Error("failed to refresh feed", "feed", feed.ID, "error", err) + } + } + + return nil +} + +func (s *FetchScheduler) refreshFeed(ctx context.Context, feed dto.Feed) error { + logger := s.logger.With("method", "refreshFeeds") + + fetcher := s.fetchers[feed.Provider] + if fetcher == nil { + logger.Error("fetcher not exists", "feed", feed.ID, "provider", feed.Provider, "source", feed.Source) + return nil + } + now := time.Now() + + info, items, err := fetcher.Fetch(ctx, feed.Source) + if err != nil { + return s.feeds.SaveFeedError(ctx, &dto.UpdateFeedError{ + ID: feed.ID, + Error: err.Error(), + LastFetchedAt: now, + }) + } + + inserted, err := s.items.UpsertItems(ctx, feed.ID, items) + if err != nil { + return err + } + + // find out how often we should refresh this feed: + // * if ratio of new items is greater than 90% then shorten interval in half + // * if ratio of new items is less than 10% then double interval + multiplier := 1.0 + ratio := float64(inserted) / float64(len(items)) + if ratio >= 0.9 { + multiplier = 0.5 + } else if ratio <= 0.1 { + multiplier = 2 + } + + logger.Debug("items upserted", + "feed", feed.ID, + "provider", feed.Provider, + "source", feed.Source, + "ratio", ratio, + "items", inserted, + "total", len(items)) + + if err := s.feeds.UpdateFeed(ctx, &dto.UpdateFeed{ + ID: feed.ID, + Title: null.StringFrom(info.Title), + FetchIntervalMultiplier: multiplier, + LastFetchedAt: now, + }); err != nil { + return err + } + + return nil +} + +func (s *FetchScheduler) ForceRefresh(ctx context.Context, feed dto.Feed) error { + return s.refreshFeed(ctx, feed) +} diff --git a/internal/minireader/fetcher/options.go b/internal/minireader/fetcher/options.go new file mode 100644 index 0000000..b61530a --- /dev/null +++ b/internal/minireader/fetcher/options.go @@ -0,0 +1,38 @@ +package fetcher + +import ( + "log/slog" + + "borodyadka.dev/borodyadka/minireader/internal/pkg/validator" +) + +type Option func(fetcher *FetchScheduler) + +type Repositories struct { + Feed feedRepository `validate:"required"` + Item itemRepository `validate:"required"` +} + +func WithRepositories(repo Repositories) Option { + return func(fetcher *FetchScheduler) { + if err := validator.Struct(repo); err != nil { + panic("one or more repositories passed into FetchScheduler instance is invalid or not defined") + } + + fetcher.feeds = repo.Feed + fetcher.items = repo.Item + } +} + +func WithConfig(config Config) Option { + return func(fetcher *FetchScheduler) { + fetcher.checkInterval = config.CheckInterval + fetcher.fetchers = config.Fetchers + } +} + +func WithLogger(logger *slog.Logger) Option { + return func(fetcher *FetchScheduler) { + fetcher.logger = logger + } +} diff --git a/internal/minireader/fetcher/rss/fetcher.go b/internal/minireader/fetcher/rss/fetcher.go new file mode 100644 index 0000000..a1dc12b --- /dev/null +++ b/internal/minireader/fetcher/rss/fetcher.go @@ -0,0 +1,64 @@ +package rss + +import ( + "context" + "net/http" + "time" + + "github.com/mmcdole/gofeed" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" + "borodyadka.dev/borodyadka/minireader/internal/pkg/utils" +) + +type Config struct { + HTTPTimeout time.Duration +} + +type Fetcher struct { + config Config +} + +func New(config Config) *Fetcher { + return &Fetcher{config: config} +} + +func (f *Fetcher) Fetch(ctx context.Context, source string) (dto.Feed, []dto.Item, error) { + cl := http.Client{Timeout: f.config.HTTPTimeout} + + req, err := http.NewRequest("GET", source, nil) + if err != nil { + return dto.Feed{}, nil, err + } + + resp, err := cl.Do(req) + if err != nil { + return dto.Feed{}, nil, err + } + defer resp.Body.Close() + + feed, err := gofeed.NewParser().Parse(resp.Body) + if err != nil { + return dto.Feed{}, nil, err + } + + now := time.Now() + items := make([]dto.Item, 0, len(feed.Items)) + for _, item := range feed.Items { + items = append(items, dto.Item{ + UID: item.GUID, + CreatedAt: utils.DerefOrDefault(item.PublishedParsed, now), + FetchedAt: now, + Title: item.Title, + Content: item.Content, + Link: item.Link, + // TODO: rest of fields + }) + } + + return dto.Feed{ + Title: feed.Title, + Provider: dto.ProviderRSS, + Source: source, + }, items, nil +} diff --git a/internal/minireader/logger.go b/internal/minireader/logger.go new file mode 100644 index 0000000..742b38a --- /dev/null +++ b/internal/minireader/logger.go @@ -0,0 +1,83 @@ +package minireader + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/url" + "os" + "strings" + "time" + + "github.com/samber/slog-loki" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/config" +) + +func parseLoggerOutput(dsn string) (typ string, host string, err error) { + switch dsn { + case "none", "stdout": + return dsn, "", nil + } + + u, err := url.Parse(dsn) + if err != nil { + return "", "", err + } + + if strings.Contains(u.Scheme, "loki") { + u.Scheme = strings.NewReplacer("loki", "", "+", "").Replace(u.Scheme) + return "loki", u.String(), nil + } + + return "", "", errors.New("cannot parse logger dsn") +} + +func newLogger(config config.LogConfig) (*slog.Logger, error) { + var level slog.Level + if err := level.UnmarshalText([]byte(config.Level)); err != nil { + return nil, err + } + + typ, host, err := parseLoggerOutput(config.Output) + if err != nil { + return nil, err + } + + switch typ { + case "none": + return slog.New(discardHandler{}), nil + case "loki": + handler := slogloki.Option{ + Level: level, + Endpoint: host, + BatchWait: time.Second, + BatchEntriesNumber: 32, + }.NewLokiHandler() + return slog.New(handler), nil + } + + return slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{ + Level: level, + })), nil +} + +type discardHandler struct{} + +func (discardHandler) Enabled(context.Context, slog.Level) bool { return false } +func (discardHandler) Handle(context.Context, slog.Record) error { return nil } +func (d discardHandler) WithAttrs([]slog.Attr) slog.Handler { return d } +func (d discardHandler) WithGroup(string) slog.Handler { return d } + +type gooseWrapper struct { + logger *slog.Logger +} + +func (l *gooseWrapper) Printf(msg string, args ...any) { + l.logger.Info(fmt.Sprintf(strings.TrimSpace(msg), args...)) +} + +func (l *gooseWrapper) Fatalf(msg string, args ...any) { + l.logger.Error(fmt.Sprintf(strings.TrimSpace(msg), args...)) +} diff --git a/internal/minireader/metadata/urlinfo.go b/internal/minireader/metadata/urlinfo.go new file mode 100644 index 0000000..3772d86 --- /dev/null +++ b/internal/minireader/metadata/urlinfo.go @@ -0,0 +1,46 @@ +package metadata + +import ( + "context" + "io" + "net/http" + "net/url" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" +) + +func fetch(ctx context.Context, cl *http.Client, page string) (io.ReadCloser, error) { + req, err := http.NewRequestWithContext(ctx, "GET", page, nil) + if err != nil { + return nil, err + } + + resp, err := cl.Do(req) + if err != nil { + return nil, err + } + + return resp.Body, nil +} + +func FetchPageInfo(ctx context.Context, page string) (dto.WebsiteInfo, error) { + info := dto.WebsiteInfo{} + + u, err := url.Parse(page) + if err != nil { + return info, err + } + + cl := &http.Client{} + for { + body, err := fetch(ctx, cl, u.String()) + if err != nil { + return info, err + } + _ = body + // TODO: handle http status codes (404, 500, 502) + // TODO: if no info on this page go to parent dir ant try again (maybe add some sleep time?) + } + + return info, nil +} diff --git a/internal/minireader/reader/options.go b/internal/minireader/reader/options.go new file mode 100644 index 0000000..a0bab2e --- /dev/null +++ b/internal/minireader/reader/options.go @@ -0,0 +1,41 @@ +package reader + +import ( + "log/slog" + + "borodyadka.dev/borodyadka/minireader/internal/pkg/validator" +) + +type Option func(reader *Reader) + +type Repositories struct { + Feed feedRepository `validate:"required"` + Subscription subscriptionRepository `validate:"required"` + Item itemRepository `validate:"required"` + User userRepository `validate:"required"` +} + +func WithRepositories(repo Repositories) Option { + return func(reader *Reader) { + if err := validator.Struct(repo); err != nil { + panic("one or more repositories passed into Reader instance is invalid or not defined") + } + + reader.feeds = repo.Feed + reader.subscriptions = repo.Subscription + reader.items = repo.Item + reader.users = repo.User + } +} + +func WithFetcher(fetcher fetcher) Option { + return func(reader *Reader) { + reader.fetcher = fetcher + } +} + +func WithLogger(logger *slog.Logger) Option { + return func(reader *Reader) { + reader.logger = logger + } +} diff --git a/internal/minireader/reader/reader.go b/internal/minireader/reader/reader.go new file mode 100644 index 0000000..d789c7b --- /dev/null +++ b/internal/minireader/reader/reader.go @@ -0,0 +1,157 @@ +package reader + +import ( + "context" + "log/slog" + + "github.com/google/uuid" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" + "borodyadka.dev/borodyadka/minireader/internal/minireader/repositories" + "borodyadka.dev/borodyadka/minireader/internal/pkg/validator" +) + +type feedRepository interface { + ListAllUserFeeds(ctx context.Context, user *dto.User) ([]dto.Feed, int, error) + FindOrCreate(ctx context.Context, input *dto.CreateFeed) (dto.Feed, bool, error) + Delete(ctx context.Context, id uuid.UUID) (bool, error) +} + +type itemRepository interface{} + +type subscriptionRepository interface { + Create(ctx context.Context, user *dto.User, feed uuid.UUID) error + Update(ctx context.Context, user *dto.User, feed uuid.UUID, input dto.UpdateFeed) (bool, error) + Delete(ctx context.Context, user *dto.User) (bool, error) + CountByFeed(ctx context.Context, feed uuid.UUID) (int, error) +} + +type userRepository interface { + EnsureUser(ctx context.Context, user *dto.User) error +} + +type fetcher interface { + ForceRefresh(ctx context.Context, feed dto.Feed) error +} + +type Reader struct { + feeds feedRepository + subscriptions subscriptionRepository + items itemRepository + users userRepository + fetcher fetcher + + logger *slog.Logger +} + +func New(opts ...Option) *Reader { + reader := &Reader{} + + for _, opt := range opts { + opt(reader) + } + + return reader +} + +func (r *Reader) EnsureUser(ctx context.Context, user *dto.User) error { + return r.users.EnsureUser(ctx, user) +} + +func (r *Reader) ListAllUserFeeds(ctx context.Context, user *dto.User) ([]dto.Feed, int, error) { + return r.feeds.ListAllUserFeeds(ctx, user) +} + +func (r *Reader) FindOrCreateFeed(ctx context.Context, input *dto.CreateFeed) (dto.Feed, error) { + logger := r.logger.With("method", "FindOrCreateFeed") + var feed dto.Feed + + err := repositories.Transactional(ctx, func(ctx context.Context) (err error) { + var created bool + feed, created, err = r.feeds.FindOrCreate(ctx, input) + if err != nil { + logger.Error("find or create feed failed", "error", err) + return err + } + + if created { + logger.Debug("feed created", "feed", feed.ID) + err := r.fetcher.ForceRefresh(ctx, feed) + if err != nil { + return err + } + } + + return nil + }) + + return feed, err +} + +// CreateSubscription will create a new subscription to feed (if feed not exists, it will be created) +func (r *Reader) CreateSubscription(ctx context.Context, user *dto.User, input *dto.CreateFeed) (dto.Feed, error) { + logger := r.logger.With("method", "CreateSubscription") + var feed dto.Feed + + if err := validator.Struct(input); err != nil { + return feed, err + } + + err := repositories.Transactional(ctx, func(ctx context.Context) (err error) { + feed, err = r.FindOrCreateFeed(ctx, input) + if err != nil { + return err + } + + logger.Debug("create subscription", "feed", feed.ID) + return r.subscriptions.Create(ctx, user, feed.ID) + }) + if err != nil { + return feed, err + } + + return feed, nil +} + +// UpdateSubscription will update subscription info (e.g. change title) +func (r *Reader) UpdateSubscription(ctx context.Context, user *dto.User, feed uuid.UUID, input dto.UpdateFeed) error { + logger := r.logger.With("method", "UpdateSubscription") + + updated, err := r.subscriptions.Update(ctx, user, feed, input) + if !updated { + return dto.ErrNotFound + } + + logger.Debug("subscription updated", "feed", feed) + + return err +} + +// DeleteSubscription will delete user subscription and remove feed if no active subscriptions left +func (r *Reader) DeleteSubscription(ctx context.Context, user *dto.User, feed uuid.UUID) error { + logger := r.logger.With("method", "DeleteSubscription") + + return repositories.Transactional(ctx, func(ctx context.Context) error { + deleted, err := r.subscriptions.Delete(ctx, user) + if err != nil { + return err + } + if !deleted { + return dto.ErrNotFound + } + + logger.Debug("subscription removed", "feed", feed) + + subs, err := r.subscriptions.CountByFeed(ctx, feed) + if err != nil { + logger.Error("failed to count feed subscriptions", "error", err) + } else if subs == 0 { + _, err = r.feeds.Delete(ctx, feed) + if err != nil { + logger.Error("failed to delete feed", "error", err) + } + } + + return nil + }) +} diff --git a/internal/minireader/repositories/database.go b/internal/minireader/repositories/database.go new file mode 100644 index 0000000..3e2f003 --- /dev/null +++ b/internal/minireader/repositories/database.go @@ -0,0 +1,77 @@ +package repositories + +import ( + "context" + "errors" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" +) + +type ctxkey string + +const ( + ConnKey ctxkey = "pg_conn" +) + +var ( + ErrCannotStartTransaction = errors.New("cannot start transaction (method not supported)") +) + +type txbeginner interface { + Begin(ctx context.Context) (pgx.Tx, error) +} + +type Conn interface { + Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...any) pgx.Row + Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) + SendBatch(context.Context, *pgx.Batch) pgx.BatchResults +} + +func NewDatabase(ctx context.Context, dsn string) (*pgxpool.Pool, error) { + cfg, err := pgxpool.ParseConfig(dsn) + if err != nil { + return nil, err + } + // TODO: logger + + return pgxpool.NewWithConfig(ctx, cfg) +} + +func ConnFromContext(ctx context.Context) Conn { + return ctx.Value(ConnKey).(Conn) +} + +func WithConn(ctx context.Context, conn Conn) context.Context { + return context.WithValue(ctx, ConnKey, conn) +} + +type TransactionalFn func(ctx context.Context) error + +func Transactional(ctx context.Context, fn TransactionalFn) error { + conn := ConnFromContext(ctx) + + // already in transaction + if _, ok := conn.(*pgxpool.Tx); ok { + return fn(ctx) + } + + b, ok := conn.(txbeginner) + if !ok { + return ErrCannotStartTransaction + } + + tx, err := b.Begin(ctx) + if err != nil { + return err + } + defer tx.Rollback(ctx) + + if err := fn(WithConn(ctx, tx)); err != nil { + return err + } + + return tx.Commit(ctx) +} diff --git a/internal/minireader/repositories/feed.go b/internal/minireader/repositories/feed.go new file mode 100644 index 0000000..6d094d6 --- /dev/null +++ b/internal/minireader/repositories/feed.go @@ -0,0 +1,92 @@ +package repositories + +import ( + "context" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" + "borodyadka.dev/borodyadka/minireader/internal/minireader/repositories/sql" + "borodyadka.dev/borodyadka/minireader/internal/pkg/utils" +) + +type FeedRepository struct { +} + +func NewFeedRepository() *FeedRepository { + return &FeedRepository{} +} + +func (r *FeedRepository) FindOrCreate(ctx context.Context, input *dto.CreateFeed) (dto.Feed, bool, error) { + result, err := sql.New(ConnFromContext(ctx)).FindOrCreateFeed(ctx, sql.FindOrCreateFeedParams{ + Title: input.Title, + Provider: input.Provider, + Source: input.Source, + }) + if err != nil { + return dto.Feed{}, false, err + } + + return dto.Feed{ + ID: result.ID, + Title: result.Title, + Provider: input.Provider, + Source: input.Source, + }, result.Created, nil +} + +func (r *FeedRepository) ListOutdatedFeeds(ctx context.Context) ([]dto.Feed, error) { + items, err := sql.New(ConnFromContext(ctx)).ListOutdatedFeeds(ctx) + if err != nil { + return nil, err + } + + return utils.MapSlice(items, func(t sql.ListOutdatedFeedsRow) dto.Feed { + return dto.Feed{ + ID: t.ID, + Provider: t.Provider, + Source: t.Source, + } + }), nil +} + +func (r *FeedRepository) ListAllUserFeeds(ctx context.Context, user *dto.User) ([]dto.Feed, int, error) { + items, err := sql.New(ConnFromContext(ctx)).ListAllUserFeeds(ctx, user.ID) + if err != nil { + return nil, 0, err + } + + return utils.MapSlice(items, func(t sql.ListAllUserFeedsRow) dto.Feed { + return dto.Feed{ + ID: t.ID, + Title: t.Title, + Provider: t.Provider, + Source: t.Source, + UnreadCount: int(t.Unread), + State: dto.FeedState{}, + } + }), len(items), nil +} + +func (r *FeedRepository) UpdateFeed(ctx context.Context, input *dto.UpdateFeed) error { + return sql.New(ConnFromContext(ctx)).UpdateFeed(ctx, sql.UpdateFeedParams{ + ID: input.ID, + Title: input.Title.String, + IntervalMultiplier: input.FetchIntervalMultiplier, + LastFetchedAt: pgtype.Timestamptz{Time: input.LastFetchedAt, Valid: true}, + }) +} + +func (r *FeedRepository) SaveFeedError(ctx context.Context, input *dto.UpdateFeedError) error { + return sql.New(ConnFromContext(ctx)).SaveFeedError(ctx, sql.SaveFeedErrorParams{ + ID: input.ID, + LastError: input.Error, + LastFetchedAt: pgtype.Timestamptz{Time: input.LastFetchedAt, Valid: true}, + }) +} + +func (r *FeedRepository) Delete(ctx context.Context, id uuid.UUID) (bool, error) { + tag, err := sql.New(ConnFromContext(ctx)).DeleteFeed(ctx, id) + return tag.RowsAffected() > 0, err +} diff --git a/internal/minireader/repositories/item.go b/internal/minireader/repositories/item.go new file mode 100644 index 0000000..1ddf6af --- /dev/null +++ b/internal/minireader/repositories/item.go @@ -0,0 +1,53 @@ +package repositories + +import ( + "context" + "sync/atomic" + + "github.com/google/uuid" + "github.com/jackc/pgx/v5/pgtype" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" + "borodyadka.dev/borodyadka/minireader/internal/minireader/repositories/sql" + "borodyadka.dev/borodyadka/minireader/internal/pkg/utils" +) + +type ItemRepository struct { +} + +func NewItemRepository() *ItemRepository { + return &ItemRepository{} +} + +func (r *ItemRepository) UpsertItems(ctx context.Context, feed uuid.UUID, items []dto.Item) (int, error) { + input := utils.MapSlice(items, func(t dto.Item) sql.UpsertFeedItemBatchParams { + return sql.UpsertFeedItemBatchParams{ + FeedID: feed, + CreatedAt: pgtype.Timestamptz{Time: t.CreatedAt, Valid: true}, + FetchedAt: pgtype.Timestamptz{Time: t.FetchedAt, Valid: true}, + UID: t.UID, + Title: t.Title, + Content: t.Content, + Link: t.Link, + Author: t.Author, + } + }) + + res := sql.New(ConnFromContext(ctx)).UpsertFeedItemBatch(ctx, input) + + var inserted int64 + res.QueryRow(func(_ int, created bool, err error) { + if created { + atomic.AddInt64(&inserted, 1) + } + if err != nil { + // TODO: handle error + } + }) + + if err := res.Close(); err != nil { + return 0, err + } + + return int(inserted), nil +} diff --git a/internal/minireader/repositories/sql/.gitignore b/internal/minireader/repositories/sql/.gitignore new file mode 100644 index 0000000..e796b66 --- /dev/null +++ b/internal/minireader/repositories/sql/.gitignore @@ -0,0 +1 @@ +*.go diff --git a/internal/minireader/repositories/sql/query.sql b/internal/minireader/repositories/sql/query.sql new file mode 100644 index 0000000..64c6963 --- /dev/null +++ b/internal/minireader/repositories/sql/query.sql @@ -0,0 +1,63 @@ +-- name: ListOutdatedFeeds :many +SELECT "id", "provider", "source" +FROM "feeds" +WHERE NOW() > "next_fetch_at" +ORDER BY "next_fetch_at" +LIMIT 1000; + +-- name: ListAllUserFeeds :many +WITH "counters" AS (SELECT fi."feed_id", si."user_id", COUNT(*) AS "unread" + FROM "feed_items" AS fi + LEFT JOIN "subscription_items" AS si + ON (fi."feed_id" = "si".feed_id AND fi."id" = si."item_id") + WHERE si."is_readed" = FALSE + OR si."is_readed" IS NULL + GROUP BY fi."feed_id", si."user_id") +SELECT f."id", + COALESCE(s."title", f."title") AS "title", + f."provider", + f."source", + COALESCE(c."unread", 0) AS "unread" +FROM "subscriptions" AS s + JOIN "feeds" AS f ON (s."feed_id" = f."id") + LEFT JOIN "counters" AS c ON (c."feed_id" = f."id" AND (s."user_id" = c."user_id" OR c."user_id" IS NULL)) +WHERE s."user_id" = @user_id; + +-- name: FindOrCreateFeed :one +INSERT INTO "feeds" ("title", "provider", "source") +VALUES (COALESCE(@title, 'unnamed feed'), @provider, @source) +ON CONFLICT ("source") DO NOTHING +RETURNING (xmax = 0)::BOOLEAN AS "created", "id", "title", "last_fetched_at", "fetch_interval", "last_error", "errors_count"; + +-- name: UpdateFeed :exec +UPDATE "feeds" +SET "title" = COALESCE(NULLIF(@title, ''), "title"), + "fetch_interval" = LEAST(GREATEST("fetch_interval" * @interval_multiplier::FLOAT, '1 minute'::interval), '12 hours'::interval), + "last_fetched_at" = @last_fetched_at +WHERE "id" = @id; + +-- name: SaveFeedError :exec +UPDATE "feeds" +SET "last_error" = @last_error::TEXT, + "errors_count" = "errors_count" + 1, + "last_fetched_at" = @last_fetched_at +WHERE "id" = @id; + +-- name: DeleteFeed :execresult +DELETE +FROM "feeds" +WHERE "id" = @id; + +-- name: UpsertFeedItemBatch :batchone +INSERT INTO "feed_items" ("feed_id", "created_at", "fetched_at", "uid", "title", "content", "link", "author", "tags") +VALUES (@feed_id::UUID, @created_at::TIMESTAMPTZ, @fetched_at::TIMESTAMPTZ, @uid::TEXT, @title::TEXT, @content::TEXT, + @link::TEXT, @author::TEXT, '{}'::TEXT[]) +ON CONFLICT ("feed_id", "uid") DO UPDATE SET "title" = @title::TEXT, + "content" = @content::TEXT, + "link" = @link::TEXT +RETURNING (xmax = 0)::BOOLEAN AS "created"; + +-- name: CreateSubscription :exec +INSERT INTO "subscriptions" ("user_id", "feed_id") +VALUES (@user_id::TEXT, @feed_id::UUID) +ON CONFLICT ("user_id", "feed_id") DO NOTHING; diff --git a/internal/minireader/repositories/subscription.go b/internal/minireader/repositories/subscription.go new file mode 100644 index 0000000..c3e2bd2 --- /dev/null +++ b/internal/minireader/repositories/subscription.go @@ -0,0 +1,36 @@ +package repositories + +import ( + "context" + + "github.com/google/uuid" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" + "borodyadka.dev/borodyadka/minireader/internal/minireader/repositories/sql" +) + +type SubscriptionRepository struct { +} + +func NewSubscriptionRepository() *SubscriptionRepository { + return &SubscriptionRepository{} +} + +func (r *SubscriptionRepository) Create(ctx context.Context, user *dto.User, feed uuid.UUID) error { + return sql.New(ConnFromContext(ctx)).CreateSubscription(ctx, sql.CreateSubscriptionParams{ + UserID: user.ID, + FeedID: feed, + }) +} + +func (r *SubscriptionRepository) Update(ctx context.Context, user *dto.User, feed uuid.UUID, input dto.UpdateFeed) (bool, error) { + return false, nil +} + +func (r *SubscriptionRepository) Delete(ctx context.Context, user *dto.User) (bool, error) { + return false, nil +} + +func (r *SubscriptionRepository) CountByFeed(ctx context.Context, feed uuid.UUID) (int, error) { + return 0, nil +} diff --git a/internal/minireader/repositories/user.go b/internal/minireader/repositories/user.go new file mode 100644 index 0000000..6648561 --- /dev/null +++ b/internal/minireader/repositories/user.go @@ -0,0 +1,19 @@ +package repositories + +import ( + "context" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" +) + +type UserRepository struct { +} + +func NewUserRepository() *UserRepository { + return &UserRepository{} +} + +func (r *UserRepository) EnsureUser(ctx context.Context, user *dto.User) error { + _, err := ConnFromContext(ctx).Exec(ctx, `INSERT INTO "users" ("id") VALUES ($1) ON CONFLICT DO NOTHING`, user.ID) + return err +} diff --git a/internal/minireader/transport/http/auth.go b/internal/minireader/transport/http/auth.go new file mode 100644 index 0000000..389811d --- /dev/null +++ b/internal/minireader/transport/http/auth.go @@ -0,0 +1,90 @@ +package http + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + + jwtware "github.com/gofiber/contrib/jwt" + "github.com/gofiber/fiber/v2" + "github.com/gofiber/fiber/v2/log" + "github.com/golang-jwt/jwt/v5" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/config" + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" +) + +type ctxKey string + +const userCtxKey ctxKey = "__user__" + +func GetUserFromContext(ctx context.Context) *dto.User { + user, ok := ctx.Value(userCtxKey).(*dto.User) + if !ok { + return nil + } + return user +} + +func fetchUserID(ctx context.Context, url, token string) (string, error) { + cl := http.DefaultClient + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return "", err + } + req.Header.Add(fiber.HeaderAuthorization, fmt.Sprintf("Bearer %s", token)) + + resp, err := cl.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + var info struct { + Sub string `json:"sub"` + } + + if err := json.NewDecoder(resp.Body).Decode(&info); err != nil { + return "", err + } + + return info.Sub, nil +} + +func authMiddleware(security config.Security) fiber.Handler { + return jwtware.New(jwtware.Config{ + SuccessHandler: func(ctx *fiber.Ctx) error { + claims := ctx.Locals("claims") + + tok, ok := claims.(*jwt.Token) + if !ok { + return errors.New("unauthorized") // TODO: normal error + } + + var sub string + var err error + if security.UserInfoURL != "" { + sub, err = fetchUserID(ctx.Context(), security.UserInfoURL, tok.Raw) + } else { + sub, err = tok.Claims.GetSubject() + } + if err != nil { + return err + } + + if sub == "" { + log.Error("unable to fetch user ID nor from token nor from auth provider") + return errors.New("unauthorized") // TODO: normal error + } + + ctx.Locals(userCtxKey, &dto.User{ + ID: sub, + }) + return ctx.Next() + }, + ContextKey: "claims", + JWKSetURLs: []string{security.JWKSURL}, + }) +} diff --git a/internal/minireader/transport/http/dto.go b/internal/minireader/transport/http/dto.go new file mode 100644 index 0000000..d3cb695 --- /dev/null +++ b/internal/minireader/transport/http/dto.go @@ -0,0 +1,27 @@ +package http + +import ( + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" +) + +type User = dto.User + +type Feed = dto.Feed +type CreateFeed = dto.CreateFeed +type UpdateFeed = dto.UpdateFeed + +type Item = dto.Item + +type Meta struct { + Total int `json:"total"` + Cursor string `json:"cursor,omitempty"` +} + +type Result[T any] struct { + Data T `json:"data"` +} + +type Results[T any] struct { + Data []T `json:"data"` + Meta Meta `json:"meta"` +} diff --git a/internal/minireader/transport/http/handlers.go b/internal/minireader/transport/http/handlers.go new file mode 100644 index 0000000..74dc167 --- /dev/null +++ b/internal/minireader/transport/http/handlers.go @@ -0,0 +1,86 @@ +package http + +import ( + "github.com/gofiber/fiber/v2" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" +) + +func (s *Server) UserInfo(ctx *fiber.Ctx) error { + user := GetUserFromContext(ctx.Context()) + + if err := s.reader.EnsureUser(ctx.Context(), user); err != nil { + return err + } + + return ctx.JSON(Result[*User]{ + Data: user, + }) +} + +func (s *Server) ImportOPML(ctx *fiber.Ctx) error { + return nil +} + +func (s *Server) ListAllUserFeeds(ctx *fiber.Ctx) error { + user := GetUserFromContext(ctx.Context()) + + feeds, total, err := s.reader.ListAllUserFeeds(ctx.Context(), user) + if err != nil { + return err + } + + return ctx.JSON(Results[Feed]{ + Data: feeds, + Meta: Meta{ + Total: total, + }, + }) +} + +func (s *Server) CreateNewFeed(ctx *fiber.Ctx) error { + user := GetUserFromContext(ctx.Context()) + + var input dto.CreateFeed + if err := ctx.BodyParser(&input); err != nil { + return fiber.ErrBadRequest + } + + feed, err := s.reader.CreateSubscription(ctx.Context(), user, &input) + if err != nil { + // TODO: log + return err + } + + return ctx.JSON(Result[Feed]{ + Data: feed, + }) +} + +func (s *Server) GetFeedInfo(ctx *fiber.Ctx) error { + return nil +} + +func (s *Server) UpdateFeedInfo(ctx *fiber.Ctx) error { + return nil +} + +func (s *Server) DeleteFeed(ctx *fiber.Ctx) error { + return nil +} + +func (s *Server) ListFeedItems(ctx *fiber.Ctx) error { + return nil +} + +func (s *Server) GetItemInfo(ctx *fiber.Ctx) error { + return nil +} + +func (s *Server) MarkItemReaded(ctx *fiber.Ctx) error { + return nil +} + +func (s *Server) MarkItemUnreaded(ctx *fiber.Ctx) error { + return nil +} diff --git a/internal/minireader/transport/http/options.go b/internal/minireader/transport/http/options.go new file mode 100644 index 0000000..cbc0e0d --- /dev/null +++ b/internal/minireader/transport/http/options.go @@ -0,0 +1,43 @@ +package http + +import ( + "log/slog" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/config" + "borodyadka.dev/borodyadka/minireader/internal/pkg/validator" +) + +type Option func(*Server) + +type Services struct { + Reader reader `validate:"required"` +} + +func WithServices(services Services) Option { + return func(server *Server) { + if err := validator.Struct(services); err != nil { + panic("one or more services passed into HTTP Server instance is invalid or not defined") + } + + server.reader = services.Reader + } +} + +func WithConfig(conf config.HTTP) Option { + return func(server *Server) { + server.config = conf + } +} + +func WithSecurity(sec config.Security) Option { + // TODO: security MUST be a security provider, not a JWKS URL! + return func(server *Server) { + server.sec = sec + } +} + +func WithLogger(logger *slog.Logger) Option { + return func(server *Server) { + server.logger = logger + } +} diff --git a/internal/minireader/transport/http/server.go b/internal/minireader/transport/http/server.go new file mode 100644 index 0000000..595c64e --- /dev/null +++ b/internal/minireader/transport/http/server.go @@ -0,0 +1,103 @@ +package http + +import ( + "context" + "log/slog" + + "github.com/gofiber/fiber/v2" + slogfiber "github.com/samber/slog-fiber" + + "borodyadka.dev/borodyadka/minireader/internal/minireader/config" + "borodyadka.dev/borodyadka/minireader/internal/minireader/dto" +) + +type reader interface { + EnsureUser(ctx context.Context, user *dto.User) error + + ListAllUserFeeds(ctx context.Context, user *dto.User) ([]dto.Feed, int, error) + CreateSubscription(ctx context.Context, user *dto.User, input *dto.CreateFeed) (dto.Feed, error) +} + +type Server struct { + server *fiber.App + config config.HTTP + sec config.Security + reader reader + logger *slog.Logger + injections [][2]any +} + +func New(opts ...Option) *Server { + server := &Server{ + server: fiber.New(fiber.Config{ + ReadTimeout: 0, // TODO + WriteTimeout: 0, // TODO + IdleTimeout: 0, // TODO + ReadBufferSize: 0, // TODO + WriteBufferSize: 0, // TODO + ErrorHandler: nil, // TODO + DisableStartupMessage: true, + DisablePreParseMultipartForm: true, + EnableSplittingOnParsers: false, + }), + } + + for _, opt := range opts { + opt(server) + } + + server.init() + + return server +} + +func (s *Server) Inject(key, value any) { + s.injections = append(s.injections, [2]any{key, value}) +} + +func (s *Server) init() { + s.server.Use(func(c *fiber.Ctx) error { + for _, v := range s.injections { + c.Locals(v[0], v[1]) + } + return c.Next() + }) + + if s.config.AccessLog { + s.server.Use(slogfiber.New(s.logger)) + } + + if s.sec.JWKSURL != "" { + s.server.Use(authMiddleware(s.sec)) + } + + api := s.server.Group(s.config.APIPrefix) + v1 := api.Group("/v1") + + user := v1.Group("/me") + user.Get("/", s.UserInfo) // TODO: init user if not exists + + feeds := v1.Group("/feeds") + feeds.Get("/", s.ListAllUserFeeds) + feeds.Post("/", s.CreateNewFeed) + feeds.Post("/_import/opml", s.ImportOPML) + feeds.Get("/:feed", s.GetFeedInfo) // there is some special :feed keys, such as "_all" + feeds.Patch("/:feed", s.UpdateFeedInfo) + feeds.Delete("/:feed", s.DeleteFeed) + + items := feeds.Group("/:feed/items") + items.Get("/", s.ListFeedItems) + items.Get("/:item", s.GetItemInfo) + items.Put("/:item/readed", s.MarkItemReaded) + items.Delete("/:item/readed", s.MarkItemUnreaded) +} + +func (s *Server) Start() error { + s.logger.Info("listen", "addr", s.config.Listen) + return s.server.Listen(s.config.Listen) +} + +func (s *Server) Shutdown() error { + s.logger.Info("shutdown") + return s.server.Shutdown() +} diff --git a/internal/pkg/errx/details.go b/internal/pkg/errx/details.go new file mode 100644 index 0000000..8c5e4b3 --- /dev/null +++ b/internal/pkg/errx/details.go @@ -0,0 +1,66 @@ +package errx + +// Package `errx` named like this to not confuse with built-in `errors` package. + +import "errors" + +type detailedError struct { + error + details []any +} + +func (e *detailedError) Unwrap() error { + return e.error +} + +func (e *detailedError) Details() []any { + return e.details +} + +// WithDetails creates new error value with embedded details +func WithDetails(err error, details ...any) error { + derr := &detailedError{} + if errors.As(err, &derr) { + derr.details = append(derr.details, details...) + return err + } + + return &detailedError{ + error: err, + details: details, + } +} + +// Details will extract details array from error +func Details(err error) []any { + if err == nil { + return nil + } + + // detailedError on top level of errors chain + if derr, ok := err.(interface{ Details() []any }); ok { + return derr.Details() + } + + // detailed error were wrapped into another error, and we need to dig it up + derr := &detailedError{} + if errors.As(err, &derr) { + return derr.Details() + } + + return nil +} + +// LookupDetails will find some specific details by its type +// NOTE: I'm not sure it's good idea or expected behaviour for function named Lookup +func LookupDetails[T any](err error) T { + var empty T + + for _, d := range Details(err) { + if v, ok := d.(T); ok { + return v + } + } + + return empty +} diff --git a/internal/pkg/errx/details_test.go b/internal/pkg/errx/details_test.go new file mode 100644 index 0000000..a2690ed --- /dev/null +++ b/internal/pkg/errx/details_test.go @@ -0,0 +1,101 @@ +package errx + +import ( + "errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/suite" + "testing" +) + +type DetailsSuite struct { + suite.Suite +} + +func (suite *DetailsSuite) TestUnwrap() { + err := errors.New("test") + + tests := []struct { + err error + want error + }{ + { + err: &detailedError{}, + want: nil, + }, + { + err: WithDetails(err), + want: err, + }, + } + + for _, test := range tests { + suite.Run("", func() { + assert.Equal(suite.T(), test.want, errors.Unwrap(test.err)) + }) + } +} + +func (suite *DetailsSuite) TestIs() { + errSame := errors.New("test same") + errWrong := errors.New("test wrong") + + tests := []struct { + err error + want bool + }{ + { + err: WithDetails(errSame), + want: true, + }, + { + err: WithDetails(errWrong), + want: false, + }, + } + + for _, test := range tests { + suite.Run("", func() { + assert.Equal(suite.T(), test.want, errors.Is(test.err, errSame)) + }) + } +} + +func (suite *DetailsSuite) TestDetails() { + err := errors.New("test") + + tests := []struct { + err error + want []any + }{ + { + err: WithDetails(err), + want: nil, + }, + { + err: WithDetails(err, 1, 2, 3), + want: []any{1, 2, 3}, + }, + { + err: WithDetails(WithDetails(err, 4), 5, 6), + want: []any{4, 5, 6}, + }, + } + + for _, test := range tests { + suite.Run("", func() { + assert.Equal(suite.T(), test.want, Details(test.err)) + }) + } +} + +func (suite *DetailsSuite) TestLookupDetails() { + err := WithDetails(errors.New("test"), true, 2, "3") + + assert.Equal(suite.T(), true, LookupDetails[bool](err)) + assert.Equal(suite.T(), 2, LookupDetails[int](err)) + assert.Equal(suite.T(), "3", LookupDetails[string](err)) +} + +func TestDetails(t *testing.T) { + suite.Run(t, new(DetailsSuite)) +} diff --git a/internal/pkg/utils/deref.go b/internal/pkg/utils/deref.go new file mode 100644 index 0000000..bcf2203 --- /dev/null +++ b/internal/pkg/utils/deref.go @@ -0,0 +1,9 @@ +package utils + +func DerefOrDefault[T any](val *T, fallback T) T { + if val == nil { + return fallback + } + + return *val +} diff --git a/internal/pkg/utils/slices.go b/internal/pkg/utils/slices.go new file mode 100644 index 0000000..2f92e1b --- /dev/null +++ b/internal/pkg/utils/slices.go @@ -0,0 +1,13 @@ +package utils + +type Mapper[T, R any] func(T) R + +func MapSlice[T, R any](input []T, mapper Mapper[T, R]) []R { + output := make([]R, len(input)) + + for i := range input { + output[i] = mapper(input[i]) + } + + return output +} diff --git a/internal/pkg/validator/validator.go b/internal/pkg/validator/validator.go new file mode 100644 index 0000000..e1cea1a --- /dev/null +++ b/internal/pkg/validator/validator.go @@ -0,0 +1,19 @@ +package validator + +import ( + "github.com/go-playground/validator/v10" +) + +var ( + v = validator.New() +) + +func Struct(val any) error { + return v.Struct(val) +} + +func init() { + _ = v.RegisterValidation("optional", func(f validator.FieldLevel) bool { + return true + }) +} diff --git a/migrations/20231021105313_init.sql b/migrations/20231021105313_init.sql new file mode 100644 index 0000000..e51c6c8 --- /dev/null +++ b/migrations/20231021105313_init.sql @@ -0,0 +1,145 @@ +-- +goose Up +CREATE TABLE "users" +( + "id" TEXT PRIMARY KEY +); +COMMENT ON COLUMN "users"."id" + IS 'ID of the user. It is came from external system, so we can save it as a text field only'; + + +CREATE TYPE FEED_PROVIDER AS ENUM ('rss'); + + +CREATE TABLE "feeds" +( + "id" UUID NOT NULL PRIMARY KEY DEFAULT gen_random_uuid(), + "title" TEXT NOT NULL DEFAULT '', + "provider" FEED_PROVIDER NOT NULL, + "source" TEXT NOT NULL, + "last_fetched_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "fetch_interval" INTERVAL NOT NULL DEFAULT '1 hour'::INTERVAL, + "next_fetch_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "last_error" TEXT DEFAULT NULL, + "errors_count" INTEGER NOT NULL DEFAULT 0 +); +CREATE UNIQUE INDEX "u_feeds__source" ON "feeds" ("source"); +CREATE INDEX "i_feeds__next_fetch_at" ON "feeds" ("next_fetch_at"); +COMMENT ON COLUMN "feeds"."id" IS 'Feed ID'; +COMMENT ON COLUMN "feeds"."title" IS 'Title'; +COMMENT ON COLUMN "feeds"."provider" + IS 'Data provider, such as RSS or some another supported source type'; +COMMENT ON COLUMN "feeds"."source" IS 'Address of feed data'; +COMMENT ON COLUMN "feeds"."last_fetched_at" IS 'Date of last fetch of current feed'; +COMMENT ON COLUMN "feeds"."fetch_interval" + IS 'Period of fetching new data. Will be shorten for active feeds with lots of updates, and longen for inactive or errored feeds'; +COMMENT ON COLUMN "feeds"."next_fetch_at" IS 'Computed field used only for speed-up queries. DO NOT EDIT!'; +COMMENT ON COLUMN "feeds"."last_error" IS 'Text description of last fetching error'; +COMMENT ON COLUMN "feeds"."errors_count" + IS 'Number of fetching errors. Will increase for every time fetch was failed. Will be reseted if fetch was successful'; + +-- +goose StatementBegin +CREATE FUNCTION feeds_set_next_fetch_at() RETURNS trigger AS +$feeds_set_next_fetch_at$ +BEGIN + NEW."next_fetch_at" := NEW."last_fetched_at" + NEW."fetch_interval"; + RETURN NEW; +END; +$feeds_set_next_fetch_at$ LANGUAGE plpgsql; +-- +goose StatementEnd + +CREATE TRIGGER "feeds_set_next_fetch_at" + BEFORE INSERT OR UPDATE + ON "feeds" + FOR EACH ROW +EXECUTE FUNCTION feeds_set_next_fetch_at(); + + +CREATE TABLE "feed_icons" +( + "feed_id" UUID NOT NULL PRIMARY KEY REFERENCES "feeds" ("id") ON DELETE CASCADE, + "mimetype" TEXT NOT NULL, + "icon" BYTEA DEFAULT NULL +); +COMMENT ON COLUMN "feed_icons"."feed_id" IS 'ID of feed'; +COMMENT ON COLUMN "feed_icons"."mimetype" IS 'Mimetype of the icon'; +COMMENT ON COLUMN "feed_icons"."icon" IS 'Icon of the feed (favicon)'; + + +CREATE TABLE "subscriptions" +( + "user_id" TEXT NOT NULL REFERENCES "users" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "feed_id" UUID NOT NULL REFERENCES "feeds" ("id") ON DELETE RESTRICT, + "title" TEXT DEFAULT NULL +); +CREATE UNIQUE INDEX "u_subscription__user_feed" ON "subscriptions" ("user_id", "feed_id"); +CREATE INDEX "i_subscription__user" ON "subscriptions" ("user_id"); +COMMENT ON COLUMN "subscriptions"."user_id" IS 'ID of user'; +COMMENT ON COLUMN "subscriptions"."feed_id" IS 'ID of feed'; +COMMENT ON COLUMN "subscriptions"."title" IS 'User-defined name of the feed'; + + +CREATE TABLE "feed_items" +( + "id" UUID NOT NULL PRIMARY KEY DEFAULT gen_random_uuid(), + "feed_id" UUID NOT NULL REFERENCES "feeds" ("id") ON DELETE CASCADE, + "created_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "fetched_at" TIMESTAMPTZ NOT NULL DEFAULT NOW(), + "uid" TEXT NOT NULL, + "title" TEXT NOT NULL, + "content" TEXT NOT NULL DEFAULT '', + "link" TEXT NOT NULL DEFAULT '', + "author" TEXT NOT NULL DEFAULT '', + "tags" TEXT[] NOT NULL DEFAULT '{}' +); +CREATE INDEX "i_feed_items__feed" ON "feed_items" ("feed_id"); +CREATE UNIQUE INDEX "u_feed_items__feed_uid" ON "feed_items" ("feed_id", "uid"); +CREATE INDEX "i_feed_items__fetched" ON "feed_items" ("fetched_at" DESC); +COMMENT ON COLUMN "feed_items"."id" IS 'Item ID'; +COMMENT ON COLUMN "feed_items"."feed_id" IS 'Feed ID'; +COMMENT ON COLUMN "feed_items"."created_at" IS 'Item creation time (taken from feed info)'; +COMMENT ON COLUMN "feed_items"."fetched_at" + IS 'Time when item was fetched and saved into database. This field will be used for cursor-based pagination'; +COMMENT ON COLUMN "feed_items"."uid" IS 'Unique ID of item in feed'; +COMMENT ON COLUMN "feed_items"."title" IS 'Title'; +COMMENT ON COLUMN "feed_items"."content" + IS 'Sanitized HTML content: stripped danger tags, links modified to open in new tab and to strip referrer'; +COMMENT ON COLUMN "feed_items"."link" IS 'Link to full article content'; +COMMENT ON COLUMN "feed_items"."author" IS 'Name of the author of article (taken from feed info)'; +COMMENT ON COLUMN "feed_items"."tags" IS 'List of normalized (lowercased, trimmed) tags (taken from feed info)'; + + +CREATE TABLE "feed_item_attachments" +( + "id" BIGINT NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + "item_id" UUID NOT NULL REFERENCES "feed_items" ("id") ON DELETE CASCADE, + "mimetype" TEXT NOT NULL, + "url" TEXT NOT NULL, + "size" BIGINT NOT NULL DEFAULT 0 +); + + +CREATE TABLE "subscription_items" +( + "user_id" TEXT NOT NULL REFERENCES "users" ("id") ON DELETE CASCADE ON UPDATE CASCADE, + "feed_id" UUID NOT NULL REFERENCES "feeds" ("id") ON DELETE CASCADE, + "item_id" UUID NOT NULL REFERENCES "feed_items" ("id") ON DELETE CASCADE, + "is_readed" BOOLEAN NOT NULL DEFAULT TRUE +); +CREATE UNIQUE INDEX "u_subscription_item__user_feed_item" ON "subscription_items" ("user_id", "feed_id", "item_id"); +COMMENT ON COLUMN "subscription_items"."user_id" IS 'ID of subscribed user (used in subscription composite key)'; +COMMENT ON COLUMN "subscription_items"."feed_id" IS 'ID of related feed (used in subscription composite key)'; +COMMENT ON COLUMN "subscription_items"."item_id" IS 'ID of specific feed item'; +COMMENT ON COLUMN "subscription_items"."is_readed" + IS 'This flag means item is marked as read and should not appear in feed (except special query). This flag MUST be treated as FALSE if row does not exists'; + + +-- +goose Down +DROP TABLE "subscription_items"; +DROP TABLE "feed_item_attachments"; +DROP TABLE "feed_items"; +DROP TABLE "subscriptions"; +DROP TABLE "feed_icons"; +DROP TABLE "feeds"; +DROP FUNCTION "feeds_set_next_fetch_at"; +DROP TYPE FEED_PROVIDER; +DROP TABLE "users"; diff --git a/migrations/embed.go b/migrations/embed.go new file mode 100644 index 0000000..0cd5513 --- /dev/null +++ b/migrations/embed.go @@ -0,0 +1,9 @@ +package migrations + +import ( + "embed" + _ "embed" +) + +//go:embed *.sql +var Migrations embed.FS diff --git a/sqlc.yaml b/sqlc.yaml new file mode 100644 index 0000000..a2e6641 --- /dev/null +++ b/sqlc.yaml @@ -0,0 +1,32 @@ +version: "2" +sql: + - engine: "postgresql" + queries: "internal/minireader/repositories/sql/query.sql" + schema: "migrations" + gen: + go: + package: "sql" + sql_package: "pgx/v5" + out: "internal/minireader/repositories/sql" + emit_empty_slices: true + rename: + uid: UID + overrides: + # common types + - db_type: "timestamptz" + nullable: true + engine: "postgresql" + go_type: + import: "gopkg.in/guregu/null.v4" + package: "null" + type: "Time" + - db_type: "uuid" + engine: "postgresql" + go_type: "github.com/google/uuid.UUID" + - db_type: "integer" + engine: "postgresql" + go_type: "int" + # application-specific types + - db_type: "feed_provider" + engine: "postgresql" + go_type: "borodyadka.dev/borodyadka/minireader/internal/minireader/dto.Provider"